[Frontend] Refactor ServingTokenization entrypoint. (#46022)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

tests/entrypoints/serve/tokenize/test_serving_tokenization.py

@@ -15,7 +15,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
     TokenizeChatRequest,
     TokenizeCompletionRequest,
 )
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
 from vllm.v1.engine.async_llm import AsyncLLM
 
 MODEL_NAME = "openai-community/gpt2"
@@ -58,7 +58,7 @@ class MockModelConfig:
         return self.diff_sampling_param or {}
 
 
-def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization:
+def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
     models = OpenAIServingModels(
         engine_client=engine,
         base_model_paths=BASE_MODEL_PATHS,
@@ -71,8 +71,7 @@ def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization:
         chat_template=None,
         chat_template_content_format="auto",
     )
-    return OpenAIServingTokenization(
-        engine,
+    return ServingTokenization(
         models,
         openai_serving_render=serving_render,
         request_logger=None,

vllm/entrypoints/anthropic/api_router.py

@@ -61,7 +61,7 @@ def translate_error_response(response: ErrorResponse) -> JSONResponse:
 async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
     handler = messages(raw_request)
     if handler is None:
-        base_server = raw_request.app.state.openai_serving_tokenization
+        base_server = raw_request.app.state.serving_tokenization
         error = base_server.create_error_response(
             NotImplementedError("The model does not support Messages API")
         )
@@ -107,7 +107,7 @@ async def create_messages(request: AnthropicMessagesRequest, raw_request: Reques
 async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request):
     handler = messages(raw_request)
     if handler is None:
-        base_server = raw_request.app.state.openai_serving_tokenization
+        base_server = raw_request.app.state.serving_tokenization
         error = base_server.create_error_response(
             NotImplementedError("The model does not support Messages API")
         )

vllm/entrypoints/openai/api_server.py

@@ -34,7 +34,7 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
 from vllm.entrypoints.serve.utils.api_utils import (
     cli_env_setup,
     log_non_default_args,
@@ -376,8 +376,7 @@ async def init_app_state(
         log_error_stack=args.log_error_stack,
     )
 
-    state.openai_serving_tokenization = OpenAIServingTokenization(
-        engine_client,
+    state.serving_tokenization = ServingTokenization(
         state.openai_serving_models,
         state.openai_serving_render,
         request_logger=request_logger,
@@ -461,9 +460,15 @@ async def init_render_app_state(
     )
 
     state.openai_serving_models = model_registry
-
-    # Expose tokenization via the render handler (no engine required).
-    state.openai_serving_tokenization = state.openai_serving_render
+    state.serving_tokenization = ServingTokenization(
+        model_registry,
+        state.openai_serving_render,
+        request_logger=request_logger,
+        chat_template=resolved_chat_template,
+        chat_template_content_format=args.chat_template_content_format,
+        default_chat_template_kwargs=args.default_chat_template_kwargs,
+        trust_request_chat_template=args.trust_request_chat_template,
+    )
 
     state.vllm_config = vllm_config
     # Disable stats logging — there is no engine to poll.

vllm/entrypoints/openai/engine/serving.py

@@ -5,25 +5,19 @@ import time
 from collections.abc import Awaitable, Mapping
 from dataclasses import dataclass, field
 from http import HTTPStatus
-from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
+from typing import ClassVar, Generic, TypeVar
 
 from fastapi import Request
 from pydantic import ConfigDict
 from starlette.datastructures import Headers
 
-import vllm.envs as envs
-from vllm.config import ModelConfig
 from vllm.engine.protocol import EngineClient
-from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
 from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
 from vllm.entrypoints.openai.chat_completion.protocol import (
-    BatchChatCompletionRequest,
     ChatCompletionRequest,
-    ChatCompletionResponse,
 )
 from vllm.entrypoints.openai.completion.protocol import (
     CompletionRequest,
-    CompletionResponse,
 )
 from vllm.entrypoints.openai.engine.protocol import (
     ErrorResponse,
@@ -31,81 +25,22 @@ from vllm.entrypoints.openai.engine.protocol import (
 )
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
-from vllm.entrypoints.serve.tokenize.protocol import (
-    DetokenizeRequest,
-    TokenizeChatRequest,
-    TokenizeCompletionRequest,
-    TokenizeResponse,
-)
-from vllm.entrypoints.serve.utils.error_response import create_error_response
+from vllm.entrypoints.serve.engine.serving import BaseServing
+from vllm.entrypoints.serve.engine.typing import AnyRequest
 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
-from vllm.entrypoints.speech_to_text.transcription.protocol import (
-    TranscriptionRequest,
-    TranscriptionResponse,
-)
-from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
-from vllm.inputs import EngineInput, PromptType
+from vllm.inputs import EngineInput
 from vllm.logger import init_logger
 from vllm.logprobs import Logprob, PromptLogprobs
 from vllm.lora.request import LoRARequest
-from vllm.renderers import ChatParams, TokenizeParams
-from vllm.renderers.inputs.preprocess import (
-    extract_prompt_components,
-    extract_prompt_len,
-)
-from vllm.sampling_params import BeamSearchParams, SamplingParams
 from vllm.tokenizers import TokenizerLike
 from vllm.tracing import (
     contains_trace_headers,
     extract_trace_headers,
     log_tracing_disabled_warning,
 )
-from vllm.utils import random_uuid
 
 logger = init_logger(__name__)
 
-
-class RendererRequest(Protocol):
-    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
-        raise NotImplementedError
-
-
-class RendererChatRequest(RendererRequest, Protocol):
-    def build_chat_params(
-        self,
-        default_template: str | None,
-        default_template_content_format: ChatTemplateContentFormatOption,
-    ) -> ChatParams:
-        raise NotImplementedError
-
-
-CompletionLikeRequest: TypeAlias = (
-    CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
-)
-
-ChatLikeRequest: TypeAlias = (
-    ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
-)
-
-SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
-
-AnyRequest: TypeAlias = (
-    CompletionLikeRequest
-    | ChatLikeRequest
-    | SpeechToTextRequest
-    | ResponsesRequest
-    | GenerateRequest
-)
-
-AnyResponse: TypeAlias = (
-    CompletionResponse
-    | ChatCompletionResponse
-    | TranscriptionResponse
-    | TokenizeResponse
-    | GenerateResponse
-)
-
 RequestT = TypeVar("RequestT", bound=AnyRequest)
 _T = TypeVar("_T")
 
@@ -122,7 +57,7 @@ class ServeContext(Generic[RequestT]):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
-class OpenAIServing(BeamSearchOnlineMixin):
+class OpenAIServing(BaseServing, BeamSearchOnlineMixin):
     request_id_prefix: ClassVar[str] = """
     A short string prepended to every request’s ID.
     """
@@ -135,15 +70,14 @@ class OpenAIServing(BeamSearchOnlineMixin):
         request_logger: RequestLogger | None,
         return_tokens_as_token_ids: bool = False,
     ):
-        super().__init__()
+        super().__init__(
+            models=models,
+            model_config=engine_client.model_config,
+            request_logger=request_logger,
+        )
 
         self.engine_client = engine_client
-        self.models = models
-
-        self.request_logger = request_logger
         self.return_tokens_as_token_ids = return_tokens_as_token_ids
-
-        self.model_config = engine_client.model_config
         self.renderer = engine_client.renderer
         self.input_processor = engine_client.input_processor
         vllm_config = getattr(engine_client, "vllm_config", None)
@@ -163,15 +97,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
             # Never fail server startup over the fingerprint.
             self.system_fingerprint = None
 
-    @staticmethod
-    def create_error_response(
-        message: str | Exception,
-        err_type: str = "BadRequestError",
-        status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
-        param: str | None = None,
-    ) -> ErrorResponse:
-        return create_error_response(message, err_type, status_code, param)
-
     def create_streaming_error_response(
         self,
         message: str | Exception,
@@ -208,167 +133,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
             status_code=e.status_code,
         )
 
-    async def _check_model(
-        self,
-        request: AnyRequest,
-    ) -> ErrorResponse | None:
-        error_response = None
-
-        if self._is_model_supported(request.model):
-            return None
-        if request.model in self.models.lora_requests:
-            return None
-        if (
-            envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
-            and request.model
-            and (load_result := await self.models.resolve_lora(request.model))
-        ):
-            if isinstance(load_result, LoRARequest):
-                return None
-            if (
-                isinstance(load_result, ErrorResponse)
-                and load_result.error.code == HTTPStatus.BAD_REQUEST.value
-            ):
-                error_response = load_result
-
-        return error_response or self.create_error_response(
-            message=f"The model `{request.model}` does not exist.",
-            err_type="NotFoundError",
-            status_code=HTTPStatus.NOT_FOUND,
-            param="model",
-        )
-
-    def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
-        """Determine if there are any active default multimodal loras."""
-        # TODO: Currently this is only enabled for chat completions
-        # to be better aligned with only being enabled for .generate
-        # when run offline. It would be nice to support additional
-        # tasks types in the future.
-        message_types = self._get_message_types(request)
-        default_mm_loras = set()
-
-        for lora in self.models.lora_requests.values():
-            # Best effort match for default multimodal lora adapters;
-            # There is probably a better way to do this, but currently
-            # this matches against the set of 'types' in any content lists
-            # up until '_', e.g., to match audio_url -> audio
-            if lora.lora_name in message_types:
-                default_mm_loras.add(lora)
-
-        # Currently only support default modality specific loras if
-        # we have exactly one lora matched on the request.
-        if len(default_mm_loras) == 1:
-            return default_mm_loras.pop()
-        return None
-
-    def _maybe_get_adapters(
-        self,
-        request: AnyRequest,
-        supports_default_mm_loras: bool = False,
-    ) -> LoRARequest | None:
-        if request.model in self.models.lora_requests:
-            return self.models.lora_requests[request.model]
-
-        # Currently only support default modality specific loras
-        # if we have exactly one lora matched on the request.
-        if supports_default_mm_loras:
-            default_mm_lora = self._get_active_default_mm_loras(request)
-            if default_mm_lora is not None:
-                return default_mm_lora
-
-        if self._is_model_supported(request.model):
-            return None
-
-        # if _check_model has been called earlier, this will be unreachable
-        raise ValueError(f"The model `{request.model}` does not exist.")
-
-    def _get_message_types(self, request: AnyRequest) -> set[str]:
-        """Retrieve the set of types from message content dicts up
-        until `_`; we use this to match potential multimodal data
-        with default per modality loras.
-        """
-        message_types: set[str] = set()
-
-        if not hasattr(request, "messages"):
-            return message_types
-
-        messages = request.messages
-        if messages is None or isinstance(messages, (str, bytes)):
-            return message_types
-
-        for message in messages:
-            if (
-                isinstance(message, dict)
-                and "content" in message
-                and isinstance(message["content"], list)
-            ):
-                for content_dict in message["content"]:
-                    if "type" in content_dict:
-                        message_types.add(content_dict["type"].split("_")[0])
-        return message_types
-
-    def _validate_chat_template(
-        self,
-        request_chat_template: str | None,
-        chat_template_kwargs: dict[str, Any] | None,
-        trust_request_chat_template: bool,
-    ) -> ErrorResponse | None:
-        if not trust_request_chat_template and (
-            request_chat_template is not None
-            or (
-                chat_template_kwargs
-                and chat_template_kwargs.get("chat_template") is not None
-            )
-        ):
-            return self.create_error_response(
-                "Chat template is passed with request, but "
-                "--trust-request-chat-template is not set. "
-                "Refused request with untrusted chat template."
-            )
-        return None
-
-    @staticmethod
-    def _prepare_extra_chat_template_kwargs(
-        request_chat_template_kwargs: dict[str, Any] | None = None,
-        default_chat_template_kwargs: dict[str, Any] | None = None,
-    ) -> dict[str, Any]:
-        """Helper to merge server-default and request-specific chat template kwargs."""
-        request_chat_template_kwargs = request_chat_template_kwargs or {}
-        if default_chat_template_kwargs is None:
-            return request_chat_template_kwargs
-        # Apply server defaults first, then request kwargs override.
-        return default_chat_template_kwargs | request_chat_template_kwargs
-
-    def _extract_prompt_components(self, prompt: PromptType | EngineInput):
-        return extract_prompt_components(self.model_config, prompt)
-
-    def _extract_prompt_text(self, prompt: PromptType | EngineInput):
-        return self._extract_prompt_components(prompt).text
-
-    def _extract_prompt_len(self, prompt: EngineInput):
-        return extract_prompt_len(self.model_config, prompt)
-
-    def _log_inputs(
-        self,
-        request_id: str,
-        inputs: PromptType | EngineInput,
-        params: SamplingParams | BeamSearchParams | None,
-        lora_request: LoRARequest | None,
-    ) -> None:
-        if self.request_logger is None:
-            return
-
-        components = self._extract_prompt_components(inputs)
-
-        self.request_logger.log_inputs(
-            request_id,
-            components.text,
-            components.token_ids,
-            components.embeds,
-            params=params,
-            lora_request=lora_request,
-        )
-
     async def _get_trace_headers(
         self,
         headers: Headers,
@@ -383,18 +147,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
 
         return None
 
-    @staticmethod
-    def _base_request_id(
-        raw_request: Request | None, default: str | None = None
-    ) -> str | None:
-        """Pulls the request id to use from a header, if provided"""
-        if raw_request is not None and (
-            (req_id := raw_request.headers.get("X-Request-Id")) is not None
-        ):
-            return req_id
-
-        return random_uuid() if default is None else default
-
     @staticmethod
     def _get_data_parallel_rank(raw_request: Request | None) -> int | None:
         """Pulls the data parallel rank from a header, if provided"""
@@ -464,13 +216,6 @@ class OpenAIServing(BeamSearchOnlineMixin):
 
         return tokenizer.decode([token_id])
 
-    def _is_model_supported(self, model_name: str | None) -> bool:
-        if not model_name:
-            return True
-        if envs.VLLM_SKIP_MODEL_NAME_VALIDATION:
-            return True
-        return self.models.is_base_model(model_name)
-
 
 def format_token_id_placeholder(token_id: int) -> str:
     return f"token_id:{token_id}"

vllm/entrypoints/openai/models/serving.py

@@ -42,6 +42,10 @@ class OpenAIModelRegistry:
     ) -> None:
         self.model_config = model_config
         self.base_model_paths = base_model_paths
+        self.lora_requests: dict[str, LoRARequest] = {}
+
+    def model_name(self, lora_request: LoRARequest | None = None) -> str:
+        return self.base_model_paths[0].name
 
     def is_base_model(self, model_name: str) -> bool:
         return any(model.name == model_name for model in self.base_model_paths)
@@ -72,6 +76,9 @@ class OpenAIModelRegistry:
             ]
         )
 
+    async def resolve_lora(self, lora_name: str):
+        raise RuntimeError("The OpenAIModelRegistry has no LoRA support.")
+
 
 class OpenAIServingModels:
     """Shared instance to hold data about the loaded base model(s) and adapters.

vllm/entrypoints/pooling/base/io_processor.py

@@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import (
     ChatTemplateContentFormatOption,
     ConversationMessage,
 )
-from vllm.entrypoints.openai.engine.serving import RendererChatRequest, RendererRequest
+from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
 from vllm.inputs import EngineInput, SingletonPrompt
 from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs
 from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq

vllm/entrypoints/pooling/base/serving.py

@@ -52,7 +52,7 @@ class PoolingServingBase(ABC):
         self.engine_client = engine_client
         self.models = models
         self.model_config = models.model_config
-        self.renderer = models.renderer
+        self.renderer = engine_client.renderer
         self.vllm_config = engine_client.vllm_config
         self.max_model_len = self.model_config.max_model_len
         self.request_logger = request_logger
@@ -61,7 +61,7 @@ class PoolingServingBase(ABC):
         self.chat_template_config = chat_template_config
 
         # Shared thread pool executor for preprocessing and postprocessing.
-        self._executor: Executor = models.renderer._executor
+        self._executor: Executor = self.renderer._executor
         self._preprocessing_async = make_async(
             self._preprocessing, executor=self._executor
         )

vllm/entrypoints/serve/disagg/api_router.py

@@ -20,7 +20,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
 from vllm.entrypoints.serve.disagg.serving import (
     ServingTokens,
 )
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
 from vllm.entrypoints.serve.utils.api_utils import (
     load_aware_call,
     validate_json_request,
@@ -31,8 +31,8 @@ from vllm.logger import init_logger
 logger = init_logger(__name__)
 
 
-def tokenization(request: Request) -> OpenAIServingTokenization:
-    return request.app.state.openai_serving_tokenization
+def tokenization(request: Request) -> ServingTokenization:
+    return request.app.state.serving_tokenization
 
 
 def generate_tokens(request: Request) -> ServingTokens | None:

vllm/entrypoints/serve/engine/serving.py

@@ -0,0 +1,193 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from http import HTTPStatus
+
+from fastapi import Request
+
+from vllm import PromptType, SamplingParams, envs
+from vllm.config import ModelConfig
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.openai.models.serving import (
+    OpenAIModelRegistry,
+    OpenAIServingModels,
+)
+from vllm.entrypoints.serve.engine.typing import AnyRequest
+from vllm.entrypoints.serve.utils.error_response import create_error_response
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.inputs import EngineInput
+from vllm.lora.request import LoRARequest
+from vllm.renderers.inputs.preprocess import (
+    extract_prompt_components,
+    extract_prompt_len,
+)
+from vllm.sampling_params import BeamSearchParams
+from vllm.utils import random_uuid
+
+
+class BaseServing:
+    def __init__(
+        self,
+        models: OpenAIServingModels | OpenAIModelRegistry,
+        model_config: ModelConfig,
+        request_logger: RequestLogger | None = None,
+    ):
+        self.models = models
+        self.model_config = model_config
+        self.request_logger = request_logger
+
+    async def _check_model(
+        self,
+        request: AnyRequest,
+    ) -> ErrorResponse | None:
+        error_response = None
+
+        if self._is_model_supported(request.model):
+            return None
+        if request.model in self.models.lora_requests:
+            return None
+        if (
+            envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
+            and request.model
+            and (load_result := await self.models.resolve_lora(request.model))
+        ):
+            if isinstance(load_result, LoRARequest):
+                return None
+            if (
+                isinstance(load_result, ErrorResponse)
+                and load_result.error.code == HTTPStatus.BAD_REQUEST.value
+            ):
+                error_response = load_result
+
+        return error_response or self.create_error_response(
+            message=f"The model `{request.model}` does not exist.",
+            err_type="NotFoundError",
+            status_code=HTTPStatus.NOT_FOUND,
+            param="model",
+        )
+
+    def _is_model_supported(self, model_name: str | None) -> bool:
+        if not model_name:
+            return True
+        if envs.VLLM_SKIP_MODEL_NAME_VALIDATION:
+            return True
+        return self.models.is_base_model(model_name)
+
+    @staticmethod
+    def create_error_response(
+        message: str | Exception,
+        err_type: str = "BadRequestError",
+        status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
+        param: str | None = None,
+    ) -> ErrorResponse:
+        return create_error_response(message, err_type, status_code, param)
+
+    def _extract_prompt_components(self, prompt: PromptType | EngineInput):
+        return extract_prompt_components(self.model_config, prompt)
+
+    def _extract_prompt_text(self, prompt: PromptType | EngineInput):
+        return self._extract_prompt_components(prompt).text
+
+    def _extract_prompt_len(self, prompt: EngineInput):
+        return extract_prompt_len(self.model_config, prompt)
+
+    def _log_inputs(
+        self,
+        request_id: str,
+        inputs: PromptType | EngineInput,
+        params: SamplingParams | BeamSearchParams | None,
+        lora_request: LoRARequest | None,
+    ) -> None:
+        if self.request_logger is None:
+            return
+
+        components = self._extract_prompt_components(inputs)
+
+        self.request_logger.log_inputs(
+            request_id,
+            components.text,
+            components.token_ids,
+            components.embeds,
+            params=params,
+            lora_request=lora_request,
+        )
+
+    @staticmethod
+    def _base_request_id(
+        raw_request: Request | None, default: str | None = None
+    ) -> str | None:
+        """Pulls the request id to use from a header, if provided"""
+        if raw_request is not None and (
+            (req_id := raw_request.headers.get("X-Request-Id")) is not None
+        ):
+            return req_id
+
+        return random_uuid() if default is None else default
+
+    def _get_message_types(self, request: AnyRequest) -> set[str]:
+        """Retrieve the set of types from message content dicts up
+        until `_`; we use this to match potential multimodal data
+        with default per modality loras.
+        """
+        message_types: set[str] = set()
+
+        if not hasattr(request, "messages"):
+            return message_types
+
+        messages = request.messages
+        if messages is None or isinstance(messages, (str, bytes)):
+            return message_types
+
+        for message in messages:
+            if (
+                isinstance(message, dict)
+                and "content" in message
+                and isinstance(message["content"], list)
+            ):
+                for content_dict in message["content"]:
+                    if "type" in content_dict:
+                        message_types.add(content_dict["type"].split("_")[0])
+        return message_types
+
+    def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
+        """Determine if there are any active default multimodal loras."""
+        # TODO: Currently this is only enabled for chat completions
+        # to be better aligned with only being enabled for .generate
+        # when run offline. It would be nice to support additional
+        # tasks types in the future.
+        message_types = self._get_message_types(request)
+        default_mm_loras = set()
+
+        for lora in self.models.lora_requests.values():
+            # Best effort match for default multimodal lora adapters;
+            # There is probably a better way to do this, but currently
+            # this matches against the set of 'types' in any content lists
+            # up until '_', e.g., to match audio_url -> audio
+            if lora.lora_name in message_types:
+                default_mm_loras.add(lora)
+
+        # Currently only support default modality specific loras if
+        # we have exactly one lora matched on the request.
+        if len(default_mm_loras) == 1:
+            return default_mm_loras.pop()
+        return None
+
+    def _maybe_get_adapters(
+        self,
+        request: AnyRequest,
+        supports_default_mm_loras: bool = False,
+    ) -> LoRARequest | None:
+        if request.model in self.models.lora_requests:
+            return self.models.lora_requests[request.model]
+
+        # Currently only support default modality specific loras
+        # if we have exactly one lora matched on the request.
+        if supports_default_mm_loras:
+            default_mm_lora = self._get_active_default_mm_loras(request)
+            if default_mm_lora is not None:
+                return default_mm_lora
+
+        if self._is_model_supported(request.model):
+            return None
+
+        # if _check_model has been called earlier, this will be unreachable
+        raise ValueError(f"The model `{request.model}` does not exist.")

vllm/entrypoints/serve/engine/typing.py

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import Protocol, TypeAlias
+
+from vllm.config import ModelConfig
+from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    BatchChatCompletionRequest,
+    ChatCompletionRequest,
+    ChatCompletionResponse,
+)
+from vllm.entrypoints.openai.completion.protocol import (
+    CompletionRequest,
+    CompletionResponse,
+)
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
+from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
+from vllm.entrypoints.serve.tokenize.protocol import (
+    DetokenizeRequest,
+    TokenizeChatRequest,
+    TokenizeCompletionRequest,
+    TokenizeResponse,
+)
+from vllm.entrypoints.speech_to_text.transcription.protocol import (
+    TranscriptionRequest,
+    TranscriptionResponse,
+)
+from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
+from vllm.renderers import ChatParams, TokenizeParams
+
+
+class RendererRequest(Protocol):
+    def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
+        raise NotImplementedError
+
+
+class RendererChatRequest(RendererRequest, Protocol):
+    def build_chat_params(
+        self,
+        default_template: str | None,
+        default_template_content_format: ChatTemplateContentFormatOption,
+    ) -> ChatParams:
+        raise NotImplementedError
+
+
+CompletionLikeRequest: TypeAlias = (
+    CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
+)
+
+ChatLikeRequest: TypeAlias = (
+    ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
+)
+
+SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
+
+AnyRequest: TypeAlias = (
+    CompletionLikeRequest
+    | ChatLikeRequest
+    | SpeechToTextRequest
+    | ResponsesRequest
+    | GenerateRequest
+)
+
+AnyResponse: TypeAlias = (
+    CompletionResponse
+    | ChatCompletionResponse
+    | TranscriptionResponse
+    | TokenizeResponse
+    | GenerateResponse
+)

vllm/entrypoints/serve/instrumentator/basic.py

@@ -5,8 +5,7 @@ from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
 
 from vllm.engine.protocol import EngineClient
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
 from vllm.logger import init_logger
 from vllm.version import __version__ as VLLM_VERSION
 
@@ -15,13 +14,13 @@ router = APIRouter()
 logger = init_logger(__name__)
 
 
-def base(request: Request) -> OpenAIServing:
+def base(request: Request) -> ServingTokenization:
     # Reuse the existing instance
     return tokenization(request)
 
 
-def tokenization(request: Request) -> OpenAIServingTokenization:
-    return request.app.state.openai_serving_tokenization
+def tokenization(request: Request) -> ServingTokenization:
+    return request.app.state.serving_tokenization
 
 
 def engine_client(request: Request) -> EngineClient:

vllm/entrypoints/serve/tokenize/api_router.py

@@ -9,16 +9,14 @@ from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from typing_extensions import assert_never
 
-from vllm.entrypoints.openai.engine.protocol import (
-    ErrorResponse,
-)
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
 from vllm.entrypoints.serve.tokenize.protocol import (
     DetokenizeRequest,
     DetokenizeResponse,
     TokenizeRequest,
     TokenizeResponse,
 )
-from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
+from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
 from vllm.entrypoints.serve.utils.api_utils import (
     validate_json_request,
     with_cancellation,
@@ -28,8 +26,8 @@ from vllm.logger import init_logger
 logger = init_logger(__name__)
 
 
-def tokenization(request: Request) -> OpenAIServingTokenization:
-    return request.app.state.openai_serving_tokenization
+def tokenization(request: Request) -> ServingTokenization:
+    return request.app.state.serving_tokenization
 
 
 router = APIRouter()

vllm/entrypoints/serve/tokenize/serving.py

@@ -5,11 +5,13 @@ from typing import Any, Final
 
 from fastapi import Request
 
-from vllm.engine.protocol import EngineClient
 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
-from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.openai.models.serving import (
+    OpenAIModelRegistry,
+    OpenAIServingModels,
+)
+from vllm.entrypoints.serve.engine.serving import BaseServing
 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 from vllm.entrypoints.serve.tokenize.protocol import (
     DetokenizeRequest,
@@ -27,11 +29,10 @@ from vllm.tokenizers import TokenizerLike
 logger = init_logger(__name__)
 
 
-class OpenAIServingTokenization(OpenAIServing):
+class ServingTokenization(BaseServing):
     def __init__(
         self,
-        engine_client: EngineClient,
-        models: OpenAIServingModels,
+        models: OpenAIServingModels | OpenAIModelRegistry,
         openai_serving_render: OpenAIServingRender,
         *,
         request_logger: RequestLogger | None,
@@ -41,11 +42,12 @@ class OpenAIServingTokenization(OpenAIServing):
         trust_request_chat_template: bool = False,
     ) -> None:
         super().__init__(
-            engine_client=engine_client,
             models=models,
+            model_config=openai_serving_render.model_config,
             request_logger=request_logger,
         )
 
+        self.renderer = openai_serving_render.renderer
         self.openai_serving_render = openai_serving_render
         self.chat_template = chat_template
         self.chat_template_content_format: Final = chat_template_content_format

vllm/entrypoints/speech_to_text/base/serving.py

@@ -22,8 +22,9 @@ from vllm.entrypoints.openai.engine.protocol import (
     RequestResponseMetadata,
     UsageInfo,
 )
-from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
 from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
 from vllm.exceptions import VLLMValidationError