diff --git a/tests/entrypoints/serve/tokenize/test_serving_tokenization.py b/tests/entrypoints/serve/tokenize/test_serving_tokenization.py index ba9d7989a8..7f629afb1a 100644 --- a/tests/entrypoints/serve/tokenize/test_serving_tokenization.py +++ b/tests/entrypoints/serve/tokenize/test_serving_tokenization.py @@ -15,7 +15,7 @@ from vllm.entrypoints.serve.tokenize.protocol import ( TokenizeChatRequest, TokenizeCompletionRequest, ) -from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.v1.engine.async_llm import AsyncLLM MODEL_NAME = "openai-community/gpt2" @@ -58,7 +58,7 @@ class MockModelConfig: return self.diff_sampling_param or {} -def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization: +def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization: models = OpenAIServingModels( engine_client=engine, base_model_paths=BASE_MODEL_PATHS, @@ -71,8 +71,7 @@ def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization: chat_template=None, chat_template_content_format="auto", ) - return OpenAIServingTokenization( - engine, + return ServingTokenization( models, openai_serving_render=serving_render, request_logger=None, diff --git a/vllm/entrypoints/anthropic/api_router.py b/vllm/entrypoints/anthropic/api_router.py index 31b5a3fbab..414f87f308 100644 --- a/vllm/entrypoints/anthropic/api_router.py +++ b/vllm/entrypoints/anthropic/api_router.py @@ -61,7 +61,7 @@ def translate_error_response(response: ErrorResponse) -> JSONResponse: async def create_messages(request: AnthropicMessagesRequest, raw_request: Request): handler = messages(raw_request) if handler is None: - base_server = raw_request.app.state.openai_serving_tokenization + base_server = raw_request.app.state.serving_tokenization error = base_server.create_error_response( NotImplementedError("The model does not support Messages API") ) @@ -107,7 +107,7 @@ async def create_messages(request: AnthropicMessagesRequest, raw_request: Reques async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request): handler = messages(raw_request) if handler is None: - base_server = raw_request.app.state.openai_serving_tokenization + base_server = raw_request.app.state.serving_tokenization error = base_server.create_error_response( NotImplementedError("The model does not support Messages API") ) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 34c4f0ca5d..a16f522183 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -34,7 +34,7 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap -from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( cli_env_setup, log_non_default_args, @@ -376,8 +376,7 @@ async def init_app_state( log_error_stack=args.log_error_stack, ) - state.openai_serving_tokenization = OpenAIServingTokenization( - engine_client, + state.serving_tokenization = ServingTokenization( state.openai_serving_models, state.openai_serving_render, request_logger=request_logger, @@ -461,9 +460,15 @@ async def init_render_app_state( ) state.openai_serving_models = model_registry - - # Expose tokenization via the render handler (no engine required). - state.openai_serving_tokenization = state.openai_serving_render + state.serving_tokenization = ServingTokenization( + model_registry, + state.openai_serving_render, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + default_chat_template_kwargs=args.default_chat_template_kwargs, + trust_request_chat_template=args.trust_request_chat_template, + ) state.vllm_config = vllm_config # Disable stats logging — there is no engine to poll. diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index 5eb917ef96..d32fbad52a 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -5,25 +5,19 @@ import time from collections.abc import Awaitable, Mapping from dataclasses import dataclass, field from http import HTTPStatus -from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar +from typing import ClassVar, Generic, TypeVar from fastapi import Request from pydantic import ConfigDict from starlette.datastructures import Headers -import vllm.envs as envs -from vllm.config import ModelConfig from vllm.engine.protocol import EngineClient -from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin from vllm.entrypoints.openai.chat_completion.protocol import ( - BatchChatCompletionRequest, ChatCompletionRequest, - ChatCompletionResponse, ) from vllm.entrypoints.openai.completion.protocol import ( CompletionRequest, - CompletionResponse, ) from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, @@ -31,81 +25,22 @@ from vllm.entrypoints.openai.engine.protocol import ( ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse -from vllm.entrypoints.serve.tokenize.protocol import ( - DetokenizeRequest, - TokenizeChatRequest, - TokenizeCompletionRequest, - TokenizeResponse, -) -from vllm.entrypoints.serve.utils.error_response import create_error_response +from vllm.entrypoints.serve.engine.serving import BaseServing +from vllm.entrypoints.serve.engine.typing import AnyRequest from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.entrypoints.speech_to_text.transcription.protocol import ( - TranscriptionRequest, - TranscriptionResponse, -) -from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest -from vllm.inputs import EngineInput, PromptType +from vllm.inputs import EngineInput from vllm.logger import init_logger from vllm.logprobs import Logprob, PromptLogprobs from vllm.lora.request import LoRARequest -from vllm.renderers import ChatParams, TokenizeParams -from vllm.renderers.inputs.preprocess import ( - extract_prompt_components, - extract_prompt_len, -) -from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.tracing import ( contains_trace_headers, extract_trace_headers, log_tracing_disabled_warning, ) -from vllm.utils import random_uuid logger = init_logger(__name__) - -class RendererRequest(Protocol): - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - raise NotImplementedError - - -class RendererChatRequest(RendererRequest, Protocol): - def build_chat_params( - self, - default_template: str | None, - default_template_content_format: ChatTemplateContentFormatOption, - ) -> ChatParams: - raise NotImplementedError - - -CompletionLikeRequest: TypeAlias = ( - CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest -) - -ChatLikeRequest: TypeAlias = ( - ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest -) - -SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest - -AnyRequest: TypeAlias = ( - CompletionLikeRequest - | ChatLikeRequest - | SpeechToTextRequest - | ResponsesRequest - | GenerateRequest -) - -AnyResponse: TypeAlias = ( - CompletionResponse - | ChatCompletionResponse - | TranscriptionResponse - | TokenizeResponse - | GenerateResponse -) - RequestT = TypeVar("RequestT", bound=AnyRequest) _T = TypeVar("_T") @@ -122,7 +57,7 @@ class ServeContext(Generic[RequestT]): model_config = ConfigDict(arbitrary_types_allowed=True) -class OpenAIServing(BeamSearchOnlineMixin): +class OpenAIServing(BaseServing, BeamSearchOnlineMixin): request_id_prefix: ClassVar[str] = """ A short string prepended to every request’s ID. """ @@ -135,15 +70,14 @@ class OpenAIServing(BeamSearchOnlineMixin): request_logger: RequestLogger | None, return_tokens_as_token_ids: bool = False, ): - super().__init__() + super().__init__( + models=models, + model_config=engine_client.model_config, + request_logger=request_logger, + ) self.engine_client = engine_client - self.models = models - - self.request_logger = request_logger self.return_tokens_as_token_ids = return_tokens_as_token_ids - - self.model_config = engine_client.model_config self.renderer = engine_client.renderer self.input_processor = engine_client.input_processor vllm_config = getattr(engine_client, "vllm_config", None) @@ -163,15 +97,6 @@ class OpenAIServing(BeamSearchOnlineMixin): # Never fail server startup over the fingerprint. self.system_fingerprint = None - @staticmethod - def create_error_response( - message: str | Exception, - err_type: str = "BadRequestError", - status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, - param: str | None = None, - ) -> ErrorResponse: - return create_error_response(message, err_type, status_code, param) - def create_streaming_error_response( self, message: str | Exception, @@ -208,167 +133,6 @@ class OpenAIServing(BeamSearchOnlineMixin): status_code=e.status_code, ) - async def _check_model( - self, - request: AnyRequest, - ) -> ErrorResponse | None: - error_response = None - - if self._is_model_supported(request.model): - return None - if request.model in self.models.lora_requests: - return None - if ( - envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING - and request.model - and (load_result := await self.models.resolve_lora(request.model)) - ): - if isinstance(load_result, LoRARequest): - return None - if ( - isinstance(load_result, ErrorResponse) - and load_result.error.code == HTTPStatus.BAD_REQUEST.value - ): - error_response = load_result - - return error_response or self.create_error_response( - message=f"The model `{request.model}` does not exist.", - err_type="NotFoundError", - status_code=HTTPStatus.NOT_FOUND, - param="model", - ) - - def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None: - """Determine if there are any active default multimodal loras.""" - # TODO: Currently this is only enabled for chat completions - # to be better aligned with only being enabled for .generate - # when run offline. It would be nice to support additional - # tasks types in the future. - message_types = self._get_message_types(request) - default_mm_loras = set() - - for lora in self.models.lora_requests.values(): - # Best effort match for default multimodal lora adapters; - # There is probably a better way to do this, but currently - # this matches against the set of 'types' in any content lists - # up until '_', e.g., to match audio_url -> audio - if lora.lora_name in message_types: - default_mm_loras.add(lora) - - # Currently only support default modality specific loras if - # we have exactly one lora matched on the request. - if len(default_mm_loras) == 1: - return default_mm_loras.pop() - return None - - def _maybe_get_adapters( - self, - request: AnyRequest, - supports_default_mm_loras: bool = False, - ) -> LoRARequest | None: - if request.model in self.models.lora_requests: - return self.models.lora_requests[request.model] - - # Currently only support default modality specific loras - # if we have exactly one lora matched on the request. - if supports_default_mm_loras: - default_mm_lora = self._get_active_default_mm_loras(request) - if default_mm_lora is not None: - return default_mm_lora - - if self._is_model_supported(request.model): - return None - - # if _check_model has been called earlier, this will be unreachable - raise ValueError(f"The model `{request.model}` does not exist.") - - def _get_message_types(self, request: AnyRequest) -> set[str]: - """Retrieve the set of types from message content dicts up - until `_`; we use this to match potential multimodal data - with default per modality loras. - """ - message_types: set[str] = set() - - if not hasattr(request, "messages"): - return message_types - - messages = request.messages - if messages is None or isinstance(messages, (str, bytes)): - return message_types - - for message in messages: - if ( - isinstance(message, dict) - and "content" in message - and isinstance(message["content"], list) - ): - for content_dict in message["content"]: - if "type" in content_dict: - message_types.add(content_dict["type"].split("_")[0]) - return message_types - - def _validate_chat_template( - self, - request_chat_template: str | None, - chat_template_kwargs: dict[str, Any] | None, - trust_request_chat_template: bool, - ) -> ErrorResponse | None: - if not trust_request_chat_template and ( - request_chat_template is not None - or ( - chat_template_kwargs - and chat_template_kwargs.get("chat_template") is not None - ) - ): - return self.create_error_response( - "Chat template is passed with request, but " - "--trust-request-chat-template is not set. " - "Refused request with untrusted chat template." - ) - return None - - @staticmethod - def _prepare_extra_chat_template_kwargs( - request_chat_template_kwargs: dict[str, Any] | None = None, - default_chat_template_kwargs: dict[str, Any] | None = None, - ) -> dict[str, Any]: - """Helper to merge server-default and request-specific chat template kwargs.""" - request_chat_template_kwargs = request_chat_template_kwargs or {} - if default_chat_template_kwargs is None: - return request_chat_template_kwargs - # Apply server defaults first, then request kwargs override. - return default_chat_template_kwargs | request_chat_template_kwargs - - def _extract_prompt_components(self, prompt: PromptType | EngineInput): - return extract_prompt_components(self.model_config, prompt) - - def _extract_prompt_text(self, prompt: PromptType | EngineInput): - return self._extract_prompt_components(prompt).text - - def _extract_prompt_len(self, prompt: EngineInput): - return extract_prompt_len(self.model_config, prompt) - - def _log_inputs( - self, - request_id: str, - inputs: PromptType | EngineInput, - params: SamplingParams | BeamSearchParams | None, - lora_request: LoRARequest | None, - ) -> None: - if self.request_logger is None: - return - - components = self._extract_prompt_components(inputs) - - self.request_logger.log_inputs( - request_id, - components.text, - components.token_ids, - components.embeds, - params=params, - lora_request=lora_request, - ) - async def _get_trace_headers( self, headers: Headers, @@ -383,18 +147,6 @@ class OpenAIServing(BeamSearchOnlineMixin): return None - @staticmethod - def _base_request_id( - raw_request: Request | None, default: str | None = None - ) -> str | None: - """Pulls the request id to use from a header, if provided""" - if raw_request is not None and ( - (req_id := raw_request.headers.get("X-Request-Id")) is not None - ): - return req_id - - return random_uuid() if default is None else default - @staticmethod def _get_data_parallel_rank(raw_request: Request | None) -> int | None: """Pulls the data parallel rank from a header, if provided""" @@ -464,13 +216,6 @@ class OpenAIServing(BeamSearchOnlineMixin): return tokenizer.decode([token_id]) - def _is_model_supported(self, model_name: str | None) -> bool: - if not model_name: - return True - if envs.VLLM_SKIP_MODEL_NAME_VALIDATION: - return True - return self.models.is_base_model(model_name) - def format_token_id_placeholder(token_id: int) -> str: return f"token_id:{token_id}" diff --git a/vllm/entrypoints/openai/models/serving.py b/vllm/entrypoints/openai/models/serving.py index ea330678d0..b886d92641 100644 --- a/vllm/entrypoints/openai/models/serving.py +++ b/vllm/entrypoints/openai/models/serving.py @@ -42,6 +42,10 @@ class OpenAIModelRegistry: ) -> None: self.model_config = model_config self.base_model_paths = base_model_paths + self.lora_requests: dict[str, LoRARequest] = {} + + def model_name(self, lora_request: LoRARequest | None = None) -> str: + return self.base_model_paths[0].name def is_base_model(self, model_name: str) -> bool: return any(model.name == model_name for model in self.base_model_paths) @@ -72,6 +76,9 @@ class OpenAIModelRegistry: ] ) + async def resolve_lora(self, lora_name: str): + raise RuntimeError("The OpenAIModelRegistry has no LoRA support.") + class OpenAIServingModels: """Shared instance to hold data about the loaded base model(s) and adapters. diff --git a/vllm/entrypoints/pooling/base/io_processor.py b/vllm/entrypoints/pooling/base/io_processor.py index fc24bc6578..3d672a13ec 100644 --- a/vllm/entrypoints/pooling/base/io_processor.py +++ b/vllm/entrypoints/pooling/base/io_processor.py @@ -12,7 +12,7 @@ from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, ConversationMessage, ) -from vllm.entrypoints.openai.engine.serving import RendererChatRequest, RendererRequest +from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest from vllm.inputs import EngineInput, SingletonPrompt from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq diff --git a/vllm/entrypoints/pooling/base/serving.py b/vllm/entrypoints/pooling/base/serving.py index d849baba05..e1b83e711b 100644 --- a/vllm/entrypoints/pooling/base/serving.py +++ b/vllm/entrypoints/pooling/base/serving.py @@ -52,7 +52,7 @@ class PoolingServingBase(ABC): self.engine_client = engine_client self.models = models self.model_config = models.model_config - self.renderer = models.renderer + self.renderer = engine_client.renderer self.vllm_config = engine_client.vllm_config self.max_model_len = self.model_config.max_model_len self.request_logger = request_logger @@ -61,7 +61,7 @@ class PoolingServingBase(ABC): self.chat_template_config = chat_template_config # Shared thread pool executor for preprocessing and postprocessing. - self._executor: Executor = models.renderer._executor + self._executor: Executor = self.renderer._executor self._preprocessing_async = make_async( self._preprocessing, executor=self._executor ) diff --git a/vllm/entrypoints/serve/disagg/api_router.py b/vllm/entrypoints/serve/disagg/api_router.py index 7cec4344b3..60b671d6e8 100644 --- a/vllm/entrypoints/serve/disagg/api_router.py +++ b/vllm/entrypoints/serve/disagg/api_router.py @@ -20,7 +20,7 @@ from vllm.entrypoints.serve.disagg.protocol import ( from vllm.entrypoints.serve.disagg.serving import ( ServingTokens, ) -from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, validate_json_request, @@ -31,8 +31,8 @@ from vllm.logger import init_logger logger = init_logger(__name__) -def tokenization(request: Request) -> OpenAIServingTokenization: - return request.app.state.openai_serving_tokenization +def tokenization(request: Request) -> ServingTokenization: + return request.app.state.serving_tokenization def generate_tokens(request: Request) -> ServingTokens | None: diff --git a/vllm/entrypoints/serve/engine/__init__.py b/vllm/entrypoints/serve/engine/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/entrypoints/serve/engine/serving.py b/vllm/entrypoints/serve/engine/serving.py new file mode 100644 index 0000000000..409743a599 --- /dev/null +++ b/vllm/entrypoints/serve/engine/serving.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from http import HTTPStatus + +from fastapi import Request + +from vllm import PromptType, SamplingParams, envs +from vllm.config import ModelConfig +from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.openai.models.serving import ( + OpenAIModelRegistry, + OpenAIServingModels, +) +from vllm.entrypoints.serve.engine.typing import AnyRequest +from vllm.entrypoints.serve.utils.error_response import create_error_response +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.inputs import EngineInput +from vllm.lora.request import LoRARequest +from vllm.renderers.inputs.preprocess import ( + extract_prompt_components, + extract_prompt_len, +) +from vllm.sampling_params import BeamSearchParams +from vllm.utils import random_uuid + + +class BaseServing: + def __init__( + self, + models: OpenAIServingModels | OpenAIModelRegistry, + model_config: ModelConfig, + request_logger: RequestLogger | None = None, + ): + self.models = models + self.model_config = model_config + self.request_logger = request_logger + + async def _check_model( + self, + request: AnyRequest, + ) -> ErrorResponse | None: + error_response = None + + if self._is_model_supported(request.model): + return None + if request.model in self.models.lora_requests: + return None + if ( + envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING + and request.model + and (load_result := await self.models.resolve_lora(request.model)) + ): + if isinstance(load_result, LoRARequest): + return None + if ( + isinstance(load_result, ErrorResponse) + and load_result.error.code == HTTPStatus.BAD_REQUEST.value + ): + error_response = load_result + + return error_response or self.create_error_response( + message=f"The model `{request.model}` does not exist.", + err_type="NotFoundError", + status_code=HTTPStatus.NOT_FOUND, + param="model", + ) + + def _is_model_supported(self, model_name: str | None) -> bool: + if not model_name: + return True + if envs.VLLM_SKIP_MODEL_NAME_VALIDATION: + return True + return self.models.is_base_model(model_name) + + @staticmethod + def create_error_response( + message: str | Exception, + err_type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, + param: str | None = None, + ) -> ErrorResponse: + return create_error_response(message, err_type, status_code, param) + + def _extract_prompt_components(self, prompt: PromptType | EngineInput): + return extract_prompt_components(self.model_config, prompt) + + def _extract_prompt_text(self, prompt: PromptType | EngineInput): + return self._extract_prompt_components(prompt).text + + def _extract_prompt_len(self, prompt: EngineInput): + return extract_prompt_len(self.model_config, prompt) + + def _log_inputs( + self, + request_id: str, + inputs: PromptType | EngineInput, + params: SamplingParams | BeamSearchParams | None, + lora_request: LoRARequest | None, + ) -> None: + if self.request_logger is None: + return + + components = self._extract_prompt_components(inputs) + + self.request_logger.log_inputs( + request_id, + components.text, + components.token_ids, + components.embeds, + params=params, + lora_request=lora_request, + ) + + @staticmethod + def _base_request_id( + raw_request: Request | None, default: str | None = None + ) -> str | None: + """Pulls the request id to use from a header, if provided""" + if raw_request is not None and ( + (req_id := raw_request.headers.get("X-Request-Id")) is not None + ): + return req_id + + return random_uuid() if default is None else default + + def _get_message_types(self, request: AnyRequest) -> set[str]: + """Retrieve the set of types from message content dicts up + until `_`; we use this to match potential multimodal data + with default per modality loras. + """ + message_types: set[str] = set() + + if not hasattr(request, "messages"): + return message_types + + messages = request.messages + if messages is None or isinstance(messages, (str, bytes)): + return message_types + + for message in messages: + if ( + isinstance(message, dict) + and "content" in message + and isinstance(message["content"], list) + ): + for content_dict in message["content"]: + if "type" in content_dict: + message_types.add(content_dict["type"].split("_")[0]) + return message_types + + def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None: + """Determine if there are any active default multimodal loras.""" + # TODO: Currently this is only enabled for chat completions + # to be better aligned with only being enabled for .generate + # when run offline. It would be nice to support additional + # tasks types in the future. + message_types = self._get_message_types(request) + default_mm_loras = set() + + for lora in self.models.lora_requests.values(): + # Best effort match for default multimodal lora adapters; + # There is probably a better way to do this, but currently + # this matches against the set of 'types' in any content lists + # up until '_', e.g., to match audio_url -> audio + if lora.lora_name in message_types: + default_mm_loras.add(lora) + + # Currently only support default modality specific loras if + # we have exactly one lora matched on the request. + if len(default_mm_loras) == 1: + return default_mm_loras.pop() + return None + + def _maybe_get_adapters( + self, + request: AnyRequest, + supports_default_mm_loras: bool = False, + ) -> LoRARequest | None: + if request.model in self.models.lora_requests: + return self.models.lora_requests[request.model] + + # Currently only support default modality specific loras + # if we have exactly one lora matched on the request. + if supports_default_mm_loras: + default_mm_lora = self._get_active_default_mm_loras(request) + if default_mm_lora is not None: + return default_mm_lora + + if self._is_model_supported(request.model): + return None + + # if _check_model has been called earlier, this will be unreachable + raise ValueError(f"The model `{request.model}` does not exist.") diff --git a/vllm/entrypoints/serve/engine/typing.py b/vllm/entrypoints/serve/engine/typing.py new file mode 100644 index 0000000000..253dfb8d90 --- /dev/null +++ b/vllm/entrypoints/serve/engine/typing.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Protocol, TypeAlias + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption +from vllm.entrypoints.openai.chat_completion.protocol import ( + BatchChatCompletionRequest, + ChatCompletionRequest, + ChatCompletionResponse, +) +from vllm.entrypoints.openai.completion.protocol import ( + CompletionRequest, + CompletionResponse, +) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse +from vllm.entrypoints.serve.tokenize.protocol import ( + DetokenizeRequest, + TokenizeChatRequest, + TokenizeCompletionRequest, + TokenizeResponse, +) +from vllm.entrypoints.speech_to_text.transcription.protocol import ( + TranscriptionRequest, + TranscriptionResponse, +) +from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest +from vllm.renderers import ChatParams, TokenizeParams + + +class RendererRequest(Protocol): + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: + raise NotImplementedError + + +class RendererChatRequest(RendererRequest, Protocol): + def build_chat_params( + self, + default_template: str | None, + default_template_content_format: ChatTemplateContentFormatOption, + ) -> ChatParams: + raise NotImplementedError + + +CompletionLikeRequest: TypeAlias = ( + CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest +) + +ChatLikeRequest: TypeAlias = ( + ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest +) + +SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest + +AnyRequest: TypeAlias = ( + CompletionLikeRequest + | ChatLikeRequest + | SpeechToTextRequest + | ResponsesRequest + | GenerateRequest +) + +AnyResponse: TypeAlias = ( + CompletionResponse + | ChatCompletionResponse + | TranscriptionResponse + | TokenizeResponse + | GenerateResponse +) diff --git a/vllm/entrypoints/serve/instrumentator/basic.py b/vllm/entrypoints/serve/instrumentator/basic.py index e6c96de0ba..be091a1f43 100644 --- a/vllm/entrypoints/serve/instrumentator/basic.py +++ b/vllm/entrypoints/serve/instrumentator/basic.py @@ -5,8 +5,7 @@ from fastapi import APIRouter, Request from fastapi.responses import JSONResponse from vllm.engine.protocol import EngineClient -from vllm.entrypoints.openai.engine.serving import OpenAIServing -from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.logger import init_logger from vllm.version import __version__ as VLLM_VERSION @@ -15,13 +14,13 @@ router = APIRouter() logger = init_logger(__name__) -def base(request: Request) -> OpenAIServing: +def base(request: Request) -> ServingTokenization: # Reuse the existing instance return tokenization(request) -def tokenization(request: Request) -> OpenAIServingTokenization: - return request.app.state.openai_serving_tokenization +def tokenization(request: Request) -> ServingTokenization: + return request.app.state.serving_tokenization def engine_client(request: Request) -> EngineClient: diff --git a/vllm/entrypoints/serve/tokenize/api_router.py b/vllm/entrypoints/serve/tokenize/api_router.py index eebb17c642..9695e6ccea 100644 --- a/vllm/entrypoints/serve/tokenize/api_router.py +++ b/vllm/entrypoints/serve/tokenize/api_router.py @@ -9,16 +9,14 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from typing_extensions import assert_never -from vllm.entrypoints.openai.engine.protocol import ( - ErrorResponse, -) +from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.serve.tokenize.protocol import ( DetokenizeRequest, DetokenizeResponse, TokenizeRequest, TokenizeResponse, ) -from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( validate_json_request, with_cancellation, @@ -28,8 +26,8 @@ from vllm.logger import init_logger logger = init_logger(__name__) -def tokenization(request: Request) -> OpenAIServingTokenization: - return request.app.state.openai_serving_tokenization +def tokenization(request: Request) -> ServingTokenization: + return request.app.state.serving_tokenization router = APIRouter() diff --git a/vllm/entrypoints/serve/tokenize/serving.py b/vllm/entrypoints/serve/tokenize/serving.py index 4f461c0194..d898412e2a 100644 --- a/vllm/entrypoints/serve/tokenize/serving.py +++ b/vllm/entrypoints/serve/tokenize/serving.py @@ -5,11 +5,13 @@ from typing import Any, Final from fastapi import Request -from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption from vllm.entrypoints.openai.engine.protocol import ErrorResponse -from vllm.entrypoints.openai.engine.serving import OpenAIServing -from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.openai.models.serving import ( + OpenAIModelRegistry, + OpenAIServingModels, +) +from vllm.entrypoints.serve.engine.serving import BaseServing from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.tokenize.protocol import ( DetokenizeRequest, @@ -27,11 +29,10 @@ from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) -class OpenAIServingTokenization(OpenAIServing): +class ServingTokenization(BaseServing): def __init__( self, - engine_client: EngineClient, - models: OpenAIServingModels, + models: OpenAIServingModels | OpenAIModelRegistry, openai_serving_render: OpenAIServingRender, *, request_logger: RequestLogger | None, @@ -41,11 +42,12 @@ class OpenAIServingTokenization(OpenAIServing): trust_request_chat_template: bool = False, ) -> None: super().__init__( - engine_client=engine_client, models=models, + model_config=openai_serving_render.model_config, request_logger=request_logger, ) + self.renderer = openai_serving_render.renderer self.openai_serving_render = openai_serving_render self.chat_template = chat_template self.chat_template_content_format: Final = chat_template_content_format diff --git a/vllm/entrypoints/speech_to_text/base/serving.py b/vllm/entrypoints/speech_to_text/base/serving.py index b60ac6ff95..1cbc4d8f79 100644 --- a/vllm/entrypoints/speech_to_text/base/serving.py +++ b/vllm/entrypoints/speech_to_text/base/serving.py @@ -22,8 +22,9 @@ from vllm.entrypoints.openai.engine.protocol import ( RequestResponseMetadata, UsageInfo, ) -from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest +from vllm.entrypoints.openai.engine.serving import OpenAIServing from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest from vllm.entrypoints.serve.utils.api_utils import get_max_tokens from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.exceptions import VLLMValidationError