diff --git a/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py b/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py index d800829922..12dae6dfd5 100644 --- a/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py +++ b/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py @@ -78,7 +78,6 @@ def _create_mock_engine(): mock_engine.model_config = MockModelConfig() mock_engine.input_processor = MagicMock() - # renderer is accessed by OpenAIServing.__init__ and serving.py mock_renderer = MagicMock() mock_renderer.tokenizer = get_tokenizer(MODEL_NAME) mock_engine.renderer = mock_renderer diff --git a/tests/entrypoints/openai/responses/test_errors.py b/tests/entrypoints/openai/responses/test_errors.py index e21f6aa2a4..a2f1a0f324 100644 --- a/tests/entrypoints/openai/responses/test_errors.py +++ b/tests/entrypoints/openai/responses/test_errors.py @@ -7,20 +7,20 @@ from unittest.mock import MagicMock import pytest import vllm.envs as envs -from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing +from vllm.entrypoints.generate.base.serving import GenerateBaseServing, GenerationError from vllm.envs import disable_envs_cache @pytest.mark.asyncio async def test_raise_if_error_raises_generation_error(): """test _raise_if_error raises GenerationError""" - # create a minimal OpenAIServing instance + # create a minimal GenerateBaseServing instance mock_engine = MagicMock() mock_engine.model_config = MagicMock() mock_engine.model_config.max_model_len = 100 mock_models = MagicMock() - serving = OpenAIServing( + serving = GenerateBaseServing( engine_client=mock_engine, models=mock_models, request_logger=None, @@ -47,7 +47,7 @@ async def test_convert_generation_error_to_streaming_response(): mock_engine.model_config.max_model_len = 100 mock_models = MagicMock() - serving = OpenAIServing( + serving = GenerateBaseServing( engine_client=mock_engine, models=mock_models, request_logger=None, @@ -77,7 +77,7 @@ def test_is_model_supported_skip_name_validation_env( mock_models = MagicMock() mock_models.is_base_model.return_value = False - serving = OpenAIServing( + serving = GenerateBaseServing( engine_client=mock_engine, models=mock_models, request_logger=None, diff --git a/tests/entrypoints/openai/test_return_tokens_as_ids.py b/tests/entrypoints/openai/test_return_tokens_as_ids.py index 2a311cc5c8..170bbd420c 100644 --- a/tests/entrypoints/openai/test_return_tokens_as_ids.py +++ b/tests/entrypoints/openai/test_return_tokens_as_ids.py @@ -127,13 +127,13 @@ def test_responses_api_logprobs_with_return_tokens_as_token_ids(): """Test that return_tokens_as_token_ids works in Responses API logprobs.""" from unittest.mock import MagicMock - from vllm.entrypoints.openai.engine.serving import OpenAIServing + from vllm.entrypoints.generate.base.serving import GenerateBaseServing from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses from vllm.logprobs import Logprob as SampleLogprob serving = MagicMock(spec=OpenAIServingResponses) serving.return_tokens_as_token_ids = True - serving._get_decoded_token = OpenAIServing._get_decoded_token + serving._get_decoded_token = GenerateBaseServing._get_decoded_token tokenizer = MagicMock() tokenizer.decode = lambda token_id: "decoded" diff --git a/tests/entrypoints/serve/lora/test_serving_models.py b/tests/entrypoints/serve/lora/test_serving_models.py index 0cab3fd42c..658d004580 100644 --- a/tests/entrypoints/serve/lora/test_serving_models.py +++ b/tests/entrypoints/serve/lora/test_serving_models.py @@ -14,7 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import ( ) from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.pooling.base.serving import PoolingServingBase +from vllm.entrypoints.pooling.base.serving import PoolingBaseServing from vllm.entrypoints.pooling.typing import PoolingServeContext from vllm.entrypoints.serve.lora.protocol import ( LoadLoRAAdapterRequest, @@ -136,7 +136,7 @@ async def test_unload_lora_adapter_not_found(): assert response.error.code == HTTPStatus.NOT_FOUND -class _ConcretePoolingServing(PoolingServingBase): +class _ConcretePoolingServing(PoolingBaseServing): """Minimal concrete subclass used only in these unit tests.""" request_id_prefix = "test" @@ -178,7 +178,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request(): serving = _make_pooling_serving(lora_name) ctx = _make_pooling_ctx(lora_name) - serving._maybe_get_adapters(ctx) + ctx.lora_request = serving._maybe_get_adapters(ctx.request) assert ctx.lora_request is not None assert ctx.lora_request.lora_name == lora_name @@ -190,4 +190,4 @@ def test_pooling_maybe_get_adapters_unknown_model_raises(): ctx = _make_pooling_ctx("unknown-model") with pytest.raises(VLLMNotFoundError): - serving._maybe_get_adapters(ctx) + serving._maybe_get_adapters(ctx.request) diff --git a/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py b/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py index 11c797c27c..040fc1a48f 100644 --- a/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py +++ b/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, Mock import pytest -from vllm.entrypoints.speech_to_text.base.serving import OpenAISpeechToText +from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionResponse @@ -43,7 +43,7 @@ async def test_non_streaming_cancel_aborts_engine_requests( is_tracing_enabled=AsyncMock(return_value=False), ) - server = OpenAISpeechToText.__new__(OpenAISpeechToText) + server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing) server.engine_client = engine_client server.task_type = "transcribe" server.models = SimpleNamespace(model_name=lambda: "audio") @@ -112,7 +112,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators(): {"prompt": "chunk-1"}, {"prompt": "chunk-2"}, ] - server = OpenAISpeechToText.__new__(OpenAISpeechToText) + server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing) server.engine_client = engine_client server.task_type = "transcribe" server.models = SimpleNamespace(model_name=lambda: "audio") @@ -170,7 +170,7 @@ async def test_language_detection_cancel_aborts_engine_request(): abort=AsyncMock(), ) - server = OpenAISpeechToText.__new__(OpenAISpeechToText) + server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing) server.engine_client = engine_client server.asr_config = SimpleNamespace() server.tokenizer = Mock() diff --git a/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py index c4da9a80f7..3175a9e6fb 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py +++ b/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py @@ -25,7 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import ( ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.speech_to_text.base.serving import ( - OpenAISpeechToText, + SpeechToTextBaseServing, asr_inter_chunk_separator, ) from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest @@ -234,7 +234,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language(): "vllm.model_executor.model_loader.get_model_cls", return_value=_StubTranscriptionModel, ), - patch.object(OpenAISpeechToText, "_preprocess_speech_to_text", preprocess_mock), + patch.object( + SpeechToTextBaseServing, "_preprocess_speech_to_text", preprocess_mock + ), ): serving = OpenAIServingTranscription(engine_client, models, request_logger=None) diff --git a/vllm/entrypoints/generate/base/__init__.py b/vllm/entrypoints/generate/base/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/generate/base/serving.py similarity index 99% rename from vllm/entrypoints/openai/engine/serving.py rename to vllm/entrypoints/generate/base/serving.py index 70955b49af..196b6e3f87 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/generate/base/serving.py @@ -53,7 +53,7 @@ class ServeContext(Generic[RequestT]): model_config = ConfigDict(arbitrary_types_allowed=True) -class OpenAIServing(BaseServing, BeamSearchOnlineMixin): +class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin): request_id_prefix: ClassVar[str] = """ A short string prepended to every request’s ID. """ diff --git a/vllm/entrypoints/generate/generative_scoring/serving.py b/vllm/entrypoints/generate/generative_scoring/serving.py index f656755ac0..f566b9c66e 100644 --- a/vllm/entrypoints/generate/generative_scoring/serving.py +++ b/vllm/entrypoints/generate/generative_scoring/serving.py @@ -23,8 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import ( OpenAIBaseModel, UsageInfo, ) -from vllm.entrypoints.openai.engine.serving import OpenAIServing from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.serve.engine.serving import BaseServing from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import EngineInput, tokens_input from vllm.logger import init_logger @@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel): # ============================================================================ -class ServingGenerativeScoring(OpenAIServing): +class ServingGenerativeScoring(BaseServing): """Serving class for generative scoring computation. This class handles computing the probability of specified token IDs @@ -164,10 +164,12 @@ class ServingGenerativeScoring(OpenAIServing): request_logger: RequestLogger | None, ) -> None: super().__init__( - engine_client=engine_client, models=models, + model_config=engine_client.model_config, request_logger=request_logger, ) + self.engine_client = engine_client + self.renderer = engine_client.renderer async def create_generative_scoring( self, diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 284b2511db..9ef144e6a1 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -19,6 +19,12 @@ from vllm.entrypoints.chat_utils import ( ConversationMessage, make_tool_call_id, ) +from vllm.entrypoints.generate.base.serving import ( + GenerateBaseServing, + GenerationError, + clamp_prompt_logprobs, + format_token_id_placeholder, +) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionLogProb, ChatCompletionLogProbs, @@ -40,12 +46,6 @@ from vllm.entrypoints.openai.engine.protocol import ( ToolCall, UsageInfo, ) -from vllm.entrypoints.openai.engine.serving import ( - GenerationError, - OpenAIServing, - clamp_prompt_logprobs, - format_token_id_placeholder, -) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger @@ -76,7 +76,7 @@ def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]: counted in ``usage.prompt_tokens``. """ mm_placeholders = cast( - "MultiModalPlaceholders | None", engine_input.get("mm_placeholders") + MultiModalPlaceholders | None, engine_input.get("mm_placeholders") ) return { modality: sum(p.length for p in ranges) @@ -101,7 +101,7 @@ def _make_prompt_tokens_details( ) -class OpenAIServingChat(OpenAIServing): +class OpenAIServingChat(GenerateBaseServing): def __init__( self, engine_client: EngineClient, diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 43150191ac..aeade30646 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -13,6 +13,12 @@ import pybase64 as base64 from fastapi import Request from vllm.engine.protocol import EngineClient +from vllm.entrypoints.generate.base.serving import ( + GenerateBaseServing, + GenerationError, + clamp_prompt_logprobs, + format_token_id_placeholder, +) from vllm.entrypoints.openai.completion.protocol import ( CompletionLogProbs, CompletionRequest, @@ -27,12 +33,6 @@ from vllm.entrypoints.openai.engine.protocol import ( RequestResponseMetadata, UsageInfo, ) -from vllm.entrypoints.openai.engine.serving import ( - GenerationError, - OpenAIServing, - clamp_prompt_logprobs, - format_token_id_placeholder, -) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger @@ -50,7 +50,7 @@ from vllm.utils.collection_utils import as_list logger = init_logger(__name__) -class OpenAIServingCompletion(OpenAIServing): +class OpenAIServingCompletion(GenerateBaseServing): def __init__( self, engine_client: EngineClient, diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 72dddf539a..a62c462399 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -31,16 +31,16 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, ) +from vllm.entrypoints.generate.base.serving import ( + GenerateBaseServing, + GenerationError, +) from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.engine.protocol import ( DeltaMessage, ErrorResponse, RequestResponseMetadata, ) -from vllm.entrypoints.openai.engine.serving import ( - GenerationError, - OpenAIServing, -) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.parser.harmony_utils import ( build_harmony_preamble, @@ -147,7 +147,7 @@ def _extract_allowed_tools_from_mcp_requests( return allowed_tools_map -class OpenAIServingResponses(OpenAIServing): +class OpenAIServingResponses(GenerateBaseServing): def __init__( self, engine_client: EngineClient, diff --git a/vllm/entrypoints/pooling/base/serving.py b/vllm/entrypoints/pooling/base/serving.py index e1b83e711b..79f93a1480 100644 --- a/vllm/entrypoints/pooling/base/serving.py +++ b/vllm/entrypoints/pooling/base/serving.py @@ -12,31 +12,29 @@ from fastapi import Request from fastapi.responses import Response from starlette.datastructures import Headers -from vllm import PoolingParams, PoolingRequestOutput, envs +from vllm import PoolingRequestOutput, envs from vllm.config import VllmConfig from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ChatTemplateConfig from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.serve.engine.serving import BaseServing +from vllm.entrypoints.serve.engine.typing import AnyRequest from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.exceptions import VLLMNotFoundError -from vllm.inputs import EngineInput from vllm.lora.request import LoRARequest from vllm.renderers.base import BaseRenderer -from vllm.renderers.inputs.preprocess import extract_prompt_components from vllm.tracing import ( contains_trace_headers, extract_trace_headers, log_tracing_disabled_warning, ) -from vllm.utils import random_uuid from vllm.utils.async_utils import make_async, merge_async_iterators from ..typing import AnyPoolingRequest, PoolingServeContext from .io_processor import PoolingIOProcessor -class PoolingServingBase(ABC): +class PoolingBaseServing(ABC, BaseServing): request_id_prefix: ClassVar[str] def __init__( @@ -49,13 +47,16 @@ class PoolingServingBase(ABC): return_tokens_as_token_ids: bool = False, log_error_stack: bool = False, ): + super().__init__( + models=models, + model_config=models.model_config, + request_logger=request_logger, + ) + self.engine_client = engine_client - self.models = models - self.model_config = models.model_config self.renderer = engine_client.renderer self.vllm_config = engine_client.vllm_config self.max_model_len = self.model_config.max_model_len - self.request_logger = request_logger self.return_tokens_as_token_ids = return_tokens_as_token_ids self.log_error_stack = log_error_stack self.chat_template_config = chat_template_config @@ -118,7 +119,7 @@ class PoolingServingBase(ABC): ) self._validate_request(ctx) - self._maybe_get_adapters(ctx) + ctx.lora_request = self._maybe_get_adapters(ctx.request) return ctx async def _prepare_generators( @@ -207,26 +208,9 @@ class PoolingServingBase(ABC): ) -> Response: raise NotImplementedError - @staticmethod - def _base_request_id( - raw_request: Request | None, default: str | None = None - ) -> str | None: - """Pulls the request id to use from a header, if provided""" - if raw_request is not None and ( - (req_id := raw_request.headers.get("X-Request-Id")) is not None - ): - return req_id - - return random_uuid() if default is None else default - - def _is_model_supported(self, model_name: str | None) -> bool: - if not model_name: - return True - return self.models.is_base_model(model_name) - async def _check_model( self, - request: AnyPoolingRequest, + request: AnyRequest | AnyPoolingRequest, ) -> ErrorResponse | None: if self._is_model_supported(request.model): return None @@ -275,102 +259,8 @@ class PoolingServingBase(ABC): return None - def _maybe_get_adapters( - self, - ctx: PoolingServeContext, - supports_default_mm_loras: bool = False, - ): - request = ctx.request - if request.model in self.models.lora_requests: - ctx.lora_request = self.models.lora_requests[request.model] - return None - # Currently only support default modality specific loras - # if we have exactly one lora matched on the request. - if supports_default_mm_loras: - default_mm_lora = self._get_active_default_mm_loras(request) - if default_mm_lora is not None: - ctx.lora_request = default_mm_lora - - if self._is_model_supported(request.model): - return None - - # if _check_model has been called earlier, this will be unreachable - raise VLLMNotFoundError(f"The model `{request.model}` does not exist.") - - def _get_active_default_mm_loras( - self, request: AnyPoolingRequest - ) -> LoRARequest | None: - """Determine if there are any active default multimodal loras.""" - # TODO: Currently this is only enabled for chat completions - # to be better aligned with only being enabled for .generate - # when run offline. It would be nice to support additional - # tasks types in the future. - message_types = self._get_message_types(request) - default_mm_loras = set() - - for lora in self.models.lora_requests.values(): - # Best effort match for default multimodal lora adapters; - # There is probably a better way to do this, but currently - # this matches against the set of 'types' in any content lists - # up until '_', e.g., to match audio_url -> audio - if lora.lora_name in message_types: - default_mm_loras.add(lora) - - # Currently only support default modality specific loras if - # we have exactly one lora matched on the request. - if len(default_mm_loras) == 1: - return default_mm_loras.pop() - return None - - def _get_message_types(self, request: AnyPoolingRequest) -> set[str]: - """Retrieve the set of types from message content dicts up - until `_`; we use this to match potential multimodal data - with default per modality loras. - """ - message_types: set[str] = set() - - if not hasattr(request, "messages"): - return message_types - - messages = request.messages - if messages is None or isinstance(messages, (str, bytes)): - return message_types - - for message in messages: - if ( - isinstance(message, dict) - and "content" in message - and isinstance(message["content"], list) - ): - for content_dict in message["content"]: - if "type" in content_dict: - message_types.add(content_dict["type"].split("_")[0]) - return message_types - - def _log_inputs( - self, - request_id: str, - inputs: EngineInput, - params: PoolingParams, - lora_request: LoRARequest | None, - ) -> None: - if self.request_logger is None: - return - - components = extract_prompt_components(self.model_config, inputs) - - self.request_logger.log_inputs( - request_id, - components.text, - components.token_ids, - components.embeds, - params=params, - lora_request=lora_request, - ) - - -class PoolingServing(PoolingServingBase, ABC): +class PoolingServing(PoolingBaseServing, ABC): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/vllm/entrypoints/pooling/pooling/serving.py b/vllm/entrypoints/pooling/pooling/serving.py index 1783c861fc..a73ebf7723 100644 --- a/vllm/entrypoints/pooling/pooling/serving.py +++ b/vllm/entrypoints/pooling/pooling/serving.py @@ -9,7 +9,7 @@ from vllm.tasks import SupportedTask from vllm.utils.serial_utils import EmbedDType, Endianness from ..base.io_processor import PoolingIOProcessor -from ..base.serving import PoolingServingBase +from ..base.serving import PoolingBaseServing from ..factories import init_pooling_io_processors from ..typing import AnyPoolingRequest, PoolingServeContext from ..utils import ( @@ -30,7 +30,7 @@ from .protocol import ( logger = init_logger(__name__) -class ServingPooling(PoolingServingBase): +class ServingPooling(PoolingBaseServing): request_id_prefix = "pooling" def __init__( diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 3562530a14..ef331a1bf9 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -14,6 +14,10 @@ import pybase64 as base64 from fastapi import Request from vllm.engine.protocol import EngineClient +from vllm.entrypoints.generate.base.serving import ( + GenerateBaseServing, + clamp_prompt_logprobs, +) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionLogProb, ChatCompletionLogProbs, @@ -26,7 +30,6 @@ from vllm.entrypoints.openai.engine.protocol import ( RequestResponseMetadata, UsageInfo, ) -from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger @@ -55,7 +58,7 @@ from .protocol import ( logger = init_logger(__name__) -class ServingTokens(OpenAIServing): +class ServingTokens(GenerateBaseServing): """Provides Tokens IN <> Tokens OUT functionality to vLLM API.""" def __init__( diff --git a/vllm/entrypoints/serve/engine/serving.py b/vllm/entrypoints/serve/engine/serving.py index 409743a599..5908e20f71 100644 --- a/vllm/entrypoints/serve/engine/serving.py +++ b/vllm/entrypoints/serve/engine/serving.py @@ -11,9 +11,11 @@ from vllm.entrypoints.openai.models.serving import ( OpenAIModelRegistry, OpenAIServingModels, ) +from vllm.entrypoints.pooling.typing import AnyPoolingRequest from vllm.entrypoints.serve.engine.typing import AnyRequest from vllm.entrypoints.serve.utils.error_response import create_error_response from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.exceptions import VLLMNotFoundError from vllm.inputs import EngineInput from vllm.lora.request import LoRARequest from vllm.renderers.inputs.preprocess import ( @@ -37,7 +39,7 @@ class BaseServing: async def _check_model( self, - request: AnyRequest, + request: AnyRequest | AnyPoolingRequest, ) -> ErrorResponse | None: error_response = None @@ -123,7 +125,7 @@ class BaseServing: return random_uuid() if default is None else default - def _get_message_types(self, request: AnyRequest) -> set[str]: + def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]: """Retrieve the set of types from message content dicts up until `_`; we use this to match potential multimodal data with default per modality loras. @@ -148,7 +150,9 @@ class BaseServing: message_types.add(content_dict["type"].split("_")[0]) return message_types - def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None: + def _get_active_default_mm_loras( + self, request: AnyRequest | AnyPoolingRequest + ) -> LoRARequest | None: """Determine if there are any active default multimodal loras.""" # TODO: Currently this is only enabled for chat completions # to be better aligned with only being enabled for .generate @@ -173,7 +177,7 @@ class BaseServing: def _maybe_get_adapters( self, - request: AnyRequest, + request: AnyRequest | AnyPoolingRequest, supports_default_mm_loras: bool = False, ) -> LoRARequest | None: if request.model in self.models.lora_requests: @@ -190,4 +194,4 @@ class BaseServing: return None # if _check_model has been called earlier, this will be unreachable - raise ValueError(f"The model `{request.model}` does not exist.") + raise VLLMNotFoundError(f"The model `{request.model}` does not exist.") diff --git a/vllm/entrypoints/serve/sagemaker/api_router.py b/vllm/entrypoints/serve/sagemaker/api_router.py index 82c094d161..91b58d2b62 100644 --- a/vllm/entrypoints/serve/sagemaker/api_router.py +++ b/vllm/entrypoints/serve/sagemaker/api_router.py @@ -13,9 +13,8 @@ from fastapi.responses import JSONResponse, Response from vllm.config import ModelConfig from vllm.entrypoints.generate.factories import get_generate_invocation_types from vllm.entrypoints.openai.engine.protocol import ErrorResponse -from vllm.entrypoints.openai.engine.serving import OpenAIServing -from vllm.entrypoints.pooling.base.serving import PoolingServingBase from vllm.entrypoints.pooling.factories import get_pooling_invocation_types +from vllm.entrypoints.serve.engine.serving import BaseServing from vllm.entrypoints.serve.instrumentator.basic import base from vllm.entrypoints.serve.instrumentator.health import health from vllm.entrypoints.serve.utils.api_utils import validate_json_request @@ -24,7 +23,7 @@ from vllm.tasks import SupportedTask # TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers # (requires typing_extensions >= 4.13) RequestType = Any -GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServingBase | None] +GetHandlerFn = Callable[[Request], BaseServing | None] EndpointFn = Callable[[RequestType, Request], Awaitable[Any]] diff --git a/vllm/entrypoints/speech_to_text/base/serving.py b/vllm/entrypoints/speech_to_text/base/serving.py index 1cbc4d8f79..15be7dc3f5 100644 --- a/vllm/entrypoints/speech_to_text/base/serving.py +++ b/vllm/entrypoints/speech_to_text/base/serving.py @@ -16,13 +16,13 @@ from transformers import PreTrainedTokenizerBase import vllm.envs as envs from vllm.engine.protocol import EngineClient +from vllm.entrypoints.generate.base.serving import GenerateBaseServing from vllm.entrypoints.openai.engine.protocol import ( DeltaMessage, ErrorResponse, RequestResponseMetadata, UsageInfo, ) -from vllm.entrypoints.openai.engine.serving import OpenAIServing from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest from vllm.entrypoints.serve.utils.api_utils import get_max_tokens @@ -87,7 +87,7 @@ def asr_inter_chunk_separator( return "" if language and language.lower() in no_space_languages else " " -class OpenAISpeechToText(OpenAIServing): +class SpeechToTextBaseServing(GenerateBaseServing): """Base class for speech-to-text operations like transcription and translation.""" @@ -475,7 +475,7 @@ class OpenAISpeechToText(OpenAIServing): list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None input_len = ( - OpenAISpeechToText._get_decoder_prompt_len(engine_inputs) + SpeechToTextBaseServing._get_decoder_prompt_len(engine_inputs) if request.use_beam_search else 0 ) diff --git a/vllm/entrypoints/speech_to_text/realtime/serving.py b/vllm/entrypoints/speech_to_text/realtime/serving.py index e5b5e95127..bec2b95143 100644 --- a/vllm/entrypoints/speech_to_text/realtime/serving.py +++ b/vllm/entrypoints/speech_to_text/realtime/serving.py @@ -9,7 +9,7 @@ from typing import Literal, cast import numpy as np from vllm.engine.protocol import EngineClient, StreamingInput -from vllm.entrypoints.openai.engine.serving import OpenAIServing +from vllm.entrypoints.generate.base.serving import GenerateBaseServing from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import PromptType @@ -20,7 +20,7 @@ from vllm.renderers.inputs.preprocess import parse_model_prompt logger = init_logger(__name__) -class OpenAIServingRealtime(OpenAIServing): +class OpenAIServingRealtime(GenerateBaseServing): """Realtime audio transcription service via WebSocket streaming. Provides streaming audio-to-text transcription by transforming audio chunks diff --git a/vllm/entrypoints/speech_to_text/transcription/serving.py b/vllm/entrypoints/speech_to_text/transcription/serving.py index 0d5a3c9edb..a59ae8aa31 100644 --- a/vllm/entrypoints/speech_to_text/transcription/serving.py +++ b/vllm/entrypoints/speech_to_text/transcription/serving.py @@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.logger import init_logger from vllm.outputs import RequestOutput -from ..base.serving import OpenAISpeechToText +from ..base.serving import SpeechToTextBaseServing from .protocol import ( TranscriptionRequest, TranscriptionResponse, @@ -26,7 +26,7 @@ from .protocol import ( logger = init_logger(__name__) -class OpenAIServingTranscription(OpenAISpeechToText): +class OpenAIServingTranscription(SpeechToTextBaseServing): """Handles transcription requests.""" def __init__( diff --git a/vllm/entrypoints/speech_to_text/translation/serving.py b/vllm/entrypoints/speech_to_text/translation/serving.py index a3951250f1..2b7d834a2d 100644 --- a/vllm/entrypoints/speech_to_text/translation/serving.py +++ b/vllm/entrypoints/speech_to_text/translation/serving.py @@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.logger import init_logger from vllm.outputs import RequestOutput -from ..base.serving import OpenAISpeechToText +from ..base.serving import SpeechToTextBaseServing from .protocol import ( TranslationRequest, TranslationResponse, @@ -26,7 +26,7 @@ from .protocol import ( logger = init_logger(__name__) -class OpenAIServingTranslation(OpenAISpeechToText): +class OpenAIServingTranslation(SpeechToTextBaseServing): """Handles translation requests.""" def __init__( diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index fb4b880c48..20c54eb07f 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -4,6 +4,7 @@ from typing import Any from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption +from vllm.entrypoints.generate.base.serving import resolve_token_id_placeholder from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionLogProbs, ChatCompletionRequest, @@ -15,7 +16,6 @@ from vllm.entrypoints.openai.completion.protocol import ( CompletionResponseChoice, ) from vllm.entrypoints.openai.engine.protocol import ToolCall -from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.logger import init_logger diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 4e188336cb..45a3898d4a 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -264,7 +264,7 @@ class OnlineRenderer: chat_template_kwargs: dict[str, Any] | None, trust_request_chat_template: bool, ) -> ErrorResponse | None: - """Copied from OpenAIServing._validate_chat_template.""" + """Copied from GenerateBaseServing._validate_chat_template.""" if not trust_request_chat_template and ( request_chat_template is not None or ( @@ -287,7 +287,7 @@ class OnlineRenderer: *, skip_mm_cache: bool = False, ) -> list[EngineInput]: - """Copied from OpenAIServing._preprocess_completion.""" + """Copied from GenerateBaseServing._preprocess_completion.""" prompts = list[SingletonPrompt | bytes]() if prompt_embeds is not None: # embeds take higher priority prompts.extend(prompt_to_seq(prompt_embeds)) @@ -302,7 +302,7 @@ class OnlineRenderer: *, skip_mm_cache: bool = False, ) -> list[EngineInput]: - """Copied from OpenAIServing._preprocess_cmpl.""" + """Copied from GenerateBaseServing._preprocess_cmpl.""" renderer = self.renderer model_config = self.model_config @@ -339,7 +339,7 @@ class OnlineRenderer: *, skip_mm_cache: bool = False, ) -> tuple[list[ConversationMessage], list[EngineInput]]: - """Copied from OpenAIServing._preprocess_chat.""" + """Copied from GenerateBaseServing._preprocess_chat.""" renderer = self.renderer mm_config = self.model_config.multimodal_config