[Frontend] Refine the entrypoint class's inheritance hierarchy. (#47498)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-07-03 10:50:06 +00:00
committed by GitHub
parent 18f658bb31
commit a14f57a3ac
23 changed files with 90 additions and 191 deletions
@@ -78,7 +78,6 @@ def _create_mock_engine():
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
# renderer is accessed by OpenAIServing.__init__ and serving.py
mock_renderer = MagicMock()
mock_renderer.tokenizer = get_tokenizer(MODEL_NAME)
mock_engine.renderer = mock_renderer
@@ -7,20 +7,20 @@ from unittest.mock import MagicMock
import pytest
import vllm.envs as envs
from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing
from vllm.entrypoints.generate.base.serving import GenerateBaseServing, GenerationError
from vllm.envs import disable_envs_cache
@pytest.mark.asyncio
async def test_raise_if_error_raises_generation_error():
"""test _raise_if_error raises GenerationError"""
# create a minimal OpenAIServing instance
# create a minimal GenerateBaseServing instance
mock_engine = MagicMock()
mock_engine.model_config = MagicMock()
mock_engine.model_config.max_model_len = 100
mock_models = MagicMock()
serving = OpenAIServing(
serving = GenerateBaseServing(
engine_client=mock_engine,
models=mock_models,
request_logger=None,
@@ -47,7 +47,7 @@ async def test_convert_generation_error_to_streaming_response():
mock_engine.model_config.max_model_len = 100
mock_models = MagicMock()
serving = OpenAIServing(
serving = GenerateBaseServing(
engine_client=mock_engine,
models=mock_models,
request_logger=None,
@@ -77,7 +77,7 @@ def test_is_model_supported_skip_name_validation_env(
mock_models = MagicMock()
mock_models.is_base_model.return_value = False
serving = OpenAIServing(
serving = GenerateBaseServing(
engine_client=mock_engine,
models=mock_models,
request_logger=None,
@@ -127,13 +127,13 @@ def test_responses_api_logprobs_with_return_tokens_as_token_ids():
"""Test that return_tokens_as_token_ids works in Responses API logprobs."""
from unittest.mock import MagicMock
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
from vllm.logprobs import Logprob as SampleLogprob
serving = MagicMock(spec=OpenAIServingResponses)
serving.return_tokens_as_token_ids = True
serving._get_decoded_token = OpenAIServing._get_decoded_token
serving._get_decoded_token = GenerateBaseServing._get_decoded_token
tokenizer = MagicMock()
tokenizer.decode = lambda token_id: "decoded"
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.entrypoints.serve.lora.protocol import (
LoadLoRAAdapterRequest,
@@ -136,7 +136,7 @@ async def test_unload_lora_adapter_not_found():
assert response.error.code == HTTPStatus.NOT_FOUND
class _ConcretePoolingServing(PoolingServingBase):
class _ConcretePoolingServing(PoolingBaseServing):
"""Minimal concrete subclass used only in these unit tests."""
request_id_prefix = "test"
@@ -178,7 +178,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
serving = _make_pooling_serving(lora_name)
ctx = _make_pooling_ctx(lora_name)
serving._maybe_get_adapters(ctx)
ctx.lora_request = serving._maybe_get_adapters(ctx.request)
assert ctx.lora_request is not None
assert ctx.lora_request.lora_name == lora_name
@@ -190,4 +190,4 @@ def test_pooling_maybe_get_adapters_unknown_model_raises():
ctx = _make_pooling_ctx("unknown-model")
with pytest.raises(VLLMNotFoundError):
serving._maybe_get_adapters(ctx)
serving._maybe_get_adapters(ctx.request)
@@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, Mock
import pytest
from vllm.entrypoints.speech_to_text.base.serving import OpenAISpeechToText
from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionResponse
@@ -43,7 +43,7 @@ async def test_non_streaming_cancel_aborts_engine_requests(
is_tracing_enabled=AsyncMock(return_value=False),
)
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
server.engine_client = engine_client
server.task_type = "transcribe"
server.models = SimpleNamespace(model_name=lambda: "audio")
@@ -112,7 +112,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
{"prompt": "chunk-1"},
{"prompt": "chunk-2"},
]
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
server.engine_client = engine_client
server.task_type = "transcribe"
server.models = SimpleNamespace(model_name=lambda: "audio")
@@ -170,7 +170,7 @@ async def test_language_detection_cancel_aborts_engine_request():
abort=AsyncMock(),
)
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
server.engine_client = engine_client
server.asr_config = SimpleNamespace()
server.tokenizer = Mock()
@@ -25,7 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import (
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.speech_to_text.base.serving import (
OpenAISpeechToText,
SpeechToTextBaseServing,
asr_inter_chunk_separator,
)
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
@@ -234,7 +234,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language():
"vllm.model_executor.model_loader.get_model_cls",
return_value=_StubTranscriptionModel,
),
patch.object(OpenAISpeechToText, "_preprocess_speech_to_text", preprocess_mock),
patch.object(
SpeechToTextBaseServing, "_preprocess_speech_to_text", preprocess_mock
),
):
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
@@ -53,7 +53,7 @@ class ServeContext(Generic[RequestT]):
model_config = ConfigDict(arbitrary_types_allowed=True)
class OpenAIServing(BaseServing, BeamSearchOnlineMixin):
class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
request_id_prefix: ClassVar[str] = """
A short string prepended to every request’s ID.
"""
@@ -23,8 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import (
OpenAIBaseModel,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput, tokens_input
from vllm.logger import init_logger
@@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
# ============================================================================
class ServingGenerativeScoring(OpenAIServing):
class ServingGenerativeScoring(BaseServing):
"""Serving class for generative scoring computation.
This class handles computing the probability of specified token IDs
@@ -164,10 +164,12 @@ class ServingGenerativeScoring(OpenAIServing):
request_logger: RequestLogger | None,
) -> None:
super().__init__(
engine_client=engine_client,
models=models,
model_config=engine_client.model_config,
request_logger=request_logger,
)
self.engine_client = engine_client
self.renderer = engine_client.renderer
async def create_generative_scoring(
self,
@@ -19,6 +19,12 @@ from vllm.entrypoints.chat_utils import (
ConversationMessage,
make_tool_call_id,
)
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
GenerationError,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProb,
ChatCompletionLogProbs,
@@ -40,12 +46,6 @@ from vllm.entrypoints.openai.engine.protocol import (
ToolCall,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import (
GenerationError,
OpenAIServing,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -76,7 +76,7 @@ def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]:
counted in ``usage.prompt_tokens``.
"""
mm_placeholders = cast(
"MultiModalPlaceholders | None", engine_input.get("mm_placeholders")
MultiModalPlaceholders | None, engine_input.get("mm_placeholders")
)
return {
modality: sum(p.length for p in ranges)
@@ -101,7 +101,7 @@ def _make_prompt_tokens_details(
)
class OpenAIServingChat(OpenAIServing):
class OpenAIServingChat(GenerateBaseServing):
def __init__(
self,
engine_client: EngineClient,
@@ -13,6 +13,12 @@ import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
GenerationError,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionLogProbs,
CompletionRequest,
@@ -27,12 +33,6 @@ from vllm.entrypoints.openai.engine.protocol import (
RequestResponseMetadata,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import (
GenerationError,
OpenAIServing,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -50,7 +50,7 @@ from vllm.utils.collection_utils import as_list
logger = init_logger(__name__)
class OpenAIServingCompletion(OpenAIServing):
class OpenAIServingCompletion(GenerateBaseServing):
def __init__(
self,
engine_client: EngineClient,
+5 -5
View File
@@ -31,16 +31,16 @@ from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
GenerationError,
)
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ErrorResponse,
RequestResponseMetadata,
)
from vllm.entrypoints.openai.engine.serving import (
GenerationError,
OpenAIServing,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.parser.harmony_utils import (
build_harmony_preamble,
@@ -147,7 +147,7 @@ def _extract_allowed_tools_from_mcp_requests(
return allowed_tools_map
class OpenAIServingResponses(OpenAIServing):
class OpenAIServingResponses(GenerateBaseServing):
def __init__(
self,
engine_client: EngineClient,
+13 -123
View File
@@ -12,31 +12,29 @@ from fastapi import Request
from fastapi.responses import Response
from starlette.datastructures import Headers
from vllm import PoolingParams, PoolingRequestOutput, envs
from vllm import PoolingRequestOutput, envs
from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateConfig
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import VLLMNotFoundError
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers.base import BaseRenderer
from vllm.renderers.inputs.preprocess import extract_prompt_components
from vllm.tracing import (
contains_trace_headers,
extract_trace_headers,
log_tracing_disabled_warning,
)
from vllm.utils import random_uuid
from vllm.utils.async_utils import make_async, merge_async_iterators
from ..typing import AnyPoolingRequest, PoolingServeContext
from .io_processor import PoolingIOProcessor
class PoolingServingBase(ABC):
class PoolingBaseServing(ABC, BaseServing):
request_id_prefix: ClassVar[str]
def __init__(
@@ -49,13 +47,16 @@ class PoolingServingBase(ABC):
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
super().__init__(
models=models,
model_config=models.model_config,
request_logger=request_logger,
)
self.engine_client = engine_client
self.models = models
self.model_config = models.model_config
self.renderer = engine_client.renderer
self.vllm_config = engine_client.vllm_config
self.max_model_len = self.model_config.max_model_len
self.request_logger = request_logger
self.return_tokens_as_token_ids = return_tokens_as_token_ids
self.log_error_stack = log_error_stack
self.chat_template_config = chat_template_config
@@ -118,7 +119,7 @@ class PoolingServingBase(ABC):
)
self._validate_request(ctx)
self._maybe_get_adapters(ctx)
ctx.lora_request = self._maybe_get_adapters(ctx.request)
return ctx
async def _prepare_generators(
@@ -207,26 +208,9 @@ class PoolingServingBase(ABC):
) -> Response:
raise NotImplementedError
@staticmethod
def _base_request_id(
raw_request: Request | None, default: str | None = None
) -> str | None:
"""Pulls the request id to use from a header, if provided"""
if raw_request is not None and (
(req_id := raw_request.headers.get("X-Request-Id")) is not None
):
return req_id
return random_uuid() if default is None else default
def _is_model_supported(self, model_name: str | None) -> bool:
if not model_name:
return True
return self.models.is_base_model(model_name)
async def _check_model(
self,
request: AnyPoolingRequest,
request: AnyRequest | AnyPoolingRequest,
) -> ErrorResponse | None:
if self._is_model_supported(request.model):
return None
@@ -275,102 +259,8 @@ class PoolingServingBase(ABC):
return None
def _maybe_get_adapters(
self,
ctx: PoolingServeContext,
supports_default_mm_loras: bool = False,
):
request = ctx.request
if request.model in self.models.lora_requests:
ctx.lora_request = self.models.lora_requests[request.model]
return None
# Currently only support default modality specific loras
# if we have exactly one lora matched on the request.
if supports_default_mm_loras:
default_mm_lora = self._get_active_default_mm_loras(request)
if default_mm_lora is not None:
ctx.lora_request = default_mm_lora
if self._is_model_supported(request.model):
return None
# if _check_model has been called earlier, this will be unreachable
raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
def _get_active_default_mm_loras(
self, request: AnyPoolingRequest
) -> LoRARequest | None:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
# when run offline. It would be nice to support additional
# tasks types in the future.
message_types = self._get_message_types(request)
default_mm_loras = set()
for lora in self.models.lora_requests.values():
# Best effort match for default multimodal lora adapters;
# There is probably a better way to do this, but currently
# this matches against the set of 'types' in any content lists
# up until '_', e.g., to match audio_url -> audio
if lora.lora_name in message_types:
default_mm_loras.add(lora)
# Currently only support default modality specific loras if
# we have exactly one lora matched on the request.
if len(default_mm_loras) == 1:
return default_mm_loras.pop()
return None
def _get_message_types(self, request: AnyPoolingRequest) -> set[str]:
"""Retrieve the set of types from message content dicts up
until `_`; we use this to match potential multimodal data
with default per modality loras.
"""
message_types: set[str] = set()
if not hasattr(request, "messages"):
return message_types
messages = request.messages
if messages is None or isinstance(messages, (str, bytes)):
return message_types
for message in messages:
if (
isinstance(message, dict)
and "content" in message
and isinstance(message["content"], list)
):
for content_dict in message["content"]:
if "type" in content_dict:
message_types.add(content_dict["type"].split("_")[0])
return message_types
def _log_inputs(
self,
request_id: str,
inputs: EngineInput,
params: PoolingParams,
lora_request: LoRARequest | None,
) -> None:
if self.request_logger is None:
return
components = extract_prompt_components(self.model_config, inputs)
self.request_logger.log_inputs(
request_id,
components.text,
components.token_ids,
components.embeds,
params=params,
lora_request=lora_request,
)
class PoolingServing(PoolingServingBase, ABC):
class PoolingServing(PoolingBaseServing, ABC):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
+2 -2
View File
@@ -9,7 +9,7 @@ from vllm.tasks import SupportedTask
from vllm.utils.serial_utils import EmbedDType, Endianness
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingServingBase
from ..base.serving import PoolingBaseServing
from ..factories import init_pooling_io_processors
from ..typing import AnyPoolingRequest, PoolingServeContext
from ..utils import (
@@ -30,7 +30,7 @@ from .protocol import (
logger = init_logger(__name__)
class ServingPooling(PoolingServingBase):
class ServingPooling(PoolingBaseServing):
request_id_prefix = "pooling"
def __init__(
@@ -14,6 +14,10 @@ import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
clamp_prompt_logprobs,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProb,
ChatCompletionLogProbs,
@@ -26,7 +30,6 @@ from vllm.entrypoints.openai.engine.protocol import (
RequestResponseMetadata,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -55,7 +58,7 @@ from .protocol import (
logger = init_logger(__name__)
class ServingTokens(OpenAIServing):
class ServingTokens(GenerateBaseServing):
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
def __init__(
+9 -5
View File
@@ -11,9 +11,11 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import VLLMNotFoundError
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers.inputs.preprocess import (
@@ -37,7 +39,7 @@ class BaseServing:
async def _check_model(
self,
request: AnyRequest,
request: AnyRequest | AnyPoolingRequest,
) -> ErrorResponse | None:
error_response = None
@@ -123,7 +125,7 @@ class BaseServing:
return random_uuid() if default is None else default
def _get_message_types(self, request: AnyRequest) -> set[str]:
def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]:
"""Retrieve the set of types from message content dicts up
until `_`; we use this to match potential multimodal data
with default per modality loras.
@@ -148,7 +150,9 @@ class BaseServing:
message_types.add(content_dict["type"].split("_")[0])
return message_types
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
def _get_active_default_mm_loras(
self, request: AnyRequest | AnyPoolingRequest
) -> LoRARequest | None:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
@@ -173,7 +177,7 @@ class BaseServing:
def _maybe_get_adapters(
self,
request: AnyRequest,
request: AnyRequest | AnyPoolingRequest,
supports_default_mm_loras: bool = False,
) -> LoRARequest | None:
if request.model in self.models.lora_requests:
@@ -190,4 +194,4 @@ class BaseServing:
return None
# if _check_model has been called earlier, this will be unreachable
raise ValueError(f"The model `{request.model}` does not exist.")
raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
@@ -13,9 +13,8 @@ from fastapi.responses import JSONResponse, Response
from vllm.config import ModelConfig
from vllm.entrypoints.generate.factories import get_generate_invocation_types
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.instrumentator.basic import base
from vllm.entrypoints.serve.instrumentator.health import health
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
@@ -24,7 +23,7 @@ from vllm.tasks import SupportedTask
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
# (requires typing_extensions >= 4.13)
RequestType = Any
GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServingBase | None]
GetHandlerFn = Callable[[Request], BaseServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
@@ -16,13 +16,13 @@ from transformers import PreTrainedTokenizerBase
import vllm.envs as envs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ErrorResponse,
RequestResponseMetadata,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
@@ -87,7 +87,7 @@ def asr_inter_chunk_separator(
return "" if language and language.lower() in no_space_languages else " "
class OpenAISpeechToText(OpenAIServing):
class SpeechToTextBaseServing(GenerateBaseServing):
"""Base class for speech-to-text operations like transcription and
translation."""
@@ -475,7 +475,7 @@ class OpenAISpeechToText(OpenAIServing):
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
input_len = (
OpenAISpeechToText._get_decoder_prompt_len(engine_inputs)
SpeechToTextBaseServing._get_decoder_prompt_len(engine_inputs)
if request.use_beam_search
else 0
)
@@ -9,7 +9,7 @@ from typing import Literal, cast
import numpy as np
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import PromptType
@@ -20,7 +20,7 @@ from vllm.renderers.inputs.preprocess import parse_model_prompt
logger = init_logger(__name__)
class OpenAIServingRealtime(OpenAIServing):
class OpenAIServingRealtime(GenerateBaseServing):
"""Realtime audio transcription service via WebSocket streaming.
Provides streaming audio-to-text transcription by transforming audio chunks
@@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from ..base.serving import OpenAISpeechToText
from ..base.serving import SpeechToTextBaseServing
from .protocol import (
TranscriptionRequest,
TranscriptionResponse,
@@ -26,7 +26,7 @@ from .protocol import (
logger = init_logger(__name__)
class OpenAIServingTranscription(OpenAISpeechToText):
class OpenAIServingTranscription(SpeechToTextBaseServing):
"""Handles transcription requests."""
def __init__(
@@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from ..base.serving import OpenAISpeechToText
from ..base.serving import SpeechToTextBaseServing
from .protocol import (
TranslationRequest,
TranslationResponse,
@@ -26,7 +26,7 @@ from .protocol import (
logger = init_logger(__name__)
class OpenAIServingTranslation(OpenAISpeechToText):
class OpenAIServingTranslation(SpeechToTextBaseServing):
"""Handles translation requests."""
def __init__(
+1 -1
View File
@@ -4,6 +4,7 @@ from typing import Any
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.generate.base.serving import resolve_token_id_placeholder
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
@@ -15,7 +16,6 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionResponseChoice,
)
from vllm.entrypoints.openai.engine.protocol import ToolCall
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
+4 -4
View File
@@ -264,7 +264,7 @@ class OnlineRenderer:
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
) -> ErrorResponse | None:
"""Copied from OpenAIServing._validate_chat_template."""
"""Copied from GenerateBaseServing._validate_chat_template."""
if not trust_request_chat_template and (
request_chat_template is not None
or (
@@ -287,7 +287,7 @@ class OnlineRenderer:
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_completion."""
"""Copied from GenerateBaseServing._preprocess_completion."""
prompts = list[SingletonPrompt | bytes]()
if prompt_embeds is not None: # embeds take higher priority
prompts.extend(prompt_to_seq(prompt_embeds))
@@ -302,7 +302,7 @@ class OnlineRenderer:
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_cmpl."""
"""Copied from GenerateBaseServing._preprocess_cmpl."""
renderer = self.renderer
model_config = self.model_config
@@ -339,7 +339,7 @@ class OnlineRenderer:
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]]:
"""Copied from OpenAIServing._preprocess_chat."""
"""Copied from GenerateBaseServing._preprocess_chat."""
renderer = self.renderer
mm_config = self.model_config.multimodal_config