[Frontend] Refine the entrypoint class's inheritance hierarchy. (#47498)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -78,7 +78,6 @@ def _create_mock_engine():
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
|
||||
# renderer is accessed by OpenAIServing.__init__ and serving.py
|
||||
mock_renderer = MagicMock()
|
||||
mock_renderer.tokenizer = get_tokenizer(MODEL_NAME)
|
||||
mock_engine.renderer = mock_renderer
|
||||
|
||||
@@ -7,20 +7,20 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing, GenerationError
|
||||
from vllm.envs import disable_envs_cache
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_if_error_raises_generation_error():
|
||||
"""test _raise_if_error raises GenerationError"""
|
||||
# create a minimal OpenAIServing instance
|
||||
# create a minimal GenerateBaseServing instance
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.model_config = MagicMock()
|
||||
mock_engine.model_config.max_model_len = 100
|
||||
mock_models = MagicMock()
|
||||
|
||||
serving = OpenAIServing(
|
||||
serving = GenerateBaseServing(
|
||||
engine_client=mock_engine,
|
||||
models=mock_models,
|
||||
request_logger=None,
|
||||
@@ -47,7 +47,7 @@ async def test_convert_generation_error_to_streaming_response():
|
||||
mock_engine.model_config.max_model_len = 100
|
||||
mock_models = MagicMock()
|
||||
|
||||
serving = OpenAIServing(
|
||||
serving = GenerateBaseServing(
|
||||
engine_client=mock_engine,
|
||||
models=mock_models,
|
||||
request_logger=None,
|
||||
@@ -77,7 +77,7 @@ def test_is_model_supported_skip_name_validation_env(
|
||||
mock_models = MagicMock()
|
||||
mock_models.is_base_model.return_value = False
|
||||
|
||||
serving = OpenAIServing(
|
||||
serving = GenerateBaseServing(
|
||||
engine_client=mock_engine,
|
||||
models=mock_models,
|
||||
request_logger=None,
|
||||
|
||||
@@ -127,13 +127,13 @@ def test_responses_api_logprobs_with_return_tokens_as_token_ids():
|
||||
"""Test that return_tokens_as_token_ids works in Responses API logprobs."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
|
||||
serving = MagicMock(spec=OpenAIServingResponses)
|
||||
serving.return_tokens_as_token_ids = True
|
||||
serving._get_decoded_token = OpenAIServing._get_decoded_token
|
||||
serving._get_decoded_token = GenerateBaseServing._get_decoded_token
|
||||
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.decode = lambda token_id: "decoded"
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingBaseServing
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.entrypoints.serve.lora.protocol import (
|
||||
LoadLoRAAdapterRequest,
|
||||
@@ -136,7 +136,7 @@ async def test_unload_lora_adapter_not_found():
|
||||
assert response.error.code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
class _ConcretePoolingServing(PoolingServingBase):
|
||||
class _ConcretePoolingServing(PoolingBaseServing):
|
||||
"""Minimal concrete subclass used only in these unit tests."""
|
||||
|
||||
request_id_prefix = "test"
|
||||
@@ -178,7 +178,7 @@ def test_pooling_maybe_get_adapters_lora_name_sets_lora_request():
|
||||
serving = _make_pooling_serving(lora_name)
|
||||
ctx = _make_pooling_ctx(lora_name)
|
||||
|
||||
serving._maybe_get_adapters(ctx)
|
||||
ctx.lora_request = serving._maybe_get_adapters(ctx.request)
|
||||
|
||||
assert ctx.lora_request is not None
|
||||
assert ctx.lora_request.lora_name == lora_name
|
||||
@@ -190,4 +190,4 @@ def test_pooling_maybe_get_adapters_unknown_model_raises():
|
||||
ctx = _make_pooling_ctx("unknown-model")
|
||||
|
||||
with pytest.raises(VLLMNotFoundError):
|
||||
serving._maybe_get_adapters(ctx)
|
||||
serving._maybe_get_adapters(ctx.request)
|
||||
|
||||
@@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.speech_to_text.base.serving import OpenAISpeechToText
|
||||
from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionResponse
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ async def test_non_streaming_cancel_aborts_engine_requests(
|
||||
is_tracing_enabled=AsyncMock(return_value=False),
|
||||
)
|
||||
|
||||
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
|
||||
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
|
||||
server.engine_client = engine_client
|
||||
server.task_type = "transcribe"
|
||||
server.models = SimpleNamespace(model_name=lambda: "audio")
|
||||
@@ -112,7 +112,7 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
{"prompt": "chunk-1"},
|
||||
{"prompt": "chunk-2"},
|
||||
]
|
||||
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
|
||||
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
|
||||
server.engine_client = engine_client
|
||||
server.task_type = "transcribe"
|
||||
server.models = SimpleNamespace(model_name=lambda: "audio")
|
||||
@@ -170,7 +170,7 @@ async def test_language_detection_cancel_aborts_engine_request():
|
||||
abort=AsyncMock(),
|
||||
)
|
||||
|
||||
server = OpenAISpeechToText.__new__(OpenAISpeechToText)
|
||||
server = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
|
||||
server.engine_client = engine_client
|
||||
server.asr_config = SimpleNamespace()
|
||||
server.tokenizer = Mock()
|
||||
|
||||
+4
-2
@@ -25,7 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.speech_to_text.base.serving import (
|
||||
OpenAISpeechToText,
|
||||
SpeechToTextBaseServing,
|
||||
asr_inter_chunk_separator,
|
||||
)
|
||||
from vllm.entrypoints.speech_to_text.transcription.protocol import TranscriptionRequest
|
||||
@@ -234,7 +234,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language():
|
||||
"vllm.model_executor.model_loader.get_model_cls",
|
||||
return_value=_StubTranscriptionModel,
|
||||
),
|
||||
patch.object(OpenAISpeechToText, "_preprocess_speech_to_text", preprocess_mock),
|
||||
patch.object(
|
||||
SpeechToTextBaseServing, "_preprocess_speech_to_text", preprocess_mock
|
||||
),
|
||||
):
|
||||
serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
|
||||
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ class ServeContext(Generic[RequestT]):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
|
||||
class OpenAIServing(BaseServing, BeamSearchOnlineMixin):
|
||||
class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
request_id_prefix: ClassVar[str] = """
|
||||
A short string prepended to every request’s ID.
|
||||
"""
|
||||
@@ -23,8 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
OpenAIBaseModel,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
@@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class ServingGenerativeScoring(OpenAIServing):
|
||||
class ServingGenerativeScoring(BaseServing):
|
||||
"""Serving class for generative scoring computation.
|
||||
|
||||
This class handles computing the probability of specified token IDs
|
||||
@@ -164,10 +164,12 @@ class ServingGenerativeScoring(OpenAIServing):
|
||||
request_logger: RequestLogger | None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
model_config=engine_client.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
self.engine_client = engine_client
|
||||
self.renderer = engine_client.renderer
|
||||
|
||||
async def create_generative_scoring(
|
||||
self,
|
||||
|
||||
@@ -19,6 +19,12 @@ from vllm.entrypoints.chat_utils import (
|
||||
ConversationMessage,
|
||||
make_tool_call_id,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProb,
|
||||
ChatCompletionLogProbs,
|
||||
@@ -40,12 +46,6 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
ToolCall,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
@@ -76,7 +76,7 @@ def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]:
|
||||
counted in ``usage.prompt_tokens``.
|
||||
"""
|
||||
mm_placeholders = cast(
|
||||
"MultiModalPlaceholders | None", engine_input.get("mm_placeholders")
|
||||
MultiModalPlaceholders | None, engine_input.get("mm_placeholders")
|
||||
)
|
||||
return {
|
||||
modality: sum(p.length for p in ranges)
|
||||
@@ -101,7 +101,7 @@ def _make_prompt_tokens_details(
|
||||
)
|
||||
|
||||
|
||||
class OpenAIServingChat(OpenAIServing):
|
||||
class OpenAIServingChat(GenerateBaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
|
||||
@@ -13,6 +13,12 @@ import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionLogProbs,
|
||||
CompletionRequest,
|
||||
@@ -27,12 +33,6 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
@@ -50,7 +50,7 @@ from vllm.utils.collection_utils import as_list
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIServingCompletion(OpenAIServing):
|
||||
class OpenAIServingCompletion(GenerateBaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
|
||||
@@ -31,16 +31,16 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
GenerationError,
|
||||
)
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import (
|
||||
GenerationError,
|
||||
OpenAIServing,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
@@ -147,7 +147,7 @@ def _extract_allowed_tools_from_mcp_requests(
|
||||
return allowed_tools_map
|
||||
|
||||
|
||||
class OpenAIServingResponses(OpenAIServing):
|
||||
class OpenAIServingResponses(GenerateBaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
|
||||
@@ -12,31 +12,29 @@ from fastapi import Request
|
||||
from fastapi.responses import Response
|
||||
from starlette.datastructures import Headers
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, envs
|
||||
from vllm import PoolingRequestOutput, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMNotFoundError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.tracing import (
|
||||
contains_trace_headers,
|
||||
extract_trace_headers,
|
||||
log_tracing_disabled_warning,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.async_utils import make_async, merge_async_iterators
|
||||
|
||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||
from .io_processor import PoolingIOProcessor
|
||||
|
||||
|
||||
class PoolingServingBase(ABC):
|
||||
class PoolingBaseServing(ABC, BaseServing):
|
||||
request_id_prefix: ClassVar[str]
|
||||
|
||||
def __init__(
|
||||
@@ -49,13 +47,16 @@ class PoolingServingBase(ABC):
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=models.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.engine_client = engine_client
|
||||
self.models = models
|
||||
self.model_config = models.model_config
|
||||
self.renderer = engine_client.renderer
|
||||
self.vllm_config = engine_client.vllm_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
self.request_logger = request_logger
|
||||
self.return_tokens_as_token_ids = return_tokens_as_token_ids
|
||||
self.log_error_stack = log_error_stack
|
||||
self.chat_template_config = chat_template_config
|
||||
@@ -118,7 +119,7 @@ class PoolingServingBase(ABC):
|
||||
)
|
||||
|
||||
self._validate_request(ctx)
|
||||
self._maybe_get_adapters(ctx)
|
||||
ctx.lora_request = self._maybe_get_adapters(ctx.request)
|
||||
return ctx
|
||||
|
||||
async def _prepare_generators(
|
||||
@@ -207,26 +208,9 @@ class PoolingServingBase(ABC):
|
||||
) -> Response:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def _base_request_id(
|
||||
raw_request: Request | None, default: str | None = None
|
||||
) -> str | None:
|
||||
"""Pulls the request id to use from a header, if provided"""
|
||||
if raw_request is not None and (
|
||||
(req_id := raw_request.headers.get("X-Request-Id")) is not None
|
||||
):
|
||||
return req_id
|
||||
|
||||
return random_uuid() if default is None else default
|
||||
|
||||
def _is_model_supported(self, model_name: str | None) -> bool:
|
||||
if not model_name:
|
||||
return True
|
||||
return self.models.is_base_model(model_name)
|
||||
|
||||
async def _check_model(
|
||||
self,
|
||||
request: AnyPoolingRequest,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
) -> ErrorResponse | None:
|
||||
if self._is_model_supported(request.model):
|
||||
return None
|
||||
@@ -275,102 +259,8 @@ class PoolingServingBase(ABC):
|
||||
|
||||
return None
|
||||
|
||||
def _maybe_get_adapters(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
supports_default_mm_loras: bool = False,
|
||||
):
|
||||
request = ctx.request
|
||||
if request.model in self.models.lora_requests:
|
||||
ctx.lora_request = self.models.lora_requests[request.model]
|
||||
return None
|
||||
|
||||
# Currently only support default modality specific loras
|
||||
# if we have exactly one lora matched on the request.
|
||||
if supports_default_mm_loras:
|
||||
default_mm_lora = self._get_active_default_mm_loras(request)
|
||||
if default_mm_lora is not None:
|
||||
ctx.lora_request = default_mm_lora
|
||||
|
||||
if self._is_model_supported(request.model):
|
||||
return None
|
||||
|
||||
# if _check_model has been called earlier, this will be unreachable
|
||||
raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
|
||||
|
||||
def _get_active_default_mm_loras(
|
||||
self, request: AnyPoolingRequest
|
||||
) -> LoRARequest | None:
|
||||
"""Determine if there are any active default multimodal loras."""
|
||||
# TODO: Currently this is only enabled for chat completions
|
||||
# to be better aligned with only being enabled for .generate
|
||||
# when run offline. It would be nice to support additional
|
||||
# tasks types in the future.
|
||||
message_types = self._get_message_types(request)
|
||||
default_mm_loras = set()
|
||||
|
||||
for lora in self.models.lora_requests.values():
|
||||
# Best effort match for default multimodal lora adapters;
|
||||
# There is probably a better way to do this, but currently
|
||||
# this matches against the set of 'types' in any content lists
|
||||
# up until '_', e.g., to match audio_url -> audio
|
||||
if lora.lora_name in message_types:
|
||||
default_mm_loras.add(lora)
|
||||
|
||||
# Currently only support default modality specific loras if
|
||||
# we have exactly one lora matched on the request.
|
||||
if len(default_mm_loras) == 1:
|
||||
return default_mm_loras.pop()
|
||||
return None
|
||||
|
||||
def _get_message_types(self, request: AnyPoolingRequest) -> set[str]:
|
||||
"""Retrieve the set of types from message content dicts up
|
||||
until `_`; we use this to match potential multimodal data
|
||||
with default per modality loras.
|
||||
"""
|
||||
message_types: set[str] = set()
|
||||
|
||||
if not hasattr(request, "messages"):
|
||||
return message_types
|
||||
|
||||
messages = request.messages
|
||||
if messages is None or isinstance(messages, (str, bytes)):
|
||||
return message_types
|
||||
|
||||
for message in messages:
|
||||
if (
|
||||
isinstance(message, dict)
|
||||
and "content" in message
|
||||
and isinstance(message["content"], list)
|
||||
):
|
||||
for content_dict in message["content"]:
|
||||
if "type" in content_dict:
|
||||
message_types.add(content_dict["type"].split("_")[0])
|
||||
return message_types
|
||||
|
||||
def _log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
inputs: EngineInput,
|
||||
params: PoolingParams,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
if self.request_logger is None:
|
||||
return
|
||||
|
||||
components = extract_prompt_components(self.model_config, inputs)
|
||||
|
||||
self.request_logger.log_inputs(
|
||||
request_id,
|
||||
components.text,
|
||||
components.token_ids,
|
||||
components.embeds,
|
||||
params=params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
|
||||
class PoolingServing(PoolingServingBase, ABC):
|
||||
class PoolingServing(PoolingBaseServing, ABC):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.tasks import SupportedTask
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingServingBase
|
||||
from ..base.serving import PoolingBaseServing
|
||||
from ..factories import init_pooling_io_processors
|
||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||
from ..utils import (
|
||||
@@ -30,7 +30,7 @@ from .protocol import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingPooling(PoolingServingBase):
|
||||
class ServingPooling(PoolingBaseServing):
|
||||
request_id_prefix = "pooling"
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -14,6 +14,10 @@ import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
clamp_prompt_logprobs,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProb,
|
||||
ChatCompletionLogProbs,
|
||||
@@ -26,7 +30,6 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
@@ -55,7 +58,7 @@ from .protocol import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingTokens(OpenAIServing):
|
||||
class ServingTokens(GenerateBaseServing):
|
||||
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -11,9 +11,11 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMNotFoundError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
@@ -37,7 +39,7 @@ class BaseServing:
|
||||
|
||||
async def _check_model(
|
||||
self,
|
||||
request: AnyRequest,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
) -> ErrorResponse | None:
|
||||
error_response = None
|
||||
|
||||
@@ -123,7 +125,7 @@ class BaseServing:
|
||||
|
||||
return random_uuid() if default is None else default
|
||||
|
||||
def _get_message_types(self, request: AnyRequest) -> set[str]:
|
||||
def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]:
|
||||
"""Retrieve the set of types from message content dicts up
|
||||
until `_`; we use this to match potential multimodal data
|
||||
with default per modality loras.
|
||||
@@ -148,7 +150,9 @@ class BaseServing:
|
||||
message_types.add(content_dict["type"].split("_")[0])
|
||||
return message_types
|
||||
|
||||
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
|
||||
def _get_active_default_mm_loras(
|
||||
self, request: AnyRequest | AnyPoolingRequest
|
||||
) -> LoRARequest | None:
|
||||
"""Determine if there are any active default multimodal loras."""
|
||||
# TODO: Currently this is only enabled for chat completions
|
||||
# to be better aligned with only being enabled for .generate
|
||||
@@ -173,7 +177,7 @@ class BaseServing:
|
||||
|
||||
def _maybe_get_adapters(
|
||||
self,
|
||||
request: AnyRequest,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
supports_default_mm_loras: bool = False,
|
||||
) -> LoRARequest | None:
|
||||
if request.model in self.models.lora_requests:
|
||||
@@ -190,4 +194,4 @@ class BaseServing:
|
||||
return None
|
||||
|
||||
# if _check_model has been called earlier, this will be unreachable
|
||||
raise ValueError(f"The model `{request.model}` does not exist.")
|
||||
raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
|
||||
|
||||
@@ -13,9 +13,8 @@ from fastapi.responses import JSONResponse, Response
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
|
||||
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
@@ -24,7 +23,7 @@ from vllm.tasks import SupportedTask
|
||||
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
|
||||
# (requires typing_extensions >= 4.13)
|
||||
RequestType = Any
|
||||
GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServingBase | None]
|
||||
GetHandlerFn = Callable[[Request], BaseServing | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
|
||||
|
||||
@@ -16,13 +16,13 @@ from transformers import PreTrainedTokenizerBase
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
RequestResponseMetadata,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
@@ -87,7 +87,7 @@ def asr_inter_chunk_separator(
|
||||
return "" if language and language.lower() in no_space_languages else " "
|
||||
|
||||
|
||||
class OpenAISpeechToText(OpenAIServing):
|
||||
class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
"""Base class for speech-to-text operations like transcription and
|
||||
translation."""
|
||||
|
||||
@@ -475,7 +475,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
|
||||
|
||||
input_len = (
|
||||
OpenAISpeechToText._get_decoder_prompt_len(engine_inputs)
|
||||
SpeechToTextBaseServing._get_decoder_prompt_len(engine_inputs)
|
||||
if request.use_beam_search
|
||||
else 0
|
||||
)
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Literal, cast
|
||||
import numpy as np
|
||||
|
||||
from vllm.engine.protocol import EngineClient, StreamingInput
|
||||
from vllm.entrypoints.openai.engine.serving import OpenAIServing
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import PromptType
|
||||
@@ -20,7 +20,7 @@ from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIServingRealtime(OpenAIServing):
|
||||
class OpenAIServingRealtime(GenerateBaseServing):
|
||||
"""Realtime audio transcription service via WebSocket streaming.
|
||||
|
||||
Provides streaming audio-to-text transcription by transforming audio chunks
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
from ..base.serving import OpenAISpeechToText
|
||||
from ..base.serving import SpeechToTextBaseServing
|
||||
from .protocol import (
|
||||
TranscriptionRequest,
|
||||
TranscriptionResponse,
|
||||
@@ -26,7 +26,7 @@ from .protocol import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIServingTranscription(OpenAISpeechToText):
|
||||
class OpenAIServingTranscription(SpeechToTextBaseServing):
|
||||
"""Handles transcription requests."""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
from ..base.serving import OpenAISpeechToText
|
||||
from ..base.serving import SpeechToTextBaseServing
|
||||
from .protocol import (
|
||||
TranslationRequest,
|
||||
TranslationResponse,
|
||||
@@ -26,7 +26,7 @@ from .protocol import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OpenAIServingTranslation(OpenAISpeechToText):
|
||||
class OpenAIServingTranslation(SpeechToTextBaseServing):
|
||||
"""Handles translation requests."""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Any
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.generate.base.serving import resolve_token_id_placeholder
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
@@ -15,7 +16,6 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponseChoice,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ToolCall
|
||||
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -264,7 +264,7 @@ class OnlineRenderer:
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
trust_request_chat_template: bool,
|
||||
) -> ErrorResponse | None:
|
||||
"""Copied from OpenAIServing._validate_chat_template."""
|
||||
"""Copied from GenerateBaseServing._validate_chat_template."""
|
||||
if not trust_request_chat_template and (
|
||||
request_chat_template is not None
|
||||
or (
|
||||
@@ -287,7 +287,7 @@ class OnlineRenderer:
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_completion."""
|
||||
"""Copied from GenerateBaseServing._preprocess_completion."""
|
||||
prompts = list[SingletonPrompt | bytes]()
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
@@ -302,7 +302,7 @@ class OnlineRenderer:
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_cmpl."""
|
||||
"""Copied from GenerateBaseServing._preprocess_cmpl."""
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
@@ -339,7 +339,7 @@ class OnlineRenderer:
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
"""Copied from OpenAIServing._preprocess_chat."""
|
||||
"""Copied from GenerateBaseServing._preprocess_chat."""
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
|
||||
Reference in New Issue
Block a user