[Mini-VLLM] Remove LoRA, plugins, speech-to-text and scale-out surfaces

Production runs one static model behind the OpenAI-compatible API. Every
adapter path and non-OpenAI HTTP surface is unreachable there.

- Delete the LoRA stack: vllm/lora, LoRARequest on the engine wire, the
  scheduler, worker, cudagraph and MoE plumbing, LoRA metrics, the lora
  CLI group, and the LoRA tests.
- Drop the plugin loader in vllm/plugins. Platform resolution is now
  CUDA-only and the general/stat-logger plugin hooks are gone.
- Delete the speech-to-text and realtime entrypoints, the scale-out
  render routes, and vllm/config/speech_to_text.py. GenerationTask is
  generate-only.
- Cut DecodeBenchConnector and the disagg-prefill doc and image. The
  Multi, Offloading and Example connectors stay; production uses the
  CPU KV offload tier.
- Delete the ROCm-only fused KDA chunk kernel (2346 lines) with its CMake
  arch gate, declarations and op registrations. The CUDA build never
  defined the guard and no Python caller existed.
- Delete the orphan Mistral and Pixtral chat fixtures and examples/.
- Fix _check_model: removing the LoRA early return made every request
  fail with "The model ... does not exist. Valid aliases: ...".

Verified on an RTX 3090: import sweep reports 1242 modules with 7 known
optional-dependency failures; a live server answers chat, completions,
responses, tokenize and detokenize with zero log errors and 404 for the
removed surfaces; collection yields 8558 tests with the 3 pre-existing
errors; ruff adds no new finding; the incremental _C build succeeds.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 21:06:19 +07:00
co-authored by Hinata
parent 5f3c7bd8ab
commit 5aa2ac64f8
269 changed files with 417 additions and 29932 deletions
-6
View File
@@ -8,7 +8,6 @@ import time
import uvloop
import vllm
import vllm.envs as envs
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
from vllm.entrypoints.launchers.cli_args import (
@@ -284,11 +283,6 @@ def run_multi_api_server(args: argparse.Namespace):
usage_context = UsageContext.OPENAI_API_SERVER
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
if num_api_servers > 1 and envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
raise ValueError(
"VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used with api_server_count > 1"
)
executor_class = Executor.get_class(vllm_config)
log_stats = not engine_args.disable_log_stats
-88
View File
@@ -18,7 +18,6 @@ from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
from vllm.renderers.chat_utils import (
ChatCompletionMessageParam,
@@ -55,58 +54,6 @@ class OfflineInferenceMixin:
model_config: ModelConfig
chat_template: str | None
def _resolve_mm_lora(
self,
prompt: EngineInput,
lora_request: LoRARequest | None,
) -> LoRARequest | None:
if prompt["type"] != "multimodal":
return lora_request
lora_config = self.llm_engine.vllm_config.lora_config
default_mm_loras = None if lora_config is None else lora_config.default_mm_loras
if not default_mm_loras:
return lora_request
prompt_modalities = prompt["mm_placeholders"].keys()
intersection = set(prompt_modalities).intersection(default_mm_loras.keys())
if not intersection:
return lora_request
if len(intersection) > 1:
# TODO: Would be nice to be able to have multiple loras per prompt
logger.warning(
"Multiple modality specific loras were registered and would be "
"used by a single prompt consuming several modalities; "
"currently we only support one lora per request; as such, "
"lora(s) registered with modalities: %s will be skipped",
intersection,
)
return lora_request
# Build the LoRA request; the ID of the default mm lora is the
# index of the modality name sorted alphabetically + 1.
modality_name = intersection.pop()
modality_lora_path = default_mm_loras[modality_name]
modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
# If we have a collision, warn if there is a collision,
# but always send the explicitly provided request.
if lora_request:
if lora_request.lora_int_id != modality_lora_id:
logger.warning(
"A modality with a registered lora and a lora_request "
"with a different ID were provided; falling back to the "
"lora_request as we only apply one LoRARequest per prompt"
)
return lora_request
return LoRARequest(
modality_name,
modality_lora_id,
modality_lora_path,
)
def _preprocess_cmpl(
self,
prompts: Sequence[PromptType],
@@ -262,22 +209,6 @@ class OfflineInferenceMixin:
return [params] * num_requests
def _lora_request_to_seq(
self,
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
num_requests: int,
) -> Sequence[LoRARequest | None]:
if isinstance(lora_request, Sequence):
if len(lora_request) != num_requests:
raise VLLMValidationError(
f"The lengths of prompts ({num_requests}) "
f"and lora_request ({len(lora_request)}) must be the same."
)
return lora_request
return [lora_request] * num_requests
def _priority_to_seq(
self,
priority: int | Sequence[int] | None,
@@ -305,14 +236,12 @@ class OfflineInferenceMixin:
| Sequence[SamplingParams | PoolingParams],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[str]:
seq_prompts = prompt_to_seq(prompts)
seq_params = self._params_to_seq(params, len(seq_prompts))
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_prompts))
seq_priority = self._priority_to_seq(priority, len(seq_prompts))
return self._render_and_add_requests(
@@ -329,7 +258,6 @@ class OfflineInferenceMixin:
)
),
params=seq_params,
lora_requests=seq_lora_requests,
priorities=seq_priority,
)
@@ -342,7 +270,6 @@ class OfflineInferenceMixin:
output_type: type[_O],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -351,7 +278,6 @@ class OfflineInferenceMixin:
prompts=prompts,
params=params,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
tokenization_kwargs=tokenization_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
@@ -368,7 +294,6 @@ class OfflineInferenceMixin:
output_type: type[_O],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
@@ -383,7 +308,6 @@ class OfflineInferenceMixin:
messages=messages,
params=params,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
@@ -405,7 +329,6 @@ class OfflineInferenceMixin:
| Sequence[SamplingParams | PoolingParams],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
@@ -418,7 +341,6 @@ class OfflineInferenceMixin:
) -> list[str]:
seq_convs = conversation_to_seq(messages)
seq_params = self._params_to_seq(params, len(seq_convs))
seq_lora_requests = self._lora_request_to_seq(lora_request, len(seq_convs))
seq_priority = self._priority_to_seq(priority, len(seq_convs))
# When thinking is enabled or tools are provided, and the model
@@ -452,7 +374,6 @@ class OfflineInferenceMixin:
)
),
params=seq_params,
lora_requests=seq_lora_requests,
priorities=seq_priority,
)
@@ -509,7 +430,6 @@ class OfflineInferenceMixin:
params: Sequence[SamplingParams | PoolingParams],
output_type: type[_O],
*,
lora_requests: Sequence[LoRARequest | None] | None = None,
priorities: Sequence[int] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
):
@@ -526,7 +446,6 @@ class OfflineInferenceMixin:
self._render_and_add_requests(
prompts=prompts,
params=params,
lora_requests=lora_requests,
priorities=priorities,
)
@@ -537,7 +456,6 @@ class OfflineInferenceMixin:
prompts: Iterable[EngineInput],
params: Sequence[SamplingParams | PoolingParams],
*,
lora_requests: Sequence[LoRARequest | None] | None = None,
priorities: Sequence[int] | None = None,
) -> list[str]:
added_request_ids: list[str] = []
@@ -547,10 +465,6 @@ class OfflineInferenceMixin:
request_id = self._add_request(
prompt,
params[i],
lora_request=self._resolve_mm_lora(
prompt,
None if lora_requests is None else lora_requests[i],
),
priority=0 if priorities is None else priorities[i],
)
added_request_ids.append(request_id)
@@ -565,7 +479,6 @@ class OfflineInferenceMixin:
self,
prompt: EngineInput,
params: SamplingParams | PoolingParams,
lora_request: LoRARequest | None = None,
priority: int = 0,
) -> str:
if isinstance(params, SamplingParams):
@@ -578,7 +491,6 @@ class OfflineInferenceMixin:
request_id,
prompt,
params,
lora_request=lora_request,
priority=priority,
)
@@ -29,7 +29,6 @@ from vllm.exceptions import GenerationError
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob, PromptLogprobs
from vllm.lora.request import LoRARequest
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens
from vllm.v1.metrics.stats import RequestStateStats
@@ -121,7 +120,6 @@ class ServeContext(Generic[RequestT]):
model_name: str
request_id: str
created_time: int = field(default_factory=lambda: int(time.time()))
lora_request: LoRARequest | None = None
engine_inputs: list[EngineInput] | None = None
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -11,7 +11,6 @@ from tqdm import tqdm
from vllm import RequestOutput, TextPrompt, TokensPrompt
from vllm.entrypoints.common.offline import OfflineInferenceMixin
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import (
BeamSearchParams,
@@ -60,7 +59,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
self,
prompts: list[TokensPrompt | TextPrompt],
params: BeamSearchParams,
lora_request: list[LoRARequest] | LoRARequest | None = None,
use_tqdm: bool = False,
concurrency_limit: int | None = None,
) -> list[BeamSearchOutput]:
@@ -70,7 +68,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
prompts: A list of prompts. Each prompt can be a string or a list
of token IDs.
params: The beam search parameters.
lora_request: LoRA request to use for generation, if any.
use_tqdm: Whether to use tqdm to display the progress bar.
concurrency_limit: The maximum number of concurrent requests.
If None, the number of concurrent requests is unlimited.
@@ -90,7 +87,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
engine_inputs = self._preprocess_cmpl(prompts)
lora_requests = self._lora_request_to_seq(lora_request, len(engine_inputs))
if use_tqdm and concurrency_limit is not None:
logger.warning(
@@ -131,7 +127,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
instances: list[BeamSearchInstance] = []
for lora_req, prompt in zip(lora_requests, engine_inputs):
for prompt in engine_inputs:
if prompt["type"] == "embeds":
raise NotImplementedError(
"Embedding prompt not supported for beam search"
@@ -140,7 +136,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
instances.append(
BeamSearchInstance(
prompt,
lora_request=lora_req,
logprobs=None,
),
)
@@ -279,7 +274,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
prompts=(beam.get_prompt() for beam in active_beams),
params=active_params,
output_type=RequestOutput,
lora_requests=[beam.lora_request for beam in active_beams],
use_tqdm=False,
)
@@ -346,7 +340,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
current_beam.orig_prompt,
tokens=current_beam.tokens + [token_id],
logprobs=current_beam.logprobs + [logprobs],
lora_request=current_beam.lora_request,
cum_logprob=current_beam.cum_logprob + logprob_obj.logprob,
)
@@ -11,7 +11,6 @@ import numpy as np
from vllm import CompletionOutput, RequestOutput
from vllm.engine.protocol import EngineClient
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.utils import random_uuid
@@ -34,7 +33,6 @@ class BeamSearchOnlineMixin(ABC):
prompt: EngineInput,
request_id: str,
params: BeamSearchParams,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
session_id: str | None = None,
) -> AsyncGenerator[RequestOutput, None]:
@@ -80,7 +78,6 @@ class BeamSearchOnlineMixin(ABC):
tokens=prompt_token_ids,
cum_logprob=0,
logprobs=[],
lora_request=lora_request,
)
]
completed = []
@@ -91,7 +88,6 @@ class BeamSearchOnlineMixin(ABC):
for i, beam in enumerate(all_beams):
prompt_item = beam.get_prompt()
lora_request_item = beam.lora_request
request_id_item = f"{request_id_batch}-beam-{i}"
task = asyncio.create_task(
collect_from_async_generator(
@@ -99,7 +95,6 @@ class BeamSearchOnlineMixin(ABC):
prompt_item,
sampling_params,
request_id_item,
lora_request=lora_request_item,
trace_headers=trace_headers,
session_id=session_id,
)
@@ -205,7 +200,6 @@ class BeamSearchOnlineMixin(ABC):
orig_prompt=prompt,
tokens=current_beam.tokens + [token_id],
logprobs=current_beam.logprobs + [logprobs],
lora_request=current_beam.lora_request,
cum_logprob=cum_logprob,
)
)
@@ -12,7 +12,6 @@ from vllm.inputs import (
tokens_input,
)
from vllm.logprobs import Logprob
from vllm.lora.request import LoRARequest
@dataclass
@@ -28,7 +27,6 @@ class BeamSearchSequence:
# NOTE: Tokens represents decoder tokens in the encoder / decoder case
tokens: list[int]
logprobs: list[dict[int, Logprob]]
lora_request: LoRARequest | None = None
cum_logprob: float = 0.0
text: str | None = None
finish_reason: str | None = None
@@ -113,7 +111,6 @@ class BeamSearchInstance:
def __init__(
self,
prompt: TokensInput | MultiModalInput | EncoderDecoderInput,
lora_request: LoRARequest | None = None,
logprobs: list[dict[int, Logprob]] | None = None,
**kwargs,
):
@@ -127,7 +124,6 @@ class BeamSearchInstance:
orig_prompt=prompt,
tokens=initial_tokens,
logprobs=[] if logprobs is None else list(logprobs),
lora_request=lora_request,
**kwargs,
)
]
@@ -245,12 +245,6 @@ class ServingGenerativeScoring(BaseServing):
# Note: Mixed item types (string and token list) are validated by
# Pydantic at request parsing time, so we don't need to check here.
try:
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
except (ValueError, TypeError, RuntimeError) as e:
logger.exception("Error preparing request components")
return self.create_error_response(e)
base_id = self._base_request_id(raw_request, default=request.request_id)
request_id = f"generative-scoring-{base_id}"
created_time = int(time.time())
@@ -290,7 +284,6 @@ class ServingGenerativeScoring(BaseServing):
f"{request_id}-{i}",
engine_input,
params=sampling_params,
lora_request=lora_request,
)
try:
@@ -299,7 +292,6 @@ class ServingGenerativeScoring(BaseServing):
engine_inputs,
[sampling_params] * len(engine_inputs),
request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
)
@@ -343,7 +335,7 @@ class ServingGenerativeScoring(BaseServing):
total_completion_tokens += len(read.result.outputs[0].token_ids)
# Build response
model_name = self.models.model_name(lora_request)
model_name = self.models.model_name()
response = GenerativeScoringResponse(
id=request_id,
created=created_time,
-3
View File
@@ -8,7 +8,6 @@ from dataclasses import dataclass
from vllm.engine.protocol import EngineClient
from vllm.exceptions import GenerationError
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.utils.async_utils import merge_async_iterators
@@ -28,7 +27,6 @@ async def next_token_label_reads(
sampling_params: Sequence[SamplingParams],
request_id: str,
*,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
) -> list[LabelRead]:
@@ -41,7 +39,6 @@ async def next_token_label_reads(
engine_input,
params,
f"{request_id}-{i}",
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
)
@@ -11,9 +11,7 @@ from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
@@ -68,20 +66,10 @@ async def init_app_state(
await init_tool_server(args) if "generate" in supported_tasks else None
)
# Merge default_mm_loras into the static lora_modules
default_mm_loras = (
vllm_config.lora_config.default_mm_loras
if vllm_config.lora_config is not None
else {}
)
lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
state.openai_serving_models = OpenAIServingModels(
engine_client=engine_client,
base_model_paths=base_model_paths,
lora_modules=lora_modules,
)
await state.openai_serving_models.init_static_loras()
state.online_renderer = OnlineRenderer(
model_config=engine_client.model_config,
@@ -123,18 +111,6 @@ async def init_app_state(
default_chat_template_kwargs,
)
from vllm.entrypoints.scale_out.factories import init_scale_out_state
init_scale_out_state(state, args, engine_client, request_logger)
if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
init_speech_to_text_state(
engine_client, state, args, request_logger, supported_tasks
)
await init_endpoint_plugins_state(engine_client, state, args)
state.enable_server_load_tracking = args.enable_server_load_tracking
state.server_load_metrics = 0
@@ -31,17 +31,6 @@ def register_api_routers(
register_generate_api_routers(app)
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
register_scale_out_api_routers(app, supported_tasks)
if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import (
register_speech_to_text_api_routers,
)
register_speech_to_text_api_routers(app, supported_tasks)
if getattr(args, "enable_fault_tolerance", False):
from vllm.entrypoints.serve.fault_tolerance.api_router import (
register_fault_tolerance_api_router,
-4
View File
@@ -8,7 +8,6 @@ from fastapi import FastAPI
from vllm.config import ModelConfig
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
from .api_server.routers import register_api_routers
@@ -41,9 +40,6 @@ def build_app(
register_api_routers(args, app, supported_tasks, model_config)
# Endpoint plugin hooks run after core routers so plugins can inspect them.
attach_endpoint_plugins(app, supported_tasks)
init_exception_handler(app)
init_entrypoints_middleware(args, app, supported_tasks)
return app
+1 -59
View File
@@ -7,13 +7,11 @@ It is kept in a separate file for documentation purposes.
import argparse
import json
import ssl
from collections.abc import Sequence
from dataclasses import field
from typing import Any, Literal
from vllm.config import config
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.logger import init_logger
from vllm.renderers.chat_utils import (
ChatTemplateContentFormatOption,
@@ -31,40 +29,6 @@ from .utils.constants import (
logger = init_logger(__name__)
class LoRAParserAction(argparse.Action):
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: str | Sequence[str] | None,
option_string: str | None = None,
):
if values is None:
values = []
if isinstance(values, str):
raise TypeError("Expected values to be a list")
lora_list: list[LoRAModulePath] = []
for item in values:
if item in [None, ""]: # Skip if item is None or empty string
continue
if "=" in item and "," not in item: # Old format: name=path
name, path = item.split("=", 1)
lora_list.append(LoRAModulePath(name, path))
else: # Assume JSON format
try:
lora_dict = json.loads(item)
lora = LoRAModulePath(**lora_dict)
lora_list.append(lora)
except json.JSONDecodeError:
parser.error(f"Invalid JSON format for --lora-modules: {item}")
except TypeError as e:
parser.error(
f"Invalid fields for --lora-modules: {item} - {str(e)}"
)
setattr(namespace, self.dest, lora_list)
@config
class BaseFrontendArgs:
"""Base arguments for the OpenAI-compatible frontend server.
@@ -74,11 +38,6 @@ class BaseFrontendArgs:
the subclasses.
"""
lora_modules: list[LoRAModulePath] | None = None
"""LoRA modules configurations in either 'name=path' format or JSON format
or JSON list format. Example (old format): `'name=path'` Example (new
format): `{\"name\": \"name\", \"path\": \"lora_path\",
\"base_model_name\": \"id\"}`"""
chat_template: str | None = None
"""The file path to the chat template, or the template in single-line form
for the specified model."""
@@ -170,18 +129,6 @@ class BaseFrontendArgs:
"""
log_error_stack: bool = False
"""If set to True, log the stack trace of error responses"""
tokens_only: bool = False
"""
If set to True, only enable the Tokens In<>Out endpoint.
This is intended for use in a Disaggregated Everything setup.
"""
enable_scale_out: bool = False
"""
If set to True, register the scale-out endpoints (`/inference/v1/generate`
and `/inference/v1/abort_requests`) on `vllm serve`. `vllm serve
--tokens-only` always registers the endpoints it needs regardless of this
flag.
"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
@@ -208,11 +155,6 @@ class BaseFrontendArgs:
# Special case: default_chat_template_kwargs needs json.loads type
frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
# Special case: LoRA modules need custom parser action and
# optional_type(str)
frontend_kwargs["lora_modules"]["type"] = optional_type(str)
frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
# Special case: Tool call parser shows built-in options.
valid_tool_parsers = list(ToolParserManager.list_registered())
parsers_str = ",".join(valid_tool_parsers)
-13
View File
@@ -33,7 +33,6 @@ from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
from vllm.inputs import PromptType
from vllm.logger import configure_logging_if_needed, init_logger
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.outputs import PoolingRequestOutput, RequestOutput
from vllm.platforms import current_platform
@@ -415,7 +414,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -439,7 +437,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
When it is a single value, it is applied to every prompt.
@@ -469,7 +466,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
params=sampling_params,
output_type=RequestOutput,
use_tqdm=use_tqdm,
lora_request=lora_request,
tokenization_kwargs=tokenization_kwargs,
priority=priority,
mm_processor_kwargs=mm_processor_kwargs,
@@ -479,7 +475,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
self,
prompts: PromptType | Sequence[PromptType],
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
tokenization_kwargs: dict[str, Any] | None = None,
@@ -494,7 +489,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
Args:
prompts: The prompts to the LLM. See generate() for details.
sampling_params: The sampling parameters for text generation.
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
When it is a single value, it is applied to every prompt.
@@ -519,7 +513,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
prompts=prompts,
params=sampling_params,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
tokenization_kwargs=tokenization_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
@@ -614,7 +607,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
| Sequence[list[ChatCompletionMessageParam]],
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
@@ -649,7 +641,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
When it is a single value, it is applied to every prompt.
@@ -698,7 +689,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
params=sampling_params,
output_type=RequestOutput,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
@@ -716,7 +706,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
| Sequence[list[ChatCompletionMessageParam]],
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None = None,
priority: int | Sequence[int] | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
@@ -742,7 +731,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
If None, we use the default sampling parameters.
use_tqdm: If `True`, shows a tqdm progress bar while rendering
conversations.
lora_request: LoRA request to use for generation, if any.
priority: The priority of the requests, if any.
Only applicable when priority scheduling policy is enabled.
When it is a single value, it is applied to every prompt.
@@ -780,7 +768,6 @@ class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
messages=messages,
params=sampling_params,
use_tqdm=use_tqdm,
lora_request=lora_request,
priority=priority,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
@@ -136,8 +136,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
if raw_request:
raw_request.state.request_metadata = request_metadata
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
model_name = self.models.model_name(lora_request)
model_name = self.models.model_name()
data_parallel_rank = self._get_data_parallel_rank(raw_request)
max_model_len = self.model_config.max_model_len
@@ -161,7 +160,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
sub_request_id,
engine_prompt,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
None
@@ -180,7 +178,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
engine_prompt,
sampling_params,
sub_request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
@@ -265,7 +265,7 @@ class OpenAIServingChat(GenerateBaseServing):
"""Validate the model and preprocess a chat completion request.
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
engine-aware checks (engine health).
Returns:
A tuple of (conversation, engine_inputs) on success,
@@ -322,9 +322,7 @@ class OpenAIServingChat(GenerateBaseServing):
if raw_request:
raw_request.state.request_metadata = request_metadata
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
model_name = self.models.model_name(lora_request)
model_name = self.models.model_name()
# Extract data_parallel_rank from header (router can inject it)
data_parallel_rank = self._get_data_parallel_rank(raw_request)
@@ -369,7 +367,6 @@ class OpenAIServingChat(GenerateBaseServing):
sub_request_id,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
@@ -384,7 +381,6 @@ class OpenAIServingChat(GenerateBaseServing):
prompt=engine_input,
request_id=sub_request_id,
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
session_id=session_id,
)
@@ -393,7 +389,6 @@ class OpenAIServingChat(GenerateBaseServing):
engine_input,
sampling_params,
sub_request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=self._get_priority(request, raw_request),
data_parallel_rank=data_parallel_rank,
@@ -94,7 +94,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
"""Validate the model and preprocess a completion request.
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
engine-aware checks (engine health).
Returns:
A list of engine_inputs on success, or an ErrorResponse on failure.
@@ -147,8 +147,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
if raw_request:
raw_request.state.request_metadata = request_metadata
lora_request = self._maybe_get_adapters(request)
# Extract data_parallel_rank from header (router can inject it)
data_parallel_rank = self._get_data_parallel_rank(raw_request)
@@ -182,7 +180,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
request_id_item,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
@@ -197,7 +194,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
prompt=engine_input,
request_id=request_id,
params=sampling_params,
lora_request=lora_request,
trace_headers=trace_headers,
session_id=session_id,
)
@@ -206,7 +202,6 @@ class OpenAIServingCompletion(GenerateBaseServing):
engine_input,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
priority=self._get_priority(request, raw_request),
data_parallel_rank=data_parallel_rank,
@@ -217,7 +212,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
result_generator = merge_async_iterators(*generators)
model_name = self.models.model_name(lora_request)
model_name = self.models.model_name()
num_prompts = len(engine_inputs)
# Streaming response
@@ -9,11 +9,3 @@ from dataclasses import dataclass
class BaseModelPath:
name: str
model_path: str
@dataclass
class LoRAModulePath:
name: str
path: str
base_model_name: str | None = None
is_3d_lora_weight: bool = False
+7 -271
View File
@@ -1,13 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from asyncio import Lock
from collections import defaultdict
from http import HTTPStatus
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import (
ErrorResponse,
@@ -15,24 +13,12 @@ from vllm.entrypoints.serve.engine.protocol import (
ModelList,
ModelPermission,
)
from vllm.entrypoints.serve.lora.protocol import (
LoadLoRAAdapterRequest,
UnloadLoRAAdapterRequest,
)
from vllm.exceptions import LoRAAdapterNotFoundError
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
from vllm.utils.counter import AtomicCounter
logger = init_logger(__name__)
class OpenAIModelRegistry:
"""Read-only view of the loaded base models with no engine dependency.
Suitable for CPU-only / render-only contexts that have no engine client
and no LoRA support.
Suitable for CPU-only / render-only contexts that have no engine client.
"""
def __init__(
@@ -42,9 +28,8 @@ class OpenAIModelRegistry:
) -> None:
self.model_config = model_config
self.base_model_paths = base_model_paths
self.lora_requests: dict[str, LoRARequest] = {}
def model_name(self, lora_request: LoRARequest | None = None) -> str:
def model_name(self) -> str:
return self.base_model_paths[0].name
def is_base_model(self, model_name: str) -> bool:
@@ -76,25 +61,18 @@ class OpenAIModelRegistry:
]
)
async def resolve_lora(self, lora_name: str):
raise RuntimeError("The OpenAIModelRegistry has no LoRA support.")
class OpenAIServingModels:
"""Shared instance to hold data about the loaded base model(s) and adapters.
"""Shared instance to hold data about the loaded base model(s).
Handles the routes:
- /v1/models
- /v1/load_lora_adapter
- /v1/unload_lora_adapter
"""
def __init__(
self,
engine_client: EngineClient,
base_model_paths: list[BaseModelPath],
*,
lora_modules: list[LoRAModulePath] | None = None,
):
super().__init__()
@@ -106,258 +84,16 @@ class OpenAIServingModels:
self.engine_client = engine_client
self.base_model_paths = base_model_paths
self.static_lora_modules = lora_modules
self.lora_requests: dict[str, LoRARequest] = {}
self.lora_id_counter = AtomicCounter(0)
self.lora_resolvers: list[LoRAResolver] = []
for lora_resolver_name in LoRAResolverRegistry.get_supported_resolvers():
self.lora_resolvers.append(
LoRAResolverRegistry.get_resolver(lora_resolver_name)
)
self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock)
self.model_config = self.engine_client.model_config
self.renderer = self.engine_client.renderer
self.input_processor = self.engine_client.input_processor
async def init_static_loras(self):
"""Loads all static LoRA modules.
Raises if any fail to load"""
if self.static_lora_modules is None:
return
for lora in self.static_lora_modules:
load_request = LoadLoRAAdapterRequest(
lora_path=lora.path,
lora_name=lora.name,
is_3d_lora_weight=lora.is_3d_lora_weight,
)
load_result = await self.load_lora_adapter(
request=load_request, base_model_name=lora.base_model_name
)
if isinstance(load_result, ErrorResponse):
raise ValueError(load_result.error.message)
def is_base_model(self, model_name: str) -> bool:
return self.registry.is_base_model(model_name)
def model_name(self, lora_request: LoRARequest | None = None) -> str:
if lora_request is not None:
return lora_request.lora_name
def model_name(self) -> str:
return self.base_model_paths[0].name
async def show_available_models(self) -> ModelList:
"""Show available models. This includes the base model and all
adapters."""
model_list = await self.registry.show_available_models()
lora_cards = [
ModelCard(
id=lora.lora_name,
root=lora.path,
parent=lora.base_model_name
if lora.base_model_name
else self.base_model_paths[0].name,
permission=[ModelPermission()],
)
for lora in self.lora_requests.values()
]
model_list.data.extend(lora_cards)
return model_list
async def load_lora_adapter(
self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
) -> ErrorResponse | str:
lora_name = request.lora_name
# Ensure atomicity based on the lora name
async with self.lora_resolver_lock[lora_name]:
error_check_ret = await self._check_load_lora_adapter_request(request)
if error_check_ret is not None:
return error_check_ret
lora_path = request.lora_path
lora_int_id = (
self.lora_requests[lora_name].lora_int_id
if lora_name in self.lora_requests
else self.lora_id_counter.inc(1)
)
lora_request = LoRARequest(
lora_name=lora_name,
lora_int_id=lora_int_id,
lora_path=lora_path,
load_inplace=request.load_inplace,
is_3d_lora_weight=request.is_3d_lora_weight,
)
if base_model_name is not None and self.is_base_model(base_model_name):
lora_request.base_model_name = base_model_name
# Validate that the adapter can be loaded into the engine
# This will also preload it for incoming requests
try:
await self.engine_client.add_lora(lora_request)
except Exception as e:
if str(
LoRAAdapterNotFoundError(
lora_request.lora_name, lora_request.lora_path
)
) in str(e):
return create_error_response(
LoRAAdapterNotFoundError(
lora_request.lora_name, lora_request.lora_path
)
)
return create_error_response(
message=str(e),
err_type="InternalServerError",
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
)
previous = self.lora_requests.get(lora_name)
if previous is not None and previous.lora_path == lora_path:
logger.warning(
"Reloaded LoRA adapter '%s' in place from the same path '%s'. "
"Prefix-cache blocks computed with its previous weights are "
"not invalidated; load changed weights from a new path to "
"avoid reusing them.",
lora_name,
lora_path,
)
self.lora_requests[lora_name] = lora_request
logger.info(
"Loaded new LoRA adapter: name '%s', path '%s'", lora_name, lora_path
)
return f"Success: LoRA adapter '{lora_name}' added successfully."
async def unload_lora_adapter(
self, request: UnloadLoRAAdapterRequest
) -> ErrorResponse | str:
lora_name = request.lora_name
# Ensure atomicity based on the lora name
async with self.lora_resolver_lock[lora_name]:
error_check_ret = await self._check_unload_lora_adapter_request(request)
if error_check_ret is not None:
return error_check_ret
# Safe to delete now since we hold the lock
del self.lora_requests[lora_name]
logger.info("Removed LoRA adapter: name '%s'", lora_name)
return f"Success: LoRA adapter '{lora_name}' removed successfully."
async def _check_load_lora_adapter_request(
self, request: LoadLoRAAdapterRequest
) -> ErrorResponse | None:
# Check if both 'lora_name' and 'lora_path' are provided
if not request.lora_name or not request.lora_path:
return create_error_response(
message="Both 'lora_name' and 'lora_path' must be provided.",
err_type="InvalidUserInput",
status_code=HTTPStatus.BAD_REQUEST,
)
if self.is_base_model(request.lora_name):
return create_error_response(
message=f"The lora adapter '{request.lora_name}' conflicts with a "
"served base model.",
err_type="InvalidUserInput",
status_code=HTTPStatus.BAD_REQUEST,
)
# If not loading inplace
# Check if the lora adapter with the given name already exists
if not request.load_inplace and request.lora_name in self.lora_requests:
return create_error_response(
message=f"The lora adapter '{request.lora_name}' has already been "
"loaded. If you want to load the adapter in place, set 'load_inplace'"
" to True.",
err_type="InvalidUserInput",
status_code=HTTPStatus.BAD_REQUEST,
)
return None
async def _check_unload_lora_adapter_request(
self, request: UnloadLoRAAdapterRequest
) -> ErrorResponse | None:
# Check if 'lora_name' is not provided return an error
if not request.lora_name:
return create_error_response(
message="'lora_name' needs to be provided to unload a LoRA adapter.",
err_type="InvalidUserInput",
status_code=HTTPStatus.BAD_REQUEST,
)
# Check if the lora adapter with the given name exists
if request.lora_name not in self.lora_requests:
return create_error_response(
message=f"The lora adapter '{request.lora_name}' cannot be found.",
err_type="NotFoundError",
status_code=HTTPStatus.NOT_FOUND,
)
return None
async def resolve_lora(self, lora_name: str) -> LoRARequest | ErrorResponse:
"""Attempt to resolve a LoRA adapter using available resolvers.
Args:
lora_name: Name/identifier of the LoRA adapter
Returns:
LoRARequest if found and loaded successfully.
ErrorResponse (404) if no resolver finds the adapter.
ErrorResponse (400) if adapter(s) are found but none load.
"""
async with self.lora_resolver_lock[lora_name]:
# First check if this LoRA is already loaded
if lora_name in self.lora_requests:
return self.lora_requests[lora_name]
base_model_name = self.model_config.model
unique_id = self.lora_id_counter.inc(1)
found_adapter = False
# Try to resolve using available resolvers
for resolver in self.lora_resolvers:
lora_request = await resolver.resolve_lora(base_model_name, lora_name)
if lora_request is not None:
found_adapter = True
lora_request.lora_int_id = unique_id
try:
await self.engine_client.add_lora(lora_request)
self.lora_requests[lora_name] = lora_request
logger.info(
"Resolved and loaded LoRA adapter '%s' using %s",
lora_name,
resolver.__class__.__name__,
)
return lora_request
except BaseException as e:
logger.warning(
"Failed to load LoRA '%s' resolved by %s: %s. "
"Trying next resolver.",
lora_name,
resolver.__class__.__name__,
e,
)
continue
if found_adapter:
# An adapter was found, but all attempts to load it failed.
return create_error_response(
message=(
f"LoRA adapter '{lora_name}' was found but could not be loaded."
),
err_type="BadRequestError",
status_code=HTTPStatus.BAD_REQUEST,
)
else:
# No adapter was found
return create_error_response(
message=f"LoRA adapter {lora_name} does not exist",
err_type="NotFoundError",
status_code=HTTPStatus.NOT_FOUND,
)
"""Show available models."""
return await self.registry.show_available_models()
+1 -7
View File
@@ -81,7 +81,6 @@ from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput
from vllm.parser import Parser, ParserManager
from vllm.renderers import TokenizeParams
@@ -370,8 +369,7 @@ class OpenAIServingResponses(GenerateBaseServing):
else:
prev_response = None
lora_request = self._maybe_get_adapters(request)
model_name = self.models.model_name(lora_request)
model_name = self.models.model_name()
render_result = await self._render_resolved_response_inputs(
request,
@@ -521,7 +519,6 @@ class OpenAIServingResponses(GenerateBaseServing):
if isinstance(context, HarmonyContext)
else None
),
lora_request=lora_request,
priority=self._get_priority(request, raw_request),
trace_headers=trace_headers,
session_id=session_id,
@@ -643,7 +640,6 @@ class OpenAIServingResponses(GenerateBaseServing):
sampling_params: SamplingParams,
context: ConversationContext,
tok_params: TokenizeParams | None = None,
lora_request: LoRARequest | None = None,
priority: int = 0,
trace_headers: Mapping[str, str] | None = None,
session_id: str | None = None,
@@ -663,14 +659,12 @@ class OpenAIServingResponses(GenerateBaseServing):
sub_request_id,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
generator = self.engine_client.generate(
engine_input,
sampling_params,
sub_request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
session_id=session_id,
-87
View File
@@ -1,87 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from http import HTTPStatus
from typing import TYPE_CHECKING
from fastapi import FastAPI
from fastapi.responses import JSONResponse
from fastapi.routing import APIRoute
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.logger import init_logger
from vllm.tasks import SupportedTask
if TYPE_CHECKING:
from starlette.datastructures import State
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
else:
RequestLogger = object
logger = init_logger(__name__)
def init_scale_out_state(
state: "State",
args: "Namespace",
engine_client: "EngineClient",
request_logger: RequestLogger | None,
):
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
state.serving_tokens = ServingTokens(
engine_client,
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_log_outputs=args.enable_log_outputs,
force_no_detokenize=args.tokens_only,
)
def register_scale_out_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
args = getattr(app.state, "args", None)
# `vllm serve --tokens-only` is dedicated to serving these endpoints, so it
# always registers them regardless of `--enable-scale-out`.
dedicated_mode = getattr(args, "tokens_only", False)
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
if not enabled:
message = (
"Scale-out endpoints are disabled. Restart vllm serve with "
"--enable-scale-out to enable them."
)
logger.info(message)
async def disabled_scale_out() -> JSONResponse:
error = create_error_response(
message, "PermissionDeniedError", HTTPStatus.FORBIDDEN
)
return JSONResponse(error.model_dump(), status_code=HTTPStatus.FORBIDDEN)
from .token_in_token_out.api_router import router as token_router
for route in token_router.routes:
if isinstance(route, APIRoute):
app.add_api_route(
route.path,
disabled_scale_out,
methods=sorted(route.methods),
include_in_schema=False,
)
return
if "generate" in supported_tasks:
from .token_in_token_out.api_router import (
attach_router as attach_disagg_router,
)
attach_disagg_router(app)
@@ -1,117 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
from ...serve.engine.protocol import ErrorResponse
from .protocol import (
GenerateRequest,
GenerateResponseBase,
)
from .serving import ServingTokens
logger = init_logger(__name__)
def tokenization(request: Request) -> ServingTokenization:
return request.app.state.serving_tokenization
def generate_tokens(request: Request) -> ServingTokens | None:
return request.app.state.serving_tokens
def engine_client(request: Request) -> EngineClient:
return request.app.state.engine_client
router = APIRouter()
@router.post(
"/inference/v1/generate",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def generate(request: GenerateRequest, raw_request: Request):
handler = generate_tokens(raw_request)
if handler is None:
raise NotImplementedError("The model does not support generate tokens API")
generator = await handler.serve_tokens(request, raw_request)
if isinstance(generator, ErrorResponse):
return JSONResponse(
content=generator.model_dump(), status_code=generator.error.code
)
elif isinstance(generator, GenerateResponseBase):
return JSONResponse(content=generator.model_dump())
return StreamingResponse(content=generator, media_type="text/event-stream")
abort_router = APIRouter()
async def abort_requests(raw_request: Request):
"""Abort one or more requests. To be used in a
Disaggregated Everything setup.
"""
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"JSON decode error: {e}",
) from e
request_ids = body.get("request_ids")
if request_ids is None:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="Missing 'request_ids' in request body",
)
# Abort requests in background
asyncio.create_task(engine_client(raw_request).abort(request_ids))
return Response(status_code=200)
# Under the `/inference` prefix, so `--api-key` guards it.
router.add_api_route("/inference/v1/abort_requests", abort_requests, methods=["POST"])
# Unauthenticated legacy path, kept for existing `--tokens-only` deployments.
abort_router.add_api_route("/abort_requests", abort_requests, methods=["POST"])
def attach_router(app: FastAPI):
app.include_router(router)
if not getattr(app.state.args, "tokens_only", False):
return
# An endpoint plugin may already serve /abort_requests. Registering this
# one too would only add a shadowed route with a duplicate operation ID.
has_abort_route = any(
getattr(route, "path", None) == "/abort_requests" for route in app.routes
)
if not has_abort_route:
app.include_router(abort_router)
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers for render-time multimodal feature extraction and generate input."""
from __future__ import annotations
from collections.abc import Callable, Collection, Sequence
from typing import cast
from vllm.entrypoints.generate.base.protocol import MMPayload
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item,
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.inputs import (
EngineInput,
MultiModalHashes,
MultiModalInput,
MultiModalPlaceholders,
)
from vllm.multimodal.inputs import (
MultiModalKwargsItem,
MultiModalKwargsOptionalItems,
)
def merge_mm_kwargs_items(
kwargs_item: MultiModalKwargsItem | None,
metadata_item: MultiModalKwargsItem | None,
) -> MultiModalKwargsItem | None:
"""Merge full multimodal data with its metadata-only counterpart."""
if kwargs_item is None:
return metadata_item
if metadata_item is None:
return kwargs_item
return MultiModalKwargsItem({**kwargs_item, **metadata_item})
def _encode_metadata_items(
items: Sequence[MultiModalKwargsItem | None],
*,
declared: Collection[str],
) -> list[str | None]:
"""Serialize placeholder-metadata and ``keep_on_cpu`` fields per item."""
metadata_items: list[str | None] = []
for item in items:
if item is None:
metadata_items.append(None)
continue
metadata_item = MultiModalKwargsItem(
{
key: elem
for key, elem in item.items()
if elem.field.keep_on_cpu or key in declared
}
)
metadata_items.append(
encode_mm_kwargs_item(metadata_item) if metadata_item else None
)
return metadata_items
def _encode_mm_kwargs_with_metadata(
raw_mm_kwargs: MultiModalKwargsOptionalItems,
*,
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
) -> tuple[dict[str, list[str | None]], dict[str, list[str | None]] | None]:
"""Serialize full kwargs and their metadata-only subsets per modality."""
kwargs_data: dict[str, list[str | None]] = {}
metadata_by_modality: dict[str, list[str | None]] = {}
for modality, items in raw_mm_kwargs.items():
kwargs_data[modality] = [
encode_mm_kwargs_item(item) if item is not None else None for item in items
]
declared = (
set(metadata_fields_for(modality))
if metadata_fields_for is not None
else set()
)
metadata_items = _encode_metadata_items(items, declared=declared)
if any(item is not None for item in metadata_items):
metadata_by_modality[modality] = metadata_items
mm_metadata = metadata_by_modality or None
return kwargs_data, mm_metadata
def mm_kwargs_from_features(
features: MultiModalFeatures,
) -> dict[str, list[MultiModalKwargsItem | None]]:
"""Deserialize ``features`` into per-modality kwargs for ``mm_input``."""
mm_kwargs: dict[str, list[MultiModalKwargsItem | None]] = {}
kwargs_data = features.kwargs_data or {}
mm_metadata = features.mm_metadata or {}
for modality, hashes in features.mm_hashes.items():
n = len(hashes)
kwargs_items = [
decode_mm_kwargs_item(item) if item is not None else None
for item in kwargs_data.get(modality, [None] * n)
]
metadata_items = [
decode_mm_kwargs_item(item) if item is not None else None
for item in mm_metadata.get(modality, [None] * n)
]
mm_kwargs[modality] = [
merge_mm_kwargs_items(kwargs_item, metadata_item)
for kwargs_item, metadata_item in zip(
kwargs_items, metadata_items, strict=True
)
]
return mm_kwargs
def placeholder_ranges_from_engine_input(
engine_input: EngineInput,
) -> dict[str, list[PlaceholderRangeInfo]] | None:
"""Return per-modality placeholder ranges, or ``None`` for text-only prompts."""
if engine_input.get("type") != "multimodal":
return None
raw_placeholders: MultiModalPlaceholders = cast(MultiModalInput, engine_input)[
"mm_placeholders"
]
return {
modality: [
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
]
for modality, ranges in raw_placeholders.items()
}
def extract_mm_features(
engine_input: EngineInput,
*,
metadata_fields_for: Callable[[str], Collection[str]] | None = None,
mm_payload: MMPayload = "tensors",
) -> MultiModalFeatures | None:
"""Extract multimodal features from a rendered engine prompt.
Returns ``None`` for text-only prompts. ``mm_metadata`` keeps the
intersection of processed kwargs that prefill needs after EC transfer:
fields declared as embedding metadata, plus fields marked
``keep_on_cpu`` (for example M-RoPE grid dims). With
``mm_payload="none"`` neither is serialized.
"""
if engine_input.get("type") != "multimodal":
return None
mm_engine_input = cast(MultiModalInput, engine_input)
mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
mm_placeholders = placeholder_ranges_from_engine_input(engine_input)
assert mm_placeholders is not None
kwargs_data: dict[str, list[str | None]] | None = None
mm_metadata: dict[str, list[str | None]] | None = None
if mm_payload == "tensors":
raw_mm_kwargs = mm_engine_input.get("mm_kwargs")
if raw_mm_kwargs:
kwargs_data, mm_metadata = _encode_mm_kwargs_with_metadata(
raw_mm_kwargs,
metadata_fields_for=metadata_fields_for,
)
return MultiModalFeatures(
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholders,
kwargs_data=kwargs_data,
mm_metadata=mm_metadata,
)
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encode/decode utilities for multimodal tensors and field metadata
over JSON/HTTP, used by the disaggregated generate endpoint."""
from __future__ import annotations
import pybase64
from vllm.multimodal.inputs import MultiModalKwargsItem
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
_encoder = MsgpackEncoder(size_threshold=2**62) # force all tensors inline
_decoder = MsgpackDecoder(t=MultiModalKwargsItem)
def encode_mm_kwargs_item(item: MultiModalKwargsItem) -> str:
"""Serialize a MultiModalKwargsItem to a base64 string."""
bufs = _encoder.encode(item)
assert len(bufs) == 1, "All tensors should be inline"
return pybase64.b64encode(bufs[0]).decode("ascii")
def decode_mm_kwargs_item(data: str) -> MultiModalKwargsItem:
"""Deserialize a base64 string back to a MultiModalKwargsItem."""
raw = pybase64.b64decode(data)
return _decoder.decode(raw)
@@ -1,904 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Annotated, Any, Literal, TypeAlias
from pydantic import (
BaseModel,
ConfigDict,
Discriminator,
Field,
NonNegativeInt,
PrivateAttr,
Tag,
field_validator,
model_validator,
)
from vllm.config import ModelConfig
from vllm.entrypoints.generate.base.protocol import (
PerRequestMetrics,
StreamOptions,
validate_cache_salt,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
CompletionStreamResponse,
)
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logprobs import Logprob
from vllm.outputs import WeightVersionSpan
from vllm.renderers import TokenizeParams
from vllm.sampling_params import SamplingParams
from vllm.utils import random_uuid
####### Tokens IN <> Tokens OUT #######
OutputMode: TypeAlias = Literal["tokens", "text"]
class PlaceholderRangeInfo(BaseModel):
"""Serializable placeholder location for a single multi-modal item."""
offset: int = Field(ge=0)
"""Start index of the placeholder tokens in the prompt."""
length: int = Field(gt=0)
"""Number of placeholder tokens."""
# TODO: add `is_embed: list[bool] | None` once the /generate side
# consumes features — some models (e.g. Qwen-VL) use sparse
# placeholder masks that cannot be recomputed from offset+length alone.
def _has_serialized_mm_items(
payload: dict[str, list[str | None]] | None,
) -> bool:
if not payload:
return False
return any(item is not None for items in payload.values() for item in items)
class MultiModalFeatures(BaseModel):
"""Lightweight multimodal metadata produced by the render step.
Carries hashes (for cache lookup / identification) and placeholder
positions so the downstream `/generate` service knows *where* in
the token sequence each multimodal item lives.
"""
mm_hashes: dict[str, list[str]]
"""Per-modality item hashes, e.g. `{"image": ["abc", "def"]}`."""
mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
"""Per-modality placeholder ranges in the token sequence."""
kwargs_data: dict[str, list[str | None]] | None = None
"""Per-modality serialized tensor data.
Each value is a list parallel to `mm_hashes[modality]`. A `str`
entry is a base64-encoded `MultiModalKwargsItem`; `None` means
the item should be resolved from cache. The entire field is
`None` for metadata-only (cache-hit) responses.
"""
mm_metadata: dict[str, list[str | None]] | None = None
"""Per-modality serialized metadata for disaggregated prefill.
Each value is a list parallel to `mm_hashes[modality]`. A `str`
entry is a base64-encoded `MultiModalKwargsItem` containing only
placeholder-metadata and `keep_on_cpu` fields. `None` means that
the metadata is unavailable for that item. Prefill can use this
instead of `kwargs_data` only when `ec_transfer_params` is also
set, so embeddings arrive through the EC connector rather than from
`pixel_values`.
"""
@model_validator(mode="after")
def _validate_parallel_fields(self) -> "MultiModalFeatures":
modalities = set(self.mm_hashes)
if set(self.mm_placeholders) != modalities:
raise ValueError(
"mm_hashes and mm_placeholders must use the same modalities"
)
if self.kwargs_data is not None and set(self.kwargs_data) != modalities:
raise ValueError("kwargs_data must use the same modalities as mm_hashes")
if self.mm_metadata is not None and set(self.mm_metadata) != modalities:
raise ValueError("mm_metadata must use the same modalities as mm_hashes")
flattened_ranges: list[tuple[int, int]] = []
for modality in modalities:
num_hashes = len(self.mm_hashes[modality])
num_placeholders = len(self.mm_placeholders[modality])
if num_hashes != num_placeholders:
raise ValueError(
f"{modality} mm_hashes and mm_placeholders must have "
"the same length"
)
if (
self.kwargs_data is not None
and len(self.kwargs_data[modality]) != num_hashes
):
raise ValueError(
f"{modality} kwargs_data and mm_hashes must have the same length"
)
if (
self.mm_metadata is not None
and len(self.mm_metadata[modality]) != num_hashes
):
raise ValueError(
f"{modality} mm_metadata and mm_hashes must have the same length"
)
flattened_ranges.extend(
(placeholder.offset, placeholder.offset + placeholder.length)
for placeholder in self.mm_placeholders[modality]
)
flattened_ranges.sort()
for (offset, end), (next_offset, _) in zip(
flattened_ranges, flattened_ranges[1:]
):
if next_offset < end:
raise ValueError(
"mm_placeholders must be globally non-overlapping and sorted"
)
return self
class ReasoningParserKwargs(BaseModel):
"""Kwargs for the engine-side reasoning parser that gates structured
outputs. Typed so clients cannot pass arbitrary constructor kwargs."""
model_config = ConfigDict(extra="forbid")
chat_template_kwargs: dict[str, Any] = Field(default_factory=dict)
"""The effective chat template kwargs the prompt was rendered with."""
class GenerateRequest(BaseModel):
request_id: str = Field(
default_factory=lambda: f"{random_uuid()}",
description=(
"The request_id related to this request. If the caller does "
"not set it, a random_uuid will be generated. This id is used "
"through out the inference process and return in response."
),
)
token_ids: list[int] = Field(min_length=1)
"""The token ids to generate text from."""
@field_validator("token_ids")
@classmethod
def validate_token_ids(cls, v: list[int]) -> list[int]:
if any(t < 0 for t in v):
raise ValueError("token_ids must not contain negative values")
return v
token_offsets: list[tuple[int, int]] | None = None
"""Char-level (start, end) offsets per token, relative to the
tokenized source string. Present only when the request set
`return_token_offsets=True` and the renderer was able to compute
them (Fast tokenizer, text input, no multimodal data). List length
equals `token_ids` length when present. None otherwise."""
features: MultiModalFeatures | None = None
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
content_parts: list[dict[str, Any]] | None = None
"""Raw multimodal input; server resolves media. Mutually exclusive
with `features`."""
@model_validator(mode="after")
def _check_mm_fields_exclusive(self) -> "GenerateRequest":
if self.content_parts and self.features:
raise ValueError("content_parts and features are mutually exclusive")
return self
@model_validator(mode="after")
def _require_ec_for_metadata_only(self) -> "GenerateRequest":
features = self.features
if features is None:
return self
if not _has_serialized_mm_items(features.mm_metadata):
return self
if self.ec_transfer_params:
return self
kwargs_data = features.kwargs_data or {}
for modality, metadata_items in (features.mm_metadata or {}).items():
kwargs_items = kwargs_data.get(modality, [None] * len(metadata_items))
if any(
metadata is not None and kwargs is None
for metadata, kwargs in zip(metadata_items, kwargs_items, strict=True)
):
raise ValueError(
"metadata-only multimodal items require ec_transfer_params"
)
return self
sampling_params: SamplingParams
"""The sampling parameters for the model."""
model: str | None = None
reasoning_ended: bool | None = None
"""Whether reasoning has ended before the first generated token, as
resolved by /render. `True` applies structured outputs from the first
token; `None` lets the engine check the prompt with its reasoning parser."""
reasoning_parser_kwargs: ReasoningParserKwargs | None = None
"""Set by /render when it has a reasoning parser, so the engine-side
parser agrees with the frontend on flags such as `enable_thinking`."""
return_token_ids: bool | None = Field(
default=None,
description=(
"If true, return the final prompt token IDs after multimodal "
"placeholder expansion, together with multimodal placeholder ranges. "
"In streaming mode, this metadata is included only in the first chunk."
),
)
output_mode: OutputMode = Field(
default="tokens",
description=(
"Response level. 'tokens' returns token IDs only. 'text' also "
"returns the detokenized text and logprobs with decoded token "
"strings. 'text' needs a server that loads a tokenizer and "
"sampling_params.detokenize to be true."
),
)
stream: bool | None = False
stream_options: StreamOptions | None = None
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
priority: int = Field(
default=0,
ge=-(2**63),
le=2**63 - 1,
description=(
"The priority of the request (lower means earlier handling; "
"default: 0). Any priority other than 0 will raise an error "
"if the served model does not use priority scheduling."
),
)
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
# Tracks which keys the caller explicitly set inside `sampling_params`
# when the request was parsed from a JSON body. Lets the server tell
# "client said max_tokens=16" from "client said nothing → dataclass
# default 16" so it can apply server-side defaulting only in the latter
# case. `None` means the request was constructed with a pre-built
# `SamplingParams` instance (e.g. from internal callers that have
# already resolved values), in which case all fields are considered set.
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
_response_mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = (
PrivateAttr(default=None)
)
@model_validator(mode="wrap")
@classmethod
def _capture_sampling_params_provided_keys(cls, data: Any, handler):
provided: set[str] | None = None
if isinstance(data, dict):
sp = data.get("sampling_params")
if isinstance(sp, dict):
provided = set(sp.keys())
instance = handler(data)
instance._sampling_params_provided_keys = provided
return instance
@model_validator(mode="before")
@classmethod
def _validate_cache_salt(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
@model_validator(mode="after")
def _check_output_mode_detokenizes(self) -> "GenerateRequest":
if self.output_mode != "tokens" and not self.sampling_params.detokenize:
raise ValueError(
f"output_mode={self.output_mode!r} requires "
"sampling_params.detokenize to be true"
)
return self
@model_validator(mode="after")
def _validate_multimodal_feature_bounds(self) -> "GenerateRequest":
if self.features is None:
return self
prompt_len = len(self.token_ids)
for ranges in self.features.mm_placeholders.values():
for placeholder in ranges:
if placeholder.offset + placeholder.length > prompt_len:
raise ValueError(
"mm_placeholders must remain within the token_ids sequence"
)
return self
def is_sampling_param_provided(self, name: str) -> bool:
"""Whether the caller explicitly set `sampling_params.<name>`.
For requests parsed from a JSON body, this reflects the raw input
dict. For requests constructed with a pre-built `SamplingParams`
instance, all fields are considered provided so server-side defaults
do not clobber values already resolved upstream.
"""
if self._sampling_params_provided_keys is None:
return True
return name in self._sampling_params_provided_keys
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return TokenizeParams(
max_total_tokens=None,
max_output_tokens=0,
)
class GenerateLogProb(BaseModel):
"""A single (token, logprob) candidate on the generate wire protocol.
Unlike the OpenAI logprob shapes this carries the integer token id: the
generate server has no tokenizer, so decoding to a string belongs in
derender (or the coupled chat/completions path), not here.
"""
token_id: int
logprob: float
rank: int | None = None
class GenerateLogProbsContent(GenerateLogProb):
"""The sampled token at one position, plus its top-k candidates.
``top_logprobs`` is a list, not a dict: JSON turns dict keys into strings
and the order would be implicit. It is in the engine's order: the sampled
token first, then the remaining candidates in rank order.
"""
top_logprobs: list[GenerateLogProb] = []
class GenerateLogProbs(BaseModel):
"""Output logprobs for one choice.
``content`` holds one entry per generated token.
"""
content: list[GenerateLogProbsContent] | None = None
class GenerateChoiceBase(BaseModel):
"""Fields shared by every `output_mode` of a non-streaming choice."""
index: int
# per OpenAI spec this is the default
finish_reason: str | None = "stop"
# Stop string or token id that ended generation (repetition detection
# sets it too); None on EOS, length and abort.
stop_reason: int | str | None = None
token_ids: list[int] | None = None
# Per-token expert routing decisions, base64-encoded `.npy` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
# `num_tokens - 1` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
# np.load(io.BytesIO(base64.b64decode(s)))
# `None` if (a) the request was aborted before any forward pass,
# or (b) `enable_return_routed_experts` is off server-side.
routed_experts: str | None = None
sampling_mask: list[list[int]] | None = None
# Contiguous weight version spans over output-token indexes.
weight_versions: list[WeightVersionSpan] | None = None
@field_validator("token_ids")
@classmethod
def validate_token_ids(cls, v: list[int] | None) -> list[int] | None:
if v is not None and any(t < 0 for t in v):
raise ValueError("token_ids must not contain negative values")
return v
class GenerateTokensChoice(GenerateChoiceBase):
logprobs: GenerateLogProbs | None = None
"""Logprobs with integer token ids (no tokenizer on the generate server)."""
class GenerateTextChoice(GenerateChoiceBase):
text: str
"""Detokenized output. Excludes a matched stop string unless
`include_stop_str_in_output` is set, while `token_ids` keeps every
generated token."""
logprobs: ChatCompletionLogProbs | None = None
"""Logprobs with decoded token strings and `bytes`."""
class GenerateStreamChoiceBase(BaseModel):
"""Fields shared by every `output_mode` of a streaming choice."""
index: int
finish_reason: str | None = None
# Stop string or token id that ended generation (repetition detection
# sets it too); None on EOS, length and abort.
stop_reason: int | str | None = None
token_ids: list[int] | None = None
routed_experts: str | None = None
sampling_mask: list[list[int]] | None = None
# Set on the final chunk only; covers the whole output.
weight_versions: list[WeightVersionSpan] | None = None
class GenerateTokensStreamChoice(GenerateStreamChoiceBase):
logprobs: GenerateLogProbs | None = None
class GenerateTextStreamChoice(GenerateStreamChoiceBase):
text: str
"""Text delta since the previous chunk of this choice."""
logprobs: ChatCompletionLogProbs | None = None
class GenerateStreamResponseBase(BaseModel):
request_id: str = Field(
default_factory=lambda: f"{random_uuid()}",
description=(
"The request_id related to this request. If the caller does "
"not set it, a random_uuid will be generated. This id is used "
"through out the inference process and return in response."
),
)
usage: UsageInfo | None = Field(default=None)
prompt_token_ids: list[NonNegativeInt] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
metrics: PerRequestMetrics | None = None
class GenerateTokensStreamResponse(GenerateStreamResponseBase):
output_mode: Literal["tokens"] = "tokens"
choices: list[GenerateTokensStreamChoice]
class GenerateTextStreamResponse(GenerateStreamResponseBase):
output_mode: Literal["text"] = "text"
choices: list[GenerateTextStreamChoice]
class GenerateResponseBase(BaseModel):
request_id: str = Field(
default_factory=lambda: f"{random_uuid()}",
description=(
"The request_id related to this request. If the caller does "
"not set it, a random_uuid will be generated. This id is used "
"through out the inference process and return in response."
),
)
model: str | None = None
created: int | None = None
usage: UsageInfo | None = Field(default=None)
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_id_logprobs: str | None = None
prompt_token_ids: list[NonNegativeInt] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
metrics: PerRequestMetrics | None = None
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
class GenerateTokensResponse(GenerateResponseBase):
output_mode: Literal["tokens"] = "tokens"
choices: list[GenerateTokensChoice]
class GenerateTextResponse(GenerateResponseBase):
output_mode: Literal["text"] = "text"
choices: list[GenerateTextChoice]
def output_mode_or_tokens(value: Any) -> Any:
"""Discriminator for `GenerateResponse` and `GenerateStreamResponse`.
A missing `output_mode` means `tokens`. Existing clients and servers
that predate the field never send it.
"""
if isinstance(value, dict):
return value.get("output_mode", "tokens")
return getattr(value, "output_mode", "tokens")
# Type aliases, not classes: use `GenerateResponseBase` or
# `GenerateStreamResponseBase` for `isinstance` checks and the
# `Generate<Level>...` classes to construct.
GenerateResponse: TypeAlias = Annotated[
Annotated[GenerateTokensResponse, Tag("tokens")]
| Annotated[GenerateTextResponse, Tag("text")],
Discriminator(output_mode_or_tokens),
]
GenerateStreamResponse: TypeAlias = Annotated[
Annotated[GenerateTokensStreamResponse, Tag("tokens")]
| Annotated[GenerateTextStreamResponse, Tag("text")],
Discriminator(output_mode_or_tokens),
]
class DerenderChatRequest(BaseModel):
"""Request for the /v1/chat/completions/derender endpoint (non-streaming).
Wraps a complete GenerateResponse and caller supplied metadata needed to
produce a fully formed ChatCompletionResponse without a GPU.
"""
# --8<-- [start:derender-chat-request]
stream: Literal[False] = False
model: str | None = None
"""Served model name. Defaults to the server's served model name."""
generate_response: GenerateTokensResponse
"""The complete token-in / token-out engine response to derender.
Only `output_mode="tokens"` responses are accepted. Other modes are
already detokenized and re-decoding their `token_ids` would undo the
engine's stop string truncation.
"""
prompt_tokens: int | None = None
"""Prompt token count for usage; defaults to 0 if omitted.
GenerateResponse carries only output tokens; the caller already has
len(GenerateRequest.token_ids) from the render step.
"""
prompt_token_ids: list[NonNegativeInt] | None = None
"""Prompt token IDs (`GenerateRequest.token_ids` from /render). Seeds
detokenization from the prompt tail so the first output token keeps its
leading space on SentencePiece tokenizers. Falls back to
`generate_response.prompt_token_ids`, then to unseeded decoding.
Only the last few IDs are read, so a suffix of the prompt is enough.
"""
chat_request: ChatCompletionRequest | None = None
"""The original (post-adjust_request) ChatCompletionRequest from /render.
Required by the parsing so that tool/reasoning parsers can receive the full
request context they expect (request.tools, request.tool_choice,
request._grammar_from_parser, etc.).
"""
# --8<-- [end:derender-chat-request]
class DerenderCompletionRequest(BaseModel):
"""Request for the /v1/completions/derender endpoint (non-streaming).
Parallel to DerenderChatRequest but handles the multi-prompt completions
case: one GenerateResponse per prompt, mirroring the list[GenerateRequest]
returned by /v1/completions/render.
"""
# --8<-- [start:derender-completion-request]
stream: Literal[False] = False
model: str | None = None
"""Served model name. Defaults to the server's served model name."""
generate_responses: list[GenerateTokensResponse]
"""One `output_mode="tokens"` response per prompt, parallel to the
list[GenerateRequest] returned by /v1/completions/render."""
prompt_tokens: list[int] | None = None
"""One prompt token count per response; each defaults to 0 if omitted.
If provided, len(prompt_tokens) must equal len(generate_responses).
"""
prompt_token_ids: list[list[NonNegativeInt] | None] | None = None
"""One prompt token ID list per response, used to seed detokenization.
See `DerenderChatRequest.prompt_token_ids`.
If provided, len(prompt_token_ids) must equal len(generate_responses).
A `None` entry falls back to `generate_responses[i].prompt_token_ids`.
"""
completion_request: CompletionRequest | None = None
"""The original (post-adjust_request) CompletionRequest from /render.
Mirrors chat_request on DerenderChatRequest. Required by the parsing
so parsers receive the full request context.
"""
# --8<-- [end:derender-completion-request]
@model_validator(mode="after")
def _validate_prompt_tokens_length(self) -> "DerenderCompletionRequest":
if self.prompt_tokens is not None and len(self.prompt_tokens) != len(
self.generate_responses
):
raise ValueError(
f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
f"generate_responses length ({len(self.generate_responses)})"
)
if self.prompt_token_ids is not None and len(self.prompt_token_ids) != len(
self.generate_responses
):
raise ValueError(
f"prompt_token_ids length ({len(self.prompt_token_ids)}) must "
f"equal generate_responses length ({len(self.generate_responses)})"
)
return self
class DerenderStreamState(BaseModel):
"""Per sequence state for stateless streaming derender.
The client carries this between successive per chunk HTTP calls to the
streaming derender endpoint. All fields are plain JSON serializable data.
No opaque tokenizer or parser internals are stored here.
Two separate sets of fields support two different streaming modes:
- For plain streaming (no parser configured including the completions path):
`prev_tokens`, `prefix_offset` and `read_offset` maintain a bounded incremental
decoding window. This requires O(window) transport and O(delta) computation
per chunk.
- For parser enabled chat streaming: `output_token_ids`, `output_chunk_lens`,
`tools_streamed` and `last_tool_call_ids` are used to replay `parse_delta()`
from scratch on every chunk because parser state cannot be serialized. This
incurs O(n) transport per chunk (O(n²) per generation) and O(n²)
`parse_delta()` calls per generation. Since many parsers re-scan the
entire accumulated text on each invocation, `parse_delta()` itself is
O(n) yielding a true worst case compute cost of O(n³) per generation.
No caching is performed. Work is bounded by `max_model_len`.
The detokenization strategy carries the incremental decode offsets
directly rather than re-sending the whole token history each chunk.
`detokenize_incrementally` only ever reads the trailing token window
`prev_tokens[prefix_offset:]`, so we carry just that tail plus the two
offsets. Each chunk resumes exactly where the last one stopped, including
any partially processed multi-byte character (tracked by `read_offset`),
then trims and rebases the window so it never grows with generation length.
Performance:
- Compute per chunk is O(delta). One `detokenize_incrementally` call per
new token, independent of how many tokens preceded it.
- Transport per chunk is O(window). The carried tail is bounded by the
incremental detokenization offset, so cumulative bytes over the wire are
O(n) rather than the O(n^2) a full history round trip would incur.
"""
prev_tokens: list[str] = Field(default_factory=list)
"""Trailing decode window. Token strings from `prefix_offset` onward.
Bounded, trimmed and rebased each chunk to the tail
`detokenize_incrementally` still reads, so it does not grow with the
number of chunks.
"""
prefix_offset: int = Field(default=0, ge=0)
"""Prefix offset into `prev_tokens` for incremental detokenization."""
read_offset: int = Field(default=0, ge=0)
"""Read offset into `prev_tokens` for incremental detokenization."""
@field_validator("prev_tokens")
@classmethod
def _bound_prev_tokens(cls, v: list[str]) -> list[str]:
# INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET is small (5) and the trimmed
# window is O(offset). A generous limit rejects unusually large or malformed
# payloads without restricting legitimate multi-byte sequences.
limit = 1024
if len(v) > limit:
raise ValueError(f"prev_tokens length ({len(v)}) exceeds maximum ({limit})")
return v
role_sent: bool = False
"""True once the initial `role: "assistant"` delta has been emitted.
Prevents re-emitting the role on subsequent chunks even when the detok
window is transiently empty (e.g. usage only final chunk).
"""
logprob_context_token_ids: list[int] = Field(default_factory=list, max_length=4)
"""Trailing sampled token IDs carried across chunks so byte-fallback
(U+FFFD) correction during logprob placeholder resolution has context
at chunk boundaries. Bounded to the 4-token window that
``_correct_decoded_token`` reads."""
logprob_text_offset: int = Field(default=0, ge=0)
"""Cumulative emitted text length. Seeds ``text_offset`` for completion
streaming logprobs so offsets stay absolute across chunks, mirroring
``initial_text_offset`` in the generate streaming path."""
output_token_ids: list[int] = Field(default_factory=list)
"""All output tokens seen so far. Parser path only.
Replay buffer: each chunk rebuilds a fresh parser and replays every
token in here through `parse_delta` (discarding the result) before
processing the current chunk's tokens since parser internal state
cannot be serialized into this stateless model. Unavoidably O(n)
bounded by `max_model_len` (enforced server side, not by a field
validator here since the bound is model dependent).
"""
output_chunk_lens: list[Annotated[int, Field(gt=0)]] = Field(default_factory=list)
"""Token count of each chunk in `output_token_ids`. Parser path only.
Replay uses these to reproduce the original `parse_delta` call
boundaries. Must sum to `len(output_token_ids)`.
"""
@model_validator(mode="after")
def _validate_output_chunk_lens(self) -> "DerenderStreamState":
total = sum(self.output_chunk_lens)
if total != len(self.output_token_ids):
raise ValueError(
f"output_chunk_lens must sum to len(output_token_ids) "
f"(got sum={total}, len(output_token_ids)="
f"{len(self.output_token_ids)})"
)
return self
tools_streamed: bool = False
"""True once a tool call delta has been emitted. Parser path only.
Drives the `finish_reason` -> `"tool_calls"` rewrite on the final
chunk mirroring the generate streaming path.
"""
last_tool_call_ids: list[str] = Field(default_factory=list)
"""Stable tool call IDs, assigned once when each call first appears.
Indexed by tool call index. Parser path only. Prevents ID regeneration
when replay reprocesses a tool call that already has a pinned ID.
"""
class DerenderChatStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/chat/completions/derender.
The client sends one request per SSE chunk received from
`/inference/v1/generate`. Each request carries the generate chunk
plus the `stream_state` returned by the previous call (`None` on the
first call). The response contains the derendered chunk and the updated
state to be passed to the next call.
This implements stateless no server side session. All mutable state lives in
the client carried `stream_state`.
"""
# --8<-- [start:derender-chat-stream-request]
stream: Literal[True]
model: str | None = None
generate_chunk: GenerateTokensStreamResponse
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`
(`stream=True`)."""
stream_state: DerenderStreamState | None = None
"""Client carried detok state from the previous call. `None` on first."""
prompt_tokens: int | None = None
"""Prompt token count for usage. Forwarded from the render step."""
prompt_token_ids: list[NonNegativeInt] | None = None
"""Prompt token IDs. Required by the parser path's `parse_delta` to
settle its initial reasoning state (e.g. chat templates that pre-open
`<think>`). `prompt_tokens` is a usage count and cannot serve this
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
Rejected with a 400 when a tool or reasoning
parser is configured and this is omitted. Without it, `parse_delta`
cannot tell whether the prompt left reasoning open and would silently
misclassify reasoning content as plain content. On all paths it also
seeds detokenization on the first chunk, falling back to
`generate_chunk.prompt_token_ids` when omitted (see
`DerenderChatRequest.prompt_token_ids`).
With a parser configured, send the full list. Parsers look back to the
last reasoning marker which can be anywhere in the prompt, so a suffix
can flip the initial reasoning state. Without a parser, a suffix is
enough.
"""
chat_request: ChatCompletionRequest | None = None
"""The original (post adjust_request) ChatCompletionRequest from /render."""
# --8<-- [end:derender-chat-stream-request]
class DerenderCompletionStreamRequest(BaseModel):
"""One chunk streaming derender request for /v1/completions/derender.
Parallel to `DerenderChatStreamRequest` for the completions endpoint.
Each call processes one SSE chunk (one output sequence's delta) and
returns the derendered chunk plus updated state.
"""
# --8<-- [start:derender-completion-stream-request]
stream: Literal[True]
model: str | None = None
generate_chunk: GenerateTokensStreamResponse
"""One `output_mode="tokens"` SSE chunk from `/inference/v1/generate`."""
stream_state: DerenderStreamState | None = None
"""Client-carried detok state. `None` on the first call."""
prompt_tokens: int | None = None
"""Prompt token count for usage."""
prompt_token_ids: list[NonNegativeInt] | None = None
"""Prompt token IDs, used on the first chunk to seed detokenization.
Falls back to `generate_chunk.prompt_token_ids`. See
`DerenderChatRequest.prompt_token_ids`.
"""
completion_request: CompletionRequest | None = None
"""The original (post adjust_request) CompletionRequest from /render."""
# --8<-- [end:derender-completion-stream-request]
class DerenderChatStreamResponse(BaseModel):
"""Response for one streaming chat derender chunk.
Pairs the derendered SSE chunk with the updated client carried state to
pass to the next call.
"""
chunk: ChatCompletionStreamResponse
stream_state: DerenderStreamState
class DerenderCompletionStreamResponse(BaseModel):
"""Response for one streaming completions derender chunk.
Parallel to `DerenderChatStreamResponse` for the completions endpoint.
"""
chunk: CompletionStreamResponse
stream_state: DerenderStreamState
# Determines the type by checking the `stream` field's literal value. A body without
# `stream` validates as the non-streaming member
# (`stream` defaults to `False` there), so FastAPI can validate and dispatch both
# shapes on a single path.
DerenderChatRequestUnion: TypeAlias = DerenderChatRequest | DerenderChatStreamRequest
DerenderCompletionRequestUnion: TypeAlias = (
DerenderCompletionRequest | DerenderCompletionStreamRequest
)
@@ -1,810 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import math
import time
from collections.abc import AsyncGenerator
from collections.abc import Sequence as GenericSequence
from typing import Any
import msgspec
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.protocol import (
PerRequestMetrics,
RequestResponseMetadata,
)
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
build_spec_decoding_metrics,
clamp_prompt_logprobs,
format_token_id_placeholder,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProb,
ChatCompletionLogProbs,
ChatCompletionLogProbsContent,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import (
ErrorResponse,
PromptTokenUsageInfo,
UsageInfo,
)
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import GenerationError
from vllm.inputs import EngineInput, TokensPrompt, mm_input
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import (
MultiModalKwargsItem,
MultiModalKwargsItems,
PlaceholderRange,
)
from vllm.outputs import RequestOutput
from vllm.renderers.chat_utils import AsyncMultiModalItemTracker
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import as_list
from vllm.utils.serial_utils import numpy2base64
from .mm_features import (
mm_kwargs_from_features,
placeholder_ranges_from_engine_input,
)
from .protocol import (
GenerateLogProb,
GenerateLogProbs,
GenerateLogProbsContent,
GenerateRequest,
GenerateResponse,
GenerateTextChoice,
GenerateTextResponse,
GenerateTextStreamChoice,
GenerateTextStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamChoice,
GenerateTokensStreamResponse,
)
logger = init_logger(__name__)
def _clamp_logprob(logprob: float) -> float:
"""The OpenAI shapes' floor for a logprob: ``-inf`` and NaN become ``-9999.0``.
``max(nan, -9999.0)`` is NaN in Python, so NaN needs its own check; NaN
would otherwise fail ``JSONResponse`` or reach the client as ``null``.
"""
return -9999.0 if math.isnan(logprob) else max(logprob, -9999.0)
def _logprob_token(
token_id: int, logprob: Logprob | None, tokenizer: TokenizerLike | None
) -> tuple[str, list[int] | None]:
"""Token string and UTF-8 bytes for one logprob entry.
Without a tokenizer the token is a ``token_id:N`` placeholder with no bytes.
"""
if tokenizer is None:
return format_token_id_placeholder(token_id), None
token = (
tokenizer.decode([token_id])
if logprob is None
else GenerateBaseServing._get_decoded_token(logprob, token_id, tokenizer)
)
return token, list(token.encode("utf-8", errors="replace"))
def _top_logprob(
token_id: int, logprob: Logprob, tokenizer: TokenizerLike | None
) -> ChatCompletionLogProb:
token, token_bytes = _logprob_token(token_id, logprob, tokenizer)
return ChatCompletionLogProb(
token=token, logprob=_clamp_logprob(logprob.logprob), bytes=token_bytes
)
class ServingTokens(GenerateBaseServing):
"""Provides Tokens IN <> Tokens OUT functionality to vLLM API."""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
online_renderer: OnlineRenderer,
*,
request_logger: RequestLogger | None,
force_no_detokenize: bool = False,
return_tokens_as_token_ids: bool = False,
enable_prompt_tokens_details: bool = False,
enable_log_outputs: bool = False,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.online_renderer = online_renderer
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_log_outputs = enable_log_outputs
self.force_no_detokenize = force_no_detokenize
self.has_tokenizer = (
not force_no_detokenize and self.renderer.tokenizer is not None
)
if force_no_detokenize:
logger.info(
"Tokens-only mode is enabled, skipping detokenization "
"step for incoming requests."
)
# Mirrors ``OpenAIServingChat`` so we can apply server-side
# ``max_tokens`` defaulting when the client omits it. Without this,
# ``SamplingParams.max_tokens`` falls back to its dataclass default
# of 16 and silently truncates every generation.
self.default_sampling_params = self.model_config.get_diff_sampling_param()
mc = self.model_config
self.override_max_tokens = (
self.default_sampling_params.get("max_tokens")
if mc.generation_config not in ("auto", "vllm")
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
def _validate_mm_cache_handles(
self,
mm_kwargs: dict[str, list[MultiModalKwargsItem | None]],
mm_hashes: dict[str, list[str]],
) -> ErrorResponse | None:
cache = self.online_renderer.renderer.mm_processor_cache
if cache is None:
return None
try:
for modality, items in mm_kwargs.items():
for mm_hash, item in zip(mm_hashes[modality], items, strict=True):
if item is not None:
cache.validate_input_item(item, mm_hash)
except ValueError as error:
return self.create_error_response(error)
return None
async def serve_tokens(
self,
request: GenerateRequest,
raw_request: Request | None = None,
) -> GenerateResponse | ErrorResponse | AsyncGenerator[str, None]:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
logger.error("Error with model %s", error_check_ret)
return error_check_ret
self._preflight()
lora_request = None
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
model_name = self.models.model_name(lora_request)
request_id = (
f"generate-tokens-{self._base_request_id(raw_request, request.request_id)}"
)
request_metadata = RequestResponseMetadata(request_id=request_id)
if raw_request:
raw_request.state.request_metadata = request_metadata
sampling_params = request.sampling_params
max_num_seqs = self.engine_client.vllm_config.scheduler_config.max_num_seqs
if sampling_params.n > max_num_seqs:
return self.create_error_response(
f"sampling_params.n must be at most the server's max_num_seqs "
f"({max_num_seqs}), got {sampling_params.n}."
)
# The stream schema has no field for the scores.
if request.stream and sampling_params.prompt_logprob_token_ids is not None:
return self.create_error_response(
"prompt_logprob_token_ids are not available when stream=true."
)
if self.force_no_detokenize and sampling_params.stop:
# SamplingParams rejects stop with detokenize=False at request
# validation, but this server forces detokenize=False afterwards,
# so the combination must be rejected here or stop strings are
# silently never applied.
return self.create_error_response(
"stop strings are not supported on a --tokens-only server "
"because detokenization is disabled. Check stop strings on "
"the coordinator, or use stop_token_ids."
)
if request.output_mode != "tokens" and not self.has_tokenizer:
# The no-op detokenizer returns "", so without this check the
# request would succeed with empty text.
return self.create_error_response(
f"output_mode={request.output_mode!r} requires a tokenizer, but "
"this server does not load one (--tokens-only or "
"--skip-tokenizer-init). Send the request to a server that "
"loads a tokenizer, or use output_mode='tokens'."
)
try:
msgspec.msgpack.encode(
(
sampling_params,
request.kv_transfer_params,
request.ec_transfer_params,
)
)
except (OverflowError, TypeError, ValueError) as e:
return self.create_error_response(e)
engine_input: EngineInput
if request.content_parts:
tracker = AsyncMultiModalItemTracker(self.model_config)
mm_parser = tracker.create_parser()
for part in request.content_parts:
ptype = part.get("type", "")
url = part.get("url")
uuid = part.get("uuid")
if ptype == "image_url":
mm_parser.parse_image(url, uuid)
elif ptype == "audio_url":
mm_parser.parse_audio(url, uuid)
elif ptype == "video_url":
mm_parser.parse_video(url, uuid)
mm_data, mm_uuids = await tracker.resolve_items()
prompt = TokensPrompt(prompt_token_ids=request.token_ids)
if request.cache_salt is not None:
prompt["cache_salt"] = request.cache_salt
if mm_data:
prompt["multi_modal_data"] = mm_data
if mm_uuids:
prompt["multi_modal_uuids"] = mm_uuids
(engine_input,) = await self.online_renderer.renderer.render_cmpl_async(
[prompt]
)
elif features := request.features:
# Convert PlaceholderRangeInfo → PlaceholderRange per modality.
mm_placeholders: dict[str, list[PlaceholderRange]] = {
modality: [
PlaceholderRange(offset=p.offset, length=p.length) for p in ranges
]
for modality, ranges in features.mm_placeholders.items()
}
# Deserialize full tensor data and optional metadata-only data.
# Metadata-only items are valid when ec_transfer_params is set.
mm_kwargs = mm_kwargs_from_features(features)
if error := self._validate_mm_cache_handles(mm_kwargs, features.mm_hashes):
return error
engine_input = mm_input(
prompt_token_ids=request.token_ids,
mm_kwargs=MultiModalKwargsItems(mm_kwargs),
mm_hashes=features.mm_hashes,
mm_placeholders=mm_placeholders,
cache_salt=request.cache_salt,
)
else:
(engine_input,) = await self.online_renderer.preprocess_completion(
request,
prompt_input=request.token_ids,
prompt_embeds=None,
skip_mm_cache=True,
)
# Offsets are relative to the decoder prompt, so they are not
# meaningful for encoder-decoder models.
request._response_mm_placeholders = (
placeholder_ranges_from_engine_input(engine_input)
if request.return_token_ids and not self.model_config.is_encoder_decoder
else None
)
# Schedule the request and get the result generator.
result_generator: AsyncGenerator[RequestOutput, None] | None = None
# Pass disaggregated-serving parameters through to the engine.
if request.kv_transfer_params is not None:
extra = sampling_params.extra_args or {}
extra["kv_transfer_params"] = request.kv_transfer_params
sampling_params.extra_args = extra
if request.ec_transfer_params is not None:
extra = sampling_params.extra_args or {}
extra["ec_transfer_params"] = request.ec_transfer_params
sampling_params.extra_args = extra
# Apply server-side ``max_tokens`` defaulting when the client did
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
# defaults ``max_tokens`` to 16, which would otherwise silently cap
# every generation that omits the field.
if not request.is_sampling_param_provided("max_tokens"):
sampling_params.max_tokens = get_max_tokens(
max_model_len=self.model_config.max_model_len,
max_tokens=None,
input_length=self._extract_prompt_len(engine_input),
default_sampling_params=self.default_sampling_params,
override_max_tokens=self.override_max_tokens,
)
if self.force_no_detokenize:
sampling_params.detokenize = False
sampling_params.output_kind = (
RequestOutputKind.DELTA if request.stream else RequestOutputKind.FINAL_ONLY
)
self._log_inputs(
request_id,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
# Extract data_parallel_rank from header (router can inject it)
data_parallel_rank = self._get_data_parallel_rank(raw_request)
session_id = self._get_session_id_from_headers(raw_request)
result_generator = self.engine_client.generate(
engine_input,
sampling_params,
request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
data_parallel_rank=data_parallel_rank,
session_id=session_id,
reasoning_ended=request.reasoning_ended,
reasoning_parser_kwargs=(
request.reasoning_parser_kwargs.model_dump()
if request.reasoning_parser_kwargs is not None
else None
),
)
assert result_generator is not None
if request.stream:
return self.serve_tokens_stream_generator(
request,
result_generator,
request_id,
model_name,
request_metadata,
)
return await self.serve_tokens_full_generator(
request, result_generator, request_id, model_name, request_metadata
)
async def serve_tokens_full_generator(
self,
request: GenerateRequest,
result_generator: AsyncGenerator[RequestOutput, None],
request_id: str,
model_name: str,
request_metadata: RequestResponseMetadata,
) -> ErrorResponse | GenerateResponse:
created_time = int(time.time())
final_res: RequestOutput | None = None
sampling_params: SamplingParams = request.sampling_params
text_mode = request.output_mode == "text"
tokenizer = self._logprobs_tokenizer(text_mode)
try:
async for res in result_generator:
final_res = res
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
assert final_res is not None
tokens_choices: list[GenerateTokensChoice] = []
text_choices: list[GenerateTextChoice] = []
num_generated_tokens = 0
for output in final_res.outputs:
self._raise_if_error(output.finish_reason, request_id)
token_ids = output.token_ids
out_logprobs = output.logprobs
# This is top_logprobs in completions API
if sampling_params.logprobs is not None:
assert out_logprobs is not None, "Did not output logprobs"
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
if text_mode:
logprobs = self._create_text_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=token_ids,
top_logprobs=out_logprobs,
)
else:
logprobs = None
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
else None
)
sampling_mask = None
if output.sampling_mask is not None:
sampling_mask = output.sampling_mask.token_ids
choice_fields: dict[str, Any] = dict(
index=output.index,
logprobs=logprobs,
finish_reason=output.finish_reason if output.finish_reason else "stop",
stop_reason=output.stop_reason,
token_ids=as_list(output.token_ids),
routed_experts=routed_experts_b64,
sampling_mask=sampling_mask,
weight_versions=output.weight_versions,
)
if text_mode:
text_choices.append(
GenerateTextChoice(text=output.text, **choice_fields)
)
else:
tokens_choices.append(GenerateTokensChoice(**choice_fields))
num_generated_tokens += len(output.token_ids)
assert final_res.prompt_token_ids is not None
num_prompt_tokens = len(final_res.prompt_token_ids)
if final_res.encoder_prompt_token_ids is not None:
num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
)
if (
self.enable_prompt_tokens_details
and final_res.num_cached_tokens is not None
):
# This info is not available at the /coordinator level
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=final_res.num_cached_tokens
)
request_metadata.final_usage_info = usage
per_request_metrics = None
if request.sampling_params.n == 1:
spec_stats = build_spec_decoding_metrics(final_res)
if spec_stats is not None:
per_request_metrics = PerRequestMetrics(speculative_decoding=spec_stats)
response_fields: dict[str, Any] = dict(
request_id=request_id,
created=created_time,
model=model_name,
usage=usage,
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
prompt_token_id_logprobs=(
numpy2base64(final_res.prompt_token_id_logprobs)
if final_res.prompt_token_id_logprobs is not None
else None
),
prompt_token_ids=(
final_res.prompt_token_ids if request.return_token_ids else None
),
mm_placeholders=request._response_mm_placeholders,
metrics=per_request_metrics,
kv_transfer_params=final_res.kv_transfer_params,
ec_transfer_params=final_res.ec_transfer_params,
)
response: GenerateTokensResponse | GenerateTextResponse
if text_mode:
response = GenerateTextResponse(choices=text_choices, **response_fields)
else:
response = GenerateTokensResponse(choices=tokens_choices, **response_fields)
# Log complete response if output logging is enabled
if self.enable_log_outputs and self.request_logger:
for choice in response.choices:
# Get the corresponding output token IDs
output_token_ids = None
if choice.index < len(final_res.outputs):
output_token_ids = final_res.outputs[choice.index].token_ids
if output_token_ids:
# Log token_ids only.
self.request_logger.log_outputs(
request_id=request_id,
outputs="",
output_token_ids=output_token_ids,
finish_reason=choice.finish_reason,
is_streaming=False,
delta=False,
)
return response
async def serve_tokens_stream_generator(
self,
request: GenerateRequest,
result_generator: AsyncGenerator[RequestOutput, None],
request_id: str,
model_name: str,
request_metadata: RequestResponseMetadata,
) -> AsyncGenerator[str, None]:
num_prompt_tokens = 0
first_iteration = True
prompt_token_ids: list[int] | None = None
num_cached_tokens = None
sampling_params: SamplingParams = request.sampling_params
# With n > 1, the first output may not include every choice yet.
num_generated_tokens = [0] * sampling_params.n
last_res: RequestOutput | None = None
text_mode = request.output_mode == "text"
tokenizer = self._logprobs_tokenizer(text_mode)
include_usage, include_continuous_usage = should_include_usage(
request.stream_options, False
)
try:
async for res in result_generator:
last_res = res
if first_iteration:
if res.prompt_token_ids is not None:
num_prompt_tokens = len(res.prompt_token_ids)
if request.return_token_ids:
prompt_token_ids = res.prompt_token_ids
if res.encoder_prompt_token_ids is not None:
num_prompt_tokens += len(res.encoder_prompt_token_ids)
num_cached_tokens = res.num_cached_tokens
first_iteration = False
for output in res.outputs:
i = output.index
delta_token_ids = output.token_ids
num_generated_tokens[i] += len(delta_token_ids)
finish_reason = output.finish_reason
self._raise_if_error(finish_reason, request_id)
# Terminal outputs are always emitted so the client sees
# the finish reason, e.g. an abort, which has no new
# tokens. Text mode also emits text held back for stop
# string matching, which can arrive without token IDs.
if not (
delta_token_ids
or finish_reason is not None
or (text_mode and output.text)
):
continue
if sampling_params.logprobs is not None:
out_logprobs = output.logprobs
assert out_logprobs is not None, "Did not output logprobs"
logprobs: GenerateLogProbs | ChatCompletionLogProbs | None
if text_mode:
logprobs = self._create_text_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
tokenizer=tokenizer,
)
else:
logprobs = self._create_tokens_logprobs(
token_ids=delta_token_ids,
top_logprobs=out_logprobs,
)
else:
logprobs = None
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
else None
)
sampling_mask = None
if output.sampling_mask is not None:
sampling_mask = output.sampling_mask.token_ids
choice_fields: dict[str, Any] = dict(
index=i,
logprobs=logprobs,
finish_reason=finish_reason,
stop_reason=output.stop_reason,
token_ids=as_list(delta_token_ids),
routed_experts=routed_experts_b64,
sampling_mask=sampling_mask,
weight_versions=output.weight_versions,
)
chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
if text_mode:
chunk = GenerateTextStreamResponse(
request_id=request_id,
choices=[
GenerateTextStreamChoice(
text=output.text, **choice_fields
)
],
)
else:
chunk = GenerateTokensStreamResponse(
request_id=request_id,
choices=[GenerateTokensStreamChoice(**choice_fields)],
)
if prompt_token_ids is not None:
chunk.prompt_token_ids = prompt_token_ids
chunk.mm_placeholders = request._response_mm_placeholders
prompt_token_ids = None
if include_continuous_usage:
chunk.usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=num_generated_tokens[i],
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
)
# Omit fields that are absent from token-bearing chunks.
exclude = {
name
for name in ("prompt_token_ids", "mm_placeholders", "metrics")
if getattr(chunk, name) is None
}
yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n"
total_completion_tokens = sum(num_generated_tokens)
final_usage_info = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=num_prompt_tokens + total_completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)
if include_usage:
per_request_metrics = None
if sampling_params.n == 1:
spec_stats = build_spec_decoding_metrics(last_res)
if spec_stats is not None:
per_request_metrics = PerRequestMetrics(
speculative_decoding=spec_stats
)
final_chunk: GenerateTokensStreamResponse | GenerateTextStreamResponse
if text_mode:
final_chunk = GenerateTextStreamResponse(
request_id=request_id,
choices=[],
usage=final_usage_info,
metrics=per_request_metrics,
)
else:
final_chunk = GenerateTokensStreamResponse(
request_id=request_id,
choices=[],
usage=final_usage_info,
metrics=per_request_metrics,
)
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
request_metadata.final_usage_info = final_usage_info
except GenerationError as e:
yield (
f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
)
except Exception as e:
logger.exception("Error in token generation stream.")
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
yield "data: [DONE]\n\n"
def _logprobs_tokenizer(self, text_mode: bool) -> TokenizerLike | None:
"""Tokenizer that resolves logprob tokens, or None for placeholders.
``--return-tokens-as-token-ids`` keeps placeholders at every level, as
it does on ``/v1/completions``.
"""
if not text_mode or self.return_tokens_as_token_ids:
return None
return self.renderer.tokenizer
def _create_text_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
tokenizer: TokenizerLike | None = None,
) -> ChatCompletionLogProbs:
"""Create OpenAI-style logprobs for ``output_mode="text"``.
With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``;
without one (``--return-tokens-as-token-ids``) they are ``token_id:N``
placeholders. Like tokens mode, ``top_logprobs`` carries every candidate
the engine returned (the sampled token first, then the top k), not cut
to the requested count.
"""
logprobs_content: list[ChatCompletionLogProbsContent] = []
for i, token_id in enumerate(token_ids):
step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
token, token_bytes = _logprob_token(token_id, None, tokenizer)
logprobs_content.append(
ChatCompletionLogProbsContent(token=token, bytes=token_bytes)
)
else:
step_token = step_top_logprobs[token_id]
token, token_bytes = _logprob_token(token_id, step_token, tokenizer)
logprobs_content.append(
ChatCompletionLogProbsContent(
token=token,
logprob=_clamp_logprob(step_token.logprob),
bytes=token_bytes,
top_logprobs=[
_top_logprob(top_id, logprob, tokenizer)
for top_id, logprob in step_top_logprobs.items()
],
)
)
return ChatCompletionLogProbs(content=logprobs_content)
def _create_tokens_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
) -> GenerateLogProbs:
"""Create generate-shaped logprobs (integer token ids, no tokenizer).
``top_logprobs`` carries every candidate the engine returned for the
position: the sampled token first, then the top k, so k entries when the
sampled token is in the top k and k + 1 when it is not. It is not cut to
the requested count: `/v1/chat/completions` and
`/v1/completions` cut differently, and derender applies each endpoint's
cut the way the coupled server does.
The engine reports rank 0 for a sampled token whose logprob is NaN; that
is not a rank, so it is sent as ``None`` (the logprob is clamped).
"""
logprobs_content: list[GenerateLogProbsContent] = []
for i, token_id in enumerate(token_ids):
step_top_logprobs = top_logprobs[i]
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
# Same sentinel the OpenAI shapes use when the sampled token
# has no entry in the engine's top-k map.
logprobs_content.append(
GenerateLogProbsContent(token_id=token_id, logprob=-9999.0)
)
else:
step_token = step_top_logprobs[token_id]
logprobs_content.append(
GenerateLogProbsContent(
token_id=token_id,
logprob=_clamp_logprob(step_token.logprob),
rank=step_token.rank or None,
top_logprobs=[
GenerateLogProb(
token_id=top_token_id,
logprob=_clamp_logprob(top_logprob.logprob),
rank=top_logprob.rank or None,
)
for top_token_id, top_logprob in step_top_logprobs.items()
],
)
)
return GenerateLogProbs(content=logprobs_content)
-6
View File
@@ -16,12 +16,6 @@ def register_vllm_serve_api_routers(app: FastAPI):
register_instrumentator_api_routers(app)
from vllm.entrypoints.serve.lora.api_router import (
attach_router as attach_lora_router,
)
attach_lora_router(app)
from vllm.entrypoints.serve.profile.api_router import (
attach_router as attach_profile_router,
)
+1 -90
View File
@@ -15,9 +15,7 @@ from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import VLLMNotFoundError
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
extract_prompt_len,
@@ -52,27 +50,11 @@ class BaseServing:
if self.request_logger is not None:
self.request_logger.log_request_body(request)
error_response = None
if self._is_model_supported(request.model):
return None
if request.model in self.models.lora_requests:
return None
if (
envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
and request.model
and (load_result := await self.models.resolve_lora(request.model))
):
if isinstance(load_result, LoRARequest):
return None
if (
isinstance(load_result, ErrorResponse)
and load_result.error.code == HTTPStatus.BAD_REQUEST.value
):
error_response = load_result
served_names = ", ".join(model.name for model in self.models.base_model_paths)
return error_response or self.create_error_response(
return self.create_error_response(
message=(
f"The model `{request.model}` does not exist. "
f"Valid aliases: {served_names}."
@@ -112,7 +94,6 @@ class BaseServing:
request_id: str,
inputs: PromptType | EngineInput,
params: SamplingParams | BeamSearchParams | None,
lora_request: LoRARequest | None,
) -> None:
if self.request_logger is None:
return
@@ -125,7 +106,6 @@ class BaseServing:
components.token_ids,
components.embeds,
params=params,
lora_request=lora_request,
)
@staticmethod
@@ -139,72 +119,3 @@ class BaseServing:
return req_id
return random_uuid() if default is None else default
def _get_message_types(self, request: AnyRequest) -> set[str]:
"""Retrieve the set of types from message content dicts up
until `_`; we use this to match potential multimodal data
with default per modality loras.
"""
message_types: set[str] = set()
if not hasattr(request, "messages"):
return message_types
messages = request.messages
if messages is None or isinstance(messages, (str, bytes)):
return message_types
for message in messages:
if (
isinstance(message, dict)
and "content" in message
and isinstance(message["content"], list)
):
for content_dict in message["content"]:
if "type" in content_dict:
message_types.add(content_dict["type"].split("_")[0])
return message_types
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
# when run offline. It would be nice to support additional
# tasks types in the future.
message_types = self._get_message_types(request)
default_mm_loras = set()
for lora in self.models.lora_requests.values():
# Best effort match for default multimodal lora adapters;
# There is probably a better way to do this, but currently
# this matches against the set of 'types' in any content lists
# up until '_', e.g., to match audio_url -> audio
if lora.lora_name in message_types:
default_mm_loras.add(lora)
# Currently only support default modality specific loras if
# we have exactly one lora matched on the request.
if len(default_mm_loras) == 1:
return default_mm_loras.pop()
return None
def _maybe_get_adapters(
self,
request: AnyRequest,
supports_default_mm_loras: bool = False,
) -> LoRARequest | None:
if request.model in self.models.lora_requests:
return self.models.lora_requests[request.model]
# Currently only support default modality specific loras
# if we have exactly one lora matched on the request.
if supports_default_mm_loras:
default_mm_lora = self._get_active_default_mm_loras(request)
if default_mm_lora is not None:
return default_mm_lora
if self._is_model_supported(request.model):
return None
# if _check_model has been called earlier, this will be unreachable
raise VLLMNotFoundError(f"The model `{request.model}` does not exist.")
+2 -31
View File
@@ -14,25 +14,12 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
)
from vllm.entrypoints.serve.tokenize.protocol import (
DetokenizeRequest,
TokenizeChatRequest,
TokenizeCompletionRequest,
TokenizeResponse,
)
from vllm.entrypoints.speech_to_text.transcription.protocol import (
TranscriptionRequest,
TranscriptionResponse,
)
from vllm.entrypoints.speech_to_text.translation.protocol import TranslationRequest
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
@@ -55,32 +42,16 @@ CompletionLikeRequest: TypeAlias = (
CompletionRequest
| TokenizeCompletionRequest
| DetokenizeRequest
| DerenderCompletionRequest
| DerenderCompletionStreamRequest
)
ChatLikeRequest: TypeAlias = (
ChatCompletionRequest
| BatchChatCompletionRequest
| TokenizeChatRequest
| DerenderChatRequest
| DerenderChatStreamRequest
)
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
AnyRequest: TypeAlias = (
CompletionLikeRequest
| ChatLikeRequest
| SpeechToTextRequest
| ResponsesRequest
| GenerateRequest
)
AnyRequest: TypeAlias = CompletionLikeRequest | ChatLikeRequest | ResponsesRequest
AnyResponse: TypeAlias = (
CompletionResponse
| ChatCompletionResponse
| TranscriptionResponse
| TokenizeResponse
| GenerateResponse
CompletionResponse | ChatCompletionResponse | TokenizeResponse
)
-99
View File
@@ -1,99 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, Response
from vllm import envs
from vllm.entrypoints.openai.models.api_router import models
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.lora.protocol import (
LoadLoRAAdapterRequest,
UnloadLoRAAdapterRequest,
)
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
logger = init_logger(__name__)
router = APIRouter()
def attach_router(app: FastAPI):
"""Attach the LoRA adapter load/unload endpoints to the API server.
Does nothing when dynamic LoRA updating is disabled. Handler levels are
snapshotted and restored because importing
model_hosting_container_standards may reconfigure root logging.
"""
if not envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
"""If LoRA dynamic loading & unloading is not enabled, do nothing."""
return
logger.warning(
"LoRA dynamic loading & unloading is enabled in the API server. "
"This should ONLY be used for local development!"
)
snapshot = [
(h, h.level)
for lg in (logging.getLogger(), logging.getLogger("vllm"))
for h in lg.handlers
]
try:
_attach_router(app)
finally:
for handler, level in snapshot:
handler.setLevel(level)
def _attach_router(app: FastAPI):
"""Register the LoRA adapter load/unload routes on the app."""
import model_hosting_container_standards.sagemaker as sagemaker_standards
@sagemaker_standards.register_load_adapter_handler(
request_shape={
"lora_name": "body.name",
"lora_path": "body.src",
"load_inplace": "body.load_inplace || `false`",
"is_3d_lora_weight": "body.is_3d_lora_weight || `false`",
},
)
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
"""Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
serving engine."""
handler: OpenAIServingModels = models(raw_request)
response = await handler.load_lora_adapter(request)
if isinstance(response, ErrorResponse):
return JSONResponse(
content=response.model_dump(), status_code=response.error.code
)
return Response(status_code=200, content=response)
@sagemaker_standards.register_unload_adapter_handler(
request_shape={
"lora_name": "path_params.adapter_name",
}
)
@router.post(
"/v1/unload_lora_adapter", dependencies=[Depends(validate_json_request)]
)
async def unload_lora_adapter(
request: UnloadLoRAAdapterRequest, raw_request: Request
):
"""Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
the serving engine."""
handler: OpenAIServingModels = models(raw_request)
response = await handler.unload_lora_adapter(request)
if isinstance(response, ErrorResponse):
return JSONResponse(
content=response.model_dump(), status_code=response.error.code
)
return Response(status_code=200, content=response)
# register the router
app.include_router(router)
-16
View File
@@ -1,16 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pydantic import BaseModel, Field
class LoadLoRAAdapterRequest(BaseModel):
lora_name: str
lora_path: str
load_inplace: bool = False
is_3d_lora_weight: bool = False
class UnloadLoRAAdapterRequest(BaseModel):
lora_name: str
lora_int_id: int | None = Field(default=None)
@@ -65,8 +65,6 @@ class ServingTokenization(BaseServing):
request_id = f"tokenize-{self._base_request_id(raw_request)}"
lora_request = self._maybe_get_adapters(request)
if isinstance(request, TokenizeChatRequest):
tool_dicts = (
None
@@ -104,7 +102,6 @@ class ServingTokenization(BaseServing):
request_id,
engine_input,
params=None,
lora_request=lora_request,
)
prompt_components = self._extract_prompt_components(engine_input)
@@ -134,13 +131,10 @@ class ServingTokenization(BaseServing):
request_id = f"tokenize-{self._base_request_id(raw_request)}"
lora_request = self._maybe_get_adapters(request)
self._log_inputs(
request_id,
tokens_input(request.tokens),
params=None,
lora_request=lora_request,
)
tok_prompt = await self.renderer.tokenize_prompt_async(
-22
View File
@@ -20,7 +20,6 @@ from starlette.background import BackgroundTask, BackgroundTasks
from vllm import envs
from vllm.engine.arg_utils import EngineArgs
from vllm.entrypoints.generate.base.protocol import StreamOptions
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.logger import current_formatter_type, init_logger
from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -372,27 +371,6 @@ def should_include_usage(
return include_usage, include_continuous_usage
def process_lora_modules(
args_lora_modules: list[LoRAModulePath], default_mm_loras: dict[str, str] | None
) -> list[LoRAModulePath]:
from vllm.entrypoints.openai.models.serving import LoRAModulePath
lora_modules = args_lora_modules
if default_mm_loras:
default_mm_lora_paths = [
LoRAModulePath(
name=modality,
path=lora_path,
)
for modality, lora_path in default_mm_loras.items()
]
if args_lora_modules is None:
lora_modules = default_mm_lora_paths
else:
lora_modules += default_mm_lora_paths
return lora_modules
def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
if envs.VLLM_DISABLE_LOG_LOGO or (formatter := current_formatter_type(lgr)) is None:
message = "vLLM server version %s, serving model %s"
@@ -9,7 +9,6 @@ import torch
from vllm import envs
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import BeamSearchParams, SamplingParams
@@ -41,7 +40,6 @@ class RequestLogger:
prompt_token_ids: list[int] | None,
prompt_embeds: torch.Tensor | None,
params: SamplingParams | PoolingParams | BeamSearchParams | None,
lora_request: LoRARequest | None,
) -> None:
if logger.isEnabledFor(logging.DEBUG):
max_log_len = self.max_log_len
@@ -63,10 +61,9 @@ class RequestLogger:
)
logger.info(
"Received request %s: params: %s, lora_request: %s.",
"Received request %s: params: %s.",
request_id,
params,
lora_request,
)
def log_request_body(self, request: AnyRequest) -> None:
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,12 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Literal, TypeAlias
import torch
## Protocols for Audio
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
TranscriptionResponseFormat: TypeAlias = AudioResponseFormat | Literal["diarized_json"]
_LONG_INFO = torch.iinfo(torch.long)
@@ -1,846 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import math
import time
import zlib
from collections.abc import AsyncGenerator, Callable, Set
from concurrent.futures import ThreadPoolExecutor
from functools import cached_property
from typing import Final, Literal, TypeAlias, TypeVar, cast
import numpy as np
from fastapi import Request
from transformers import PreTrainedTokenizerBase
import vllm.envs as envs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
RequestResponseMetadata,
)
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.entrypoints.serve.engine.typing import SpeechToTextRequest
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import VLLMValidationError
from vllm.inputs import EncoderDecoderInput, EngineInput
from vllm.logger import init_logger
from vllm.logprobs import FlatLogprobs, Logprob
from vllm.model_executor.models import SupportsTranscription
from vllm.multimodal.audio import get_audio_duration, split_audio
from vllm.multimodal.media.audio import load_audio
from vllm.outputs import RequestOutput
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import get_tokenizer
from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterators
from ..transcription.protocol import (
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionSegment,
TranscriptionStreamResponse,
)
from ..translation.protocol import (
TranslationResponse,
TranslationResponseStreamChoice,
TranslationResponseVerbose,
TranslationSegment,
TranslationStreamResponse,
)
SpeechToTextResponse: TypeAlias = (
TranscriptionResponse | TranslationResponse | TranscriptionResponseDiarized
)
SpeechToTextResponseVerbose: TypeAlias = (
TranscriptionResponseVerbose | TranslationResponseVerbose
)
SpeechToTextSegment: TypeAlias = TranscriptionSegment | TranslationSegment
T = TypeVar("T", bound=SpeechToTextResponse)
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
S = TypeVar("S", bound=SpeechToTextSegment)
ResponseType: TypeAlias = (
TranscriptionResponse
| TranslationResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| TranslationResponseVerbose
)
logger = init_logger(__name__)
def asr_inter_chunk_separator(
language: str | None, no_space_languages: Set[str]
) -> str:
"""Space to insert between ASR text chunks for streaming and non-streaming join.
Languages in ``no_space_languages`` (e.g. Chinese, Japanese) use an empty
separator; others use a single ASCII space.
"""
return "" if language and language.lower() in no_space_languages else " "
class SpeechToTextBaseServing(GenerateBaseServing):
"""Base class for speech-to-text operations like transcription and
translation."""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
task_type: Literal["transcribe", "translate"] = "transcribe",
enable_force_include_usage: bool = False,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.default_sampling_params = self.model_config.get_diff_sampling_param()
self.task_type: Final = task_type
self.asr_config = self.model_cls.get_speech_to_text_config(
self.model_config, task_type
)
self.streaming_post_processor_cls = (
self.model_cls.get_streaming_post_processor_cls()
)
self.enable_force_include_usage = enable_force_include_usage
self.max_audio_decode_duration_s: int = envs.VLLM_MAX_AUDIO_DECODE_DURATION_S
self.max_audio_decode_bytes: int = envs.VLLM_MAX_AUDIO_DECODE_BYTES
if self.model_cls.supports_segment_timestamp:
self.tokenizer = cast(
PreTrainedTokenizerBase,
get_tokenizer(
tokenizer_name=self.model_config.tokenizer,
tokenizer_mode=self.model_config.tokenizer_mode,
),
)
if self.default_sampling_params:
logger.info(
"Overwriting default completion sampling param with: %s",
self.default_sampling_params,
)
# setup preprocess resources
# we keep separate thread pool for frontend preprocessing instead
# of reusing the one from Renderer which showed lower throughput
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
num_audio_preprocess_workers = envs.VLLM_MAX_AUDIO_PREPROCESS_WORKERS
self._preprocess_executor = ThreadPoolExecutor(
max_workers=num_audio_preprocess_workers,
thread_name_prefix="stt-preprocess",
)
self._decode_and_chunk_speech_async = make_async_with_semaphore(
self._decode_and_chunk_speech, executor=self._preprocess_executor
)
@cached_property
def model_cls(self) -> type[SupportsTranscription]:
from vllm.model_executor.model_loader import get_model_cls
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsTranscription], model_cls)
def shutdown(self) -> None:
self._preprocess_executor.shutdown(wait=False)
def _decode_and_chunk_speech(
self,
audio_data: bytes,
) -> tuple[list[np.ndarray], float]:
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
# transparently falls back to ffmpeg via an in-memory fd.
# NOTE resample to model SR here for efficiency. This is also a
# pre-requisite for chunking, as it assumes Whisper SR.
try:
with io.BytesIO(audio_data) as buf:
y, sr = load_audio(
buf,
sr=self.asr_config.sample_rate,
mono=True,
max_duration_s=self.max_audio_decode_duration_s,
max_decode_bytes=self.max_audio_decode_bytes,
)
except ValueError:
raise
except Exception as exc:
raise ValueError("Invalid or unsupported audio file.") from exc
duration = get_audio_duration(y=y, sr=sr)
do_split_audio = self.asr_config.allow_audio_chunking and (
self.asr_config.max_audio_clip_s is not None
and duration > self.asr_config.max_audio_clip_s
)
if not do_split_audio:
chunks = [y]
else:
assert self.asr_config.max_audio_clip_s is not None
assert self.asr_config.min_energy_split_window_size is not None
chunks = split_audio(
audio_data=y,
sample_rate=int(sr),
max_clip_duration_s=self.asr_config.max_audio_clip_s,
overlap_duration_s=self.asr_config.overlap_chunk_second,
min_energy_window_size=self.asr_config.min_energy_split_window_size,
)
return chunks, duration
async def _detect_language(
self,
audio_chunk: np.ndarray,
request_id: str,
) -> str:
"""Auto-detect the spoken language from an audio chunk.
Delegates prompt construction and output parsing to the model class
via ``get_language_detection_prompt`` and
``parse_language_detection_output``.
"""
prompt = self.model_cls.get_language_detection_prompt(
audio_chunk,
self.asr_config,
)
allowed_token_ids = self.model_cls.get_language_token_ids(
self.tokenizer,
)
sampling_params = SamplingParams(
max_tokens=1,
temperature=0.0,
allowed_token_ids=allowed_token_ids,
watermarking=False,
)
result_generator = self.engine_client.generate(
prompt,
sampling_params,
request_id,
)
try:
final_output: RequestOutput
async for final_output in result_generator:
if final_output.finished:
break
except asyncio.CancelledError:
await asyncio.gather(
self.engine_client.abort(request_id),
return_exceptions=True,
)
raise
token_ids = list(final_output.outputs[0].token_ids)
lang = self.model_cls.parse_language_detection_output(
token_ids,
self.tokenizer,
)
logger.info("Auto-detected language: '%s'", lang)
return lang
async def _preprocess_speech_to_text(
self,
request: SpeechToTextRequest,
audio_data: bytes,
request_id: str,
) -> tuple[list[EngineInput], float, list[float]]:
# Validate request
request.language = self.model_cls.validate_language(request.language)
request.to_language = (
self.model_cls.validate_language(request.to_language)
if request.to_language
else None
)
# Run cpu intensive preprocess step in a separate thread pool executor.
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
chunk_start_offsets: list[float] = [0.0]
for chunk in chunks[:-1]:
chunk_start_offsets.append(
chunk_start_offsets[-1] + chunk.shape[-1] / self.asr_config.sample_rate
)
if request.language is None and getattr(
self.model_cls, "supports_explicit_language_detection", False
):
# Auto-detect language from the first chunk.
request.language = await self._detect_language(
chunks[0], f"{request_id}-lang_detect"
)
parsed_prompts: list[DictPrompt] = []
for chunk in chunks:
stt_params = request.build_stt_params(
audio=chunk,
stt_config=self.asr_config,
model_config=self.model_config,
task_type=self.task_type,
)
prompt = self.model_cls.get_generation_prompt(stt_params)
parsed_prompt: DictPrompt
if request.response_format == "verbose_json":
parsed_prompt = parse_enc_dec_prompt(prompt)
parsed_prompt = self._preprocess_verbose_prompt(parsed_prompt)
else:
parsed_prompt = parse_model_prompt(self.model_config, prompt)
parsed_prompts.append(parsed_prompt)
engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)
return engine_inputs, duration, chunk_start_offsets
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
dec_prompt = prompt["decoder_prompt"]
if not (isinstance(dec_prompt, dict) and "prompt" in dec_prompt):
raise VLLMValidationError(
"Expected decoder_prompt to contain text",
parameter="decoder_prompt",
value=type(dec_prompt).__name__,
)
dec_prompt["prompt"] = dec_prompt["prompt"].replace(
"<|notimestamps|>", "<|0.00|>"
)
return prompt
@staticmethod
def _get_decoder_prompt_len(engine_inputs: list[EngineInput]) -> int:
"""Get the length of the decoder prompt. Currently we need to offset
by the decoder prompt length when running beam search because the mm
encoder is not currently cached and runs on decode calls; because of
this, we need to make sure the redundant encoder calls won't exceed
the context :(
FIXME (Alex) - this will be removed in the very near future once the
encoder/decoder caching is implemented.
"""
input_len = 0
assert len(engine_inputs) > 0
first_input = engine_inputs[0]
if first_input.get("type") == "enc_dec":
first_input = cast(EncoderDecoderInput, first_input)
input_len = len(first_input["decoder_prompt"]["prompt_token_ids"])
return input_len
def _get_verbose_segments(
self,
tokens: tuple,
log_probs: FlatLogprobs | list[dict[int, Logprob]],
request: SpeechToTextRequest,
segment_class: type[SpeechToTextSegment],
start_time: float = 0,
) -> list[SpeechToTextSegment]:
"""Convert tokens to verbose segments.
This method expects the model to produce
timestamps as tokens (similar to Whisper).
If the tokens do not include timestamp information,
the segments may not be generated correctly.
Note: No_speech_prob field is not supported
in this implementation and will be None. See docs for details.
"""
BASE_OFFSET = 0.02
init_token = self.tokenizer.encode("<|0.00|>", add_special_tokens=False)[0]
if tokens[-1] == self.tokenizer.eos_token_id:
tokens = tokens[:-1]
tokens_with_start = (init_token,) + tokens
segments: list[SpeechToTextSegment] = []
last_timestamp_start = 0
if tokens_with_start[-2] < init_token and tokens_with_start[-1] >= init_token:
tokens_with_start = tokens_with_start + (tokens_with_start[-1],)
avg_logprob = 0.0
for idx in range(1, len(tokens_with_start)):
# Timestamp tokens (e.g., <|0.00|>) are assumed to be sorted.
# If the ordering is violated, this slicing may produce incorrect results.
token = tokens_with_start[idx]
if token >= init_token and tokens_with_start[idx - 1] >= init_token:
sliced_timestamp_tokens = tokens_with_start[last_timestamp_start:idx]
start_timestamp = sliced_timestamp_tokens[0] - init_token
end_timestamp = sliced_timestamp_tokens[-1] - init_token
text = self.tokenizer.decode(sliced_timestamp_tokens[1:-1])
text_bytes = text.encode("utf-8")
casting_segment = cast(
SpeechToTextSegment,
segment_class(
id=len(segments),
seek=int(start_time),
start=start_time + BASE_OFFSET * start_timestamp,
end=start_time + BASE_OFFSET * end_timestamp,
temperature=request.temperature,
text=text,
# The compression ratio measures
# how compressible the generated text is.
# A higher ratio indicates more repetitive content,
# which is a strong sign of hallucination in outputs.
compression_ratio=len(text_bytes)
/ len(zlib.compress(text_bytes)),
tokens=sliced_timestamp_tokens[1:-1],
avg_logprob=avg_logprob / (idx - last_timestamp_start),
),
)
segments.append(casting_segment)
last_timestamp_start = idx
avg_logprob = 0
else:
avg_logprob += log_probs[idx - 1][token].logprob
return segments
async def _create_speech_to_text(
self,
audio_data: bytes,
request: SpeechToTextRequest,
raw_request: Request,
response_class: type[ResponseType],
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
) -> T | V | AsyncGenerator[str, None] | ErrorResponse:
"""Base method for speech-to-text operations like transcription and
translation."""
if request.stream and request.use_beam_search:
return self.create_error_response(
"Streaming is not currently supported with beam search"
)
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if not request.model:
request.model = self.models.model_name()
self._preflight()
if request.response_format not in [
"text",
"json",
"verbose_json",
"diarized_json",
]:
return self.create_error_response(
"Currently only support response_format: "
"`text`, `json`, `verbose_json` or `diarized_json`"
)
if (
request.response_format == "verbose_json"
and not self.model_cls.supports_segment_timestamp
):
return self.create_error_response(
f"Currently do not support verbose_json for {request.model}"
)
if (
request.response_format == "diarized_json"
and not self.model_cls.supports_diarized_transcription
):
return self.create_error_response(
f"Currently do not support diarized_json for {request.model}"
)
if (
request.response_format in {"verbose_json", "diarized_json"}
and request.stream
):
return self.create_error_response(
f"{request.response_format} format doesn't support streaming case"
)
request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
request_metadata = RequestResponseMetadata(request_id=request_id)
if raw_request:
raw_request.state.request_metadata = request_metadata
lora_request = self._maybe_get_adapters(request)
(
engine_inputs,
duration_s,
chunk_start_offsets,
) = await self._preprocess_speech_to_text(
request=request,
audio_data=audio_data,
request_id=request_id,
)
# Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
input_len = (
SpeechToTextBaseServing._get_decoder_prompt_len(engine_inputs)
if request.use_beam_search
else 0
)
# Unlike most decoder-only models, whisper generation length is not
# constrained by the size of the input audio, which is mapped to a
# fixed-size log-mel-spectogram. Still, allow for fewer tokens to be
# generated by respecting the extra completion tokens arg.
max_tokens = get_max_tokens(
max_model_len,
request.max_completion_tokens,
input_len,
self.default_sampling_params,
)
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
)
else:
sampling_params = request.to_sampling_params(
max_tokens,
self.default_sampling_params,
)
if request.response_format == "verbose_json":
sampling_params.logprobs = 1
engine_request_ids = [
request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}"
for idx in range(len(engine_inputs))
]
list_result_generator = []
try:
for request_id_item, engine_input in zip(engine_request_ids, engine_inputs):
self._log_inputs(
request_id_item,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_input,
params=sampling_params,
request_id=request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_input,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
list_result_generator.append(generator)
except asyncio.CancelledError:
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
separator = asr_inter_chunk_separator(
request.language, self.model_cls.no_space_languages
)
if request.stream:
return stream_generator_method(
request,
list_result_generator,
request_id,
request_metadata,
duration_s,
separator,
)
# Non-streaming response.
try:
assert list_result_generator is not None
chunk_segment_parts: list[list[SpeechToTextSegment]] = [
[] for _ in list_result_generator
]
chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator]
segments_types: dict[str, type[SpeechToTextSegment]] = {
"transcribe": TranscriptionSegment,
"translate": TranslationSegment,
}
segment_class: type[SpeechToTextSegment] = segments_types[self.task_type]
chunk_size_in_s = self.asr_config.max_audio_clip_s
if chunk_size_in_s is None:
assert len(list_result_generator) == 1, (
"`max_audio_clip_s` is set to None, audio cannot be chunked"
)
assert len(chunk_start_offsets) == len(list_result_generator)
result_generator = merge_async_iterators(*list_result_generator)
async for idx, op in result_generator:
start_time = chunk_start_offsets[idx]
if request.response_format == "verbose_json":
assert op.outputs[0].logprobs
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
tokens=tuple(op.outputs[0].token_ids),
segment_class=segment_class,
request=request,
start_time=start_time,
log_probs=op.outputs[0].logprobs,
)
chunk_segment_parts[idx].extend(segments)
chunk_text_parts[idx].extend([seg.text for seg in segments])
else:
raw_text = op.outputs[0].text
chunk_text_parts[idx].append(
self.model_cls.post_process_output(raw_text)
)
total_segments = [
segment
for segment_parts in chunk_segment_parts
for segment in segment_parts
]
text_parts = [text for text_part in chunk_text_parts for text in text_part]
text = separator.join(text_parts)
if self.task_type == "transcribe":
final_response: ResponseType
# add usage in TranscriptionResponse.
usage = {
"type": "duration",
# rounded up as per openAI specs
"seconds": int(math.ceil(duration_s)),
}
if request.response_format == "diarized_json":
diarized_segments = self.model_cls.parse_diarized_transcript(text)
if not diarized_segments:
return self.create_error_response(
"Model output did not contain a valid diarized transcript"
)
final_response = cast(
T,
TranscriptionResponseDiarized(
duration=duration_s,
text=separator.join(
segment.text for segment in diarized_segments
),
segments=[
{
"id": f"seg_{index}",
"start": segment.start,
"end": segment.end,
"text": segment.text,
"speaker": segment.speaker,
}
for index, segment in enumerate(diarized_segments)
],
usage=usage,
),
)
elif request.response_format != "verbose_json":
final_response = cast(
T, TranscriptionResponse(text=text, usage=usage)
)
else:
final_response = cast(
V,
TranscriptionResponseVerbose(
text=text,
language=request.language,
duration=duration_s,
segments=total_segments,
),
)
else:
# no usage in response for translation task
if request.response_format != "verbose_json":
final_response = cast(T, TranslationResponse(text=text))
else:
final_response = cast(
V,
TranslationResponseVerbose(
text=text,
language=request.language,
duration=duration_s,
segments=total_segments,
),
)
return final_response
except asyncio.CancelledError:
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
async def _speech_to_text_stream_generator(
self,
request: SpeechToTextRequest,
list_result_generator: list[AsyncGenerator[RequestOutput, None]],
request_id: str,
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
chunk_object_type: Literal["translation.chunk", "transcription.chunk"],
response_stream_choice_class: type[TranscriptionResponseStreamChoice]
| type[TranslationResponseStreamChoice],
stream_response_class: type[TranscriptionStreamResponse]
| type[TranslationStreamResponse],
separator: str,
) -> AsyncGenerator[str, None]:
created_time = int(time.time())
model_name = request.model
completion_tokens = 0
num_prompt_tokens = 0
include_usage = self.enable_force_include_usage or request.stream_include_usage
include_continuous_usage = (
request.stream_continuous_usage_stats
if include_usage and request.stream_continuous_usage_stats
else False
)
try:
for result_generator in list_result_generator:
beginning_of_chunk = True
post_processor = self.streaming_post_processor_cls()
async for res in result_generator:
# On first result.
if res.prompt_token_ids is not None:
num_prompt_tokens = len(res.prompt_token_ids)
if audio_tokens := self.model_cls.get_num_audio_tokens(
audio_duration_s, self.asr_config, self.model_config
):
num_prompt_tokens += audio_tokens
# We need to do it here, because if there are exceptions in
# the result_generator, it needs to be sent as the FIRST
# response (by the try...catch).
# Just one output (n=1) supported.
assert len(res.outputs) == 1
output = res.outputs[0]
output_text = post_processor.process_delta(
output.text, output.finish_reason is not None
)
# dont add separator to the first chunk
if (
result_generator is not list_result_generator[0]
and beginning_of_chunk
and output_text
):
output_text = separator + output_text
beginning_of_chunk = False
if output.finish_reason is None and not output_text:
completion_tokens += len(output.token_ids)
continue
delta_message = DeltaMessage(content=output_text)
completion_tokens += len(output.token_ids)
if output.finish_reason is None:
# Still generating, send delta update.
choice_data = response_stream_choice_class(delta=delta_message)
else:
# Model is finished generating.
choice_data = response_stream_choice_class(
delta=delta_message,
finish_reason=output.finish_reason,
stop_reason=output.stop_reason,
)
chunk = stream_response_class(
id=request_id,
object=chunk_object_type,
created=created_time,
choices=[choice_data],
model=model_name,
)
# handle usage stats if requested & if continuous
if include_continuous_usage:
chunk.usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
# Once the final token is handled, if stream_options.include_usage
# is sent, send the usage.
if include_usage:
final_usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
final_usage_chunk = stream_response_class(
id=request_id,
object=chunk_object_type,
created=created_time,
choices=[],
model=model_name,
usage=final_usage,
)
final_usage_data = final_usage_chunk.model_dump_json(
exclude_unset=True, exclude_none=True
)
yield f"data: {final_usage_data}\n\n"
# report to FastAPI middleware aggregate usage across all choices
request_metadata.final_usage_info = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
except Exception as e:
logger.exception("Error in %s stream generator.", self.task_type)
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
# Send the final done message after all response.n are finished
yield "data: [DONE]\n\n"
@@ -1,65 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared utilities for speech-to-text API routes."""
from fastapi import UploadFile
import vllm.envs as envs
from vllm.exceptions import VLLMValidationError
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
_READ_CHUNK_SIZE = 64 * KiB_bytes
async def read_upload_with_limit(
file: UploadFile,
max_size_mb: float | None = None,
) -> bytes:
"""Read an uploaded file enforcing a size limit *before* full
materialization.
The function first checks the Content-Length header (``file.size``) when
available. Regardless, it then performs a chunked read that stops as soon
as the accumulated bytes exceed the limit, ensuring that an oversized
upload never fully materializes in memory.
Args:
file: The FastAPI/Starlette ``UploadFile`` object.
max_size_mb: Maximum allowed compressed file size in megabytes.
Defaults to ``envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB``.
Returns:
The file content as ``bytes``.
Raises:
VLLMValidationError: If the file exceeds the configured size limit.
"""
if max_size_mb is None:
max_size_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
max_bytes = int(max_size_mb * MiB_bytes)
if file.size is not None and file.size > max_bytes:
raise VLLMValidationError(
"Maximum file size exceeded",
parameter="audio_filesize_mb",
value=file.size / MiB_bytes,
)
chunks: list[bytes] = []
total = 0
while True:
chunk = await file.read(_READ_CHUNK_SIZE)
if not chunk:
break
total += len(chunk)
if total > max_bytes:
raise VLLMValidationError(
"Maximum file size exceeded",
parameter="audio_filesize_mb",
value=total / MiB_bytes,
)
chunks.append(chunk)
return b"".join(chunks)
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from fastapi import FastAPI
if TYPE_CHECKING:
from argparse import Namespace
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.tasks import SupportedTask
else:
RequestLogger = object
def register_speech_to_text_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
if "realtime" in supported_tasks:
from .realtime.api_router import router as realtime_router
app.include_router(realtime_router)
if "transcription" in supported_tasks:
from .transcription.api_router import router as transcription_router
app.include_router(transcription_router)
from .translation.api_router import router as translation_router
app.include_router(translation_router)
def init_speech_to_text_state(
engine_client: "EngineClient",
state: "State",
args: "Namespace",
request_logger: RequestLogger | None,
supported_tasks: tuple["SupportedTask", ...],
):
if "transcription" in supported_tasks:
from .transcription.serving import OpenAIServingTranscription
state.openai_serving_transcription = OpenAIServingTranscription(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
enable_force_include_usage=args.enable_force_include_usage,
)
from .translation.serving import OpenAIServingTranslation
state.openai_serving_translation = OpenAIServingTranslation(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
enable_force_include_usage=args.enable_force_include_usage,
)
if "realtime" in supported_tasks:
from .realtime.serving import OpenAIServingRealtime
state.openai_serving_realtime = OpenAIServingRealtime(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
)
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import APIRouter, WebSocket
from vllm.logger import init_logger
from .connection import RealtimeConnection
logger = init_logger(__name__)
router = APIRouter()
@router.websocket("/v1/realtime")
async def realtime_endpoint(websocket: WebSocket):
"""WebSocket endpoint for realtime audio transcription.
Protocol:
1. Client connects to ws://host/v1/realtime
2. Server sends session.created event
3. Client optionally sends session.update with model/params
4. Client sends input_audio_buffer.commit when ready
5. Client sends input_audio_buffer.append events with base64 PCM16 chunks
6. Server processes and sends transcription.delta events
7. Server sends transcription.done with final text + usage
8. Repeat from step 5 for next utterance
9. Optionally, client sends input_audio_buffer.commit with final=True
to signal audio input is finished. Useful when streaming audio files
Audio format: PCM16, 16kHz, mono, base64-encoded
"""
app = websocket.app
serving = app.state.openai_serving_realtime
connection = RealtimeConnection(websocket, serving)
await connection.handle_connection()
@@ -1,314 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import contextlib
import json
from collections.abc import AsyncGenerator
from http import HTTPStatus
from uuid import uuid4
import numpy as np
import pybase64 as base64
from fastapi import WebSocket
from starlette.websockets import WebSocketDisconnect
from vllm import envs
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from .protocol import (
ErrorEvent,
InputAudioBufferAppend,
InputAudioBufferCommit,
SessionCreated,
TranscriptionDelta,
TranscriptionDone,
)
from .serving import OpenAIServingRealtime
logger = init_logger(__name__)
class RealtimeConnection:
"""Manages WebSocket lifecycle and state for realtime transcription.
This class handles:
- WebSocket connection lifecycle (accept, receive, send, close)
- Event routing (session.update, append, commit)
- Audio buffering via asyncio.Queue
- Generation task management
- Error handling and cleanup
"""
def __init__(self, websocket: WebSocket, serving: OpenAIServingRealtime):
self.websocket = websocket
self.connection_id = f"ws-{uuid4()}"
self.serving = serving
self.audio_queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
self.generation_task: asyncio.Task | None = None
self.request_id: str | None = None
self._is_connected = False
self._is_model_validated = False
self._max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
async def handle_connection(self):
"""Main connection loop."""
await self.websocket.accept()
logger.debug("WebSocket connection accepted: %s", self.connection_id)
self._is_connected = True
# Send session created event
await self.send(SessionCreated())
try:
while True:
message = await self.websocket.receive_text()
try:
event = json.loads(message)
await self.handle_event(event)
except json.JSONDecodeError:
await self.send_error("Invalid JSON", "invalid_json")
except Exception as e:
logger.exception("Error handling event: %s", e)
await self.send_error(sanitize_message(str(e)), "processing_error")
except WebSocketDisconnect:
logger.debug("WebSocket disconnected: %s", self.connection_id)
self._is_connected = False
except Exception as e:
logger.exception("Unexpected error in connection: %s", e)
finally:
await self.cleanup()
def _check_model(self, model: str | None) -> None | ErrorResponse:
if self.serving._is_model_supported(model):
return None
return self.serving.create_error_response(
message=f"The model `{model}` does not exist.",
err_type="NotFoundError",
status_code=HTTPStatus.NOT_FOUND,
param="model",
)
async def handle_event(self, event: dict):
"""Route events to handlers.
Supported event types:
- session.update: Configure model
- input_audio_buffer.append: Add audio chunk to queue
- input_audio_buffer.commit: Start transcription generation
"""
event_type = event.get("type")
if event_type == "session.update":
logger.debug("Session updated: %s", event)
model = event.get("model")
if model is None:
await self.send_error("Missing required field: model", "invalid_event")
return
err = self._check_model(model)
if err is not None:
await self.send_error(err.error.message, "model_not_found")
return
self._is_model_validated = True
elif event_type == "input_audio_buffer.append":
append_event = InputAudioBufferAppend(**event)
try:
audio_bytes = base64.b64decode(append_event.audio)
# Convert PCM16 bytes to float32 numpy array
audio_array = (
np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
/ 32768.0
)
if len(audio_array) / 1024**2 > self._max_audio_filesize_mb:
raise VLLMValidationError(
"Maximum file size exceeded",
parameter="audio_filesize_mb",
value=len(audio_array) / 1024**2,
)
if len(audio_array) == 0:
raise VLLMValidationError("Can't process empty audio.")
# Put audio chunk in queue
self.audio_queue.put_nowait(audio_array)
except Exception as e:
logger.error("Failed to decode audio: %s", e)
await self.send_error("Invalid audio data", "invalid_audio")
elif event_type == "input_audio_buffer.commit":
if not self._is_model_validated:
err_msg = (
"Model not validated. Make sure to validate the"
" model by sending a session.update event."
)
await self.send_error(
err_msg,
"model_not_validated",
)
return
commit_event = InputAudioBufferCommit(**event)
# final signals that the audio is finished
if commit_event.final:
self.audio_queue.put_nowait(None)
else:
await self.start_generation()
else:
await self.send_error(f"Unknown event type: {event_type}", "unknown_event")
async def audio_stream_generator(self) -> AsyncGenerator[np.ndarray, None]:
"""Generator that yields audio chunks from the queue."""
while True:
audio_chunk = await self.audio_queue.get()
if audio_chunk is None: # Sentinel value to stop
break
yield audio_chunk
async def start_generation(self):
"""Start the transcription generation task."""
if self.generation_task is not None and not self.generation_task.done():
logger.warning("Generation already in progress, ignoring commit")
return
# Create audio stream generator
audio_stream = self.audio_stream_generator()
input_stream = asyncio.Queue[list[int]]()
# Transform to StreamingInput generator
streaming_input_gen = self.serving.transcribe_realtime(
audio_stream, input_stream
)
# Start generation task
self.generation_task = asyncio.create_task(
self._run_generation(streaming_input_gen, input_stream)
)
async def _run_generation(
self,
streaming_input_gen: AsyncGenerator,
input_stream: asyncio.Queue[list[int]],
):
"""Run the generation and stream results back to the client.
This method:
1. Creates sampling parameters from session config
2. Passes the streaming input generator to engine.generate()
3. Streams transcription.delta events as text is generated
4. Sends final transcription.done event with usage stats
5. Feeds generated token IDs back to input_stream for next iteration
6. Cleans up the audio queue
"""
request_id = f"rt-{self.connection_id}-{uuid4()}"
self.request_id = request_id
full_text = ""
prompt_token_ids_len: int = 0
completion_tokens_len: int = 0
try:
# Create sampling params
from vllm.sampling_params import RequestOutputKind, SamplingParams
sampling_params = SamplingParams.from_optional(
temperature=0.0,
max_tokens=self.serving.model_cls.realtime_max_tokens,
output_kind=RequestOutputKind.DELTA,
skip_clone=True,
)
# Pass the streaming input generator to the engine
# The engine will consume audio chunks as they arrive and
# stream back transcription results incrementally
result_gen = self.serving.engine_client.generate(
prompt=streaming_input_gen,
sampling_params=sampling_params,
request_id=request_id,
)
# Stream results back to client as they're generated
async for output in result_gen:
if output.outputs and len(output.outputs) > 0:
if not prompt_token_ids_len and output.prompt_token_ids:
prompt_token_ids_len = len(output.prompt_token_ids)
delta = output.outputs[0].text
full_text += delta
# append output to input
input_stream.put_nowait(list(output.outputs[0].token_ids))
await self.send(TranscriptionDelta(delta=delta))
completion_tokens_len += len(output.outputs[0].token_ids)
if not self._is_connected:
# finish because websocket connection was killed
break
usage = UsageInfo(
prompt_tokens=prompt_token_ids_len,
completion_tokens=completion_tokens_len,
total_tokens=prompt_token_ids_len + completion_tokens_len,
)
# Send final completion event
await self.send(TranscriptionDone(text=full_text, usage=usage))
# Clear queue for next utterance
while not self.audio_queue.empty():
self.audio_queue.get_nowait()
except Exception as e:
logger.exception("Error in generation: %s", e)
# The socket may already be closed; nothing more we can do.
with contextlib.suppress(Exception):
await self.send_error(sanitize_message(str(e)), "processing_error")
async def send(
self, event: SessionCreated | TranscriptionDelta | TranscriptionDone
):
"""Send event to client."""
data = event.model_dump_json()
await self.websocket.send_text(data)
async def send_error(self, message: str, code: str | None = None):
"""Send error event to client."""
error_event = ErrorEvent(error=message, code=code)
await self.websocket.send_text(error_event.model_dump_json())
async def cleanup(self):
"""Cleanup resources."""
# Signal audio stream to stop
self.audio_queue.put_nowait(None)
# Abort the engine request. Cancelling the asyncio task alone does NOT
# abort the underlying engine request: a request parked in the
# VAD-waiting state keeps occupying a slot and generating into a now
# dead websocket (an orphaned request that never frees and starves
# other streams). Aborting also unblocks the generation task, which is
# otherwise stuck awaiting the next engine output.
if self.request_id is not None:
try:
await self.serving.engine_client.abort(self.request_id)
except Exception:
logger.exception("Failed to abort request %s", self.request_id)
self.request_id = None
# Cancel the generation task if still running, and await it so the
# cancellation (and any generator cleanup) actually completes.
if self.generation_task and not self.generation_task.done():
self.generation_task.cancel()
try:
await self.generation_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Error awaiting cancelled generation task")
logger.debug("Connection cleanup complete: %s", self.connection_id)
@@ -1,65 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Literal
from pydantic import Field
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.utils import random_uuid
# Client -> Server Events
class InputAudioBufferAppend(OpenAIBaseModel):
"""Append audio chunk to buffer."""
type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append"
audio: str # base64-encoded PCM16 @ 16kHz
class InputAudioBufferCommit(OpenAIBaseModel):
"""Process accumulated audio buffer."""
type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit"
final: bool = False
# Server -> Client Events
class SessionUpdate(OpenAIBaseModel):
"""Configure session parameters."""
type: Literal["session.update"] = "session.update"
model: str | None = None
class SessionCreated(OpenAIBaseModel):
"""Connection established notification."""
type: Literal["session.created"] = "session.created"
id: str = Field(default_factory=lambda: f"sess-{random_uuid()}")
created: int = Field(default_factory=lambda: int(time.time()))
class TranscriptionDelta(OpenAIBaseModel):
"""Incremental transcription text."""
type: Literal["transcription.delta"] = "transcription.delta"
delta: str # Incremental text
class TranscriptionDone(OpenAIBaseModel):
"""Final transcription with usage stats."""
type: Literal["transcription.done"] = "transcription.done"
text: str # Complete transcription
usage: UsageInfo | None = None
class ErrorEvent(OpenAIBaseModel):
"""Error notification."""
type: Literal["error"] = "error"
error: str
code: str | None = None
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from collections.abc import AsyncGenerator
from functools import cached_property
from typing import Literal, cast
import numpy as np
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import PromptType
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsRealtime
from vllm.renderers.inputs.preprocess import parse_model_prompt
logger = init_logger(__name__)
class OpenAIServingRealtime(GenerateBaseServing):
"""Realtime audio transcription service via WebSocket streaming.
Provides streaming audio-to-text transcription by transforming audio chunks
into StreamingInput objects that can be consumed by the engine.
"""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
)
self.task_type: Literal["realtime"] = "realtime"
logger.info("OpenAIServingRealtime initialized for task: %s", self.task_type)
@cached_property
def model_cls(self) -> type[SupportsRealtime]:
"""Get the model class that supports transcription."""
from vllm.model_executor.model_loader import get_model_cls
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsRealtime], model_cls)
async def transcribe_realtime(
self,
audio_stream: AsyncGenerator[np.ndarray, None],
input_stream: asyncio.Queue[list[int]],
) -> AsyncGenerator[StreamingInput, None]:
"""Transform audio stream into StreamingInput for engine.generate().
Args:
audio_stream: Async generator yielding float32 numpy audio arrays
input_stream: Queue containing context token IDs from previous
generation outputs. Used for autoregressive multi-turn
processing where each generation's output becomes the context
for the next iteration.
Yields:
StreamingInput objects containing audio prompts for the engine
"""
model_config = self.model_config
renderer = self.renderer
# mypy is being stupid
# TODO(Patrick) - fix this
stream_input_iter = cast(
AsyncGenerator[PromptType, None],
self.model_cls.buffer_realtime_audio(
audio_stream, input_stream, model_config
),
)
async for prompt in stream_input_iter:
parsed_prompt = parse_model_prompt(model_config, prompt)
(engine_input,) = await renderer.render_cmpl_async([parsed_prompt])
yield StreamingInput(prompt=engine_input)
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from typing import Annotated
from fastapi import APIRouter, Form, Request
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
with_cancellation,
)
from vllm.entrypoints.speech_to_text.base.utils import read_upload_with_limit
from vllm.logger import init_logger
from ...serve.engine.protocol import ErrorResponse
from .protocol import TranscriptionRequest, TranscriptionResponseVariant
from .serving import OpenAIServingTranscription
logger = init_logger(__name__)
router = APIRouter()
def transcription(request: Request) -> OpenAIServingTranscription:
return request.app.state.openai_serving_transcription
@router.post(
"/v1/audio/transcriptions",
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_transcriptions(
raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
):
handler = transcription(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Transcriptions API")
audio_data = await read_upload_with_limit(request.file)
generator = await handler.create_transcription(audio_data, request, raw_request)
if isinstance(generator, ErrorResponse):
return JSONResponse(
content=generator.model_dump(), status_code=generator.error.code
)
elif isinstance(generator, TranscriptionResponseVariant):
return JSONResponse(content=generator.model_dump())
return StreamingResponse(content=generator, media_type="text/event-stream")
@@ -1,430 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import time
from http import HTTPStatus
from typing import TYPE_CHECKING, Literal, TypeAlias
from fastapi import HTTPException, UploadFile
from pydantic import (
Field,
model_validator,
)
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import (
BeamSearchParams,
RequestOutputKind,
SamplingParams,
)
from vllm.utils import random_uuid
from ..base.protocol import _LONG_INFO, TranscriptionResponseFormat
if TYPE_CHECKING:
import numpy as np
from vllm.config import ModelConfig, SpeechToTextConfig
logger = init_logger(__name__)
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
delta: DeltaMessage
finish_reason: str | None = None
stop_reason: int | str | None = None
class TranscriptionStreamResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
object: Literal["transcription.chunk"] = "transcription.chunk"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
choices: list[TranscriptionResponseStreamChoice]
usage: UsageInfo | None = Field(default=None)
class TranscriptionRequest(OpenAIBaseModel):
# Ordered by official OpenAI API documentation
# https://platform.openai.com/docs/api-reference/audio/createTranscription
file: UploadFile
"""
The audio file object (not file name) to transcribe, in one of these
formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
"""
model: str | None = None
"""ID of the model to use.
"""
language: str | None = None
"""The language of the input audio.
Supplying the input language in
[ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
will improve accuracy and latency.
"""
hotwords: str | None = None
"""
hotwords refers to a list of important words or phrases that the model
should pay extra attention to during transcription.
"""
prompt: str = Field(default="")
"""An optional text to guide the model's style or continue a previous audio
segment.
The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
should match the audio language.
"""
response_format: TranscriptionResponseFormat = Field(default="json")
"""
The format of the output, in one of these options: `json`, `text`, `srt`,
`verbose_json`, or `vtt`.
"""
## TODO (varun) : Support if set to 0, certain thresholds are met !!
timestamp_granularities: list[Literal["word", "segment"]] = Field(
alias="timestamp_granularities[]", default=[]
)
"""The timestamp granularities to populate for this transcription.
`response_format` must be set `verbose_json` to use timestamp granularities.
Either or both of these options are supported: `word`, or `segment`. Note:
There is no additional latency for segment timestamps, but generating word
timestamps incurs additional latency.
"""
stream: bool | None = False
"""When set, it will enable output to be streamed in a similar fashion
as the Chat Completion endpoint.
"""
# --8<-- [start:transcription-extra-params]
# Flattened stream option to simplify form data.
stream_include_usage: bool | None = False
stream_continuous_usage_stats: bool | None = False
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
default=None,
description=(
"Additional request parameters with (list of) string or "
"numeric values, used by custom extensions."
),
)
# --8<-- [end:transcription-extra-params]
to_language: str | None = None
"""The language of the output audio we transcribe to.
Please note that this is not currently used by supported models at this
time, but it is a placeholder for future use, matching translation api.
"""
# --8<-- [start:transcription-sampling-params]
use_beam_search: bool = False
"""Whether or not beam search should be used."""
n: int = 1
"""The number of beams to be used in beam search."""
length_penalty: float = 1.0
"""Length penalty to be used for beam search."""
include_stop_str_in_output: bool = False
"""Whether to include the stop strings in output text."""
temperature: float = Field(default=0.0)
"""The sampling temperature, between 0 and 1.
Higher values like 0.8 will make the output more random, while lower values
like 0.2 will make it more focused / deterministic. If set to 0, the model
will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
to automatically increase the temperature until certain thresholds are hit.
"""
watermarking: bool | None = None
"""Whether to apply the engine's configured watermark to this request."""
top_p: float | None = None
"""Enables nucleus (top-p) sampling, where tokens are selected from the
smallest possible set whose cumulative probability exceeds `p`.
"""
top_k: int | None = None
"""Limits sampling to the `k` most probable tokens at each step."""
min_p: float | None = None
"""Filters out tokens with a probability lower than `min_p`, ensuring a
minimum likelihood threshold during sampling.
"""
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
"""The seed to use for sampling."""
frequency_penalty: float | None = 0.0
"""The frequency penalty to use for sampling."""
repetition_penalty: float | None = None
"""The repetition penalty to use for sampling."""
presence_penalty: float | None = 0.0
"""The presence penalty to use for sampling."""
max_completion_tokens: int | None = None
"""The maximum number of tokens to generate."""
# --8<-- [end:transcription-sampling-params]
# Default sampling parameters for transcription requests.
_DEFAULT_SAMPLING_PARAMS: dict = {
"repetition_penalty": 1.0,
"temperature": 1.0,
"top_p": 1.0,
"top_k": 0,
"min_p": 0.0,
}
def build_stt_params(
self,
audio: "np.ndarray",
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
task_type: str,
) -> SpeechToTextParams:
return SpeechToTextParams(
audio=audio,
stt_config=stt_config,
model_config=model_config,
language=self.language,
task_type=task_type,
request_prompt=self.prompt,
to_language=self.to_language,
hotwords=self.hotwords,
)
def to_beam_search_params(
self,
default_max_tokens: int,
default_sampling_params: dict | None = None,
) -> BeamSearchParams:
if default_sampling_params is None:
default_sampling_params = {}
max_tokens = default_max_tokens
n = self.n if self.n is not None else 1
# NOTE: Temp 0 is a different fallback than completions
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get("temperature", 0)
return BeamSearchParams(
beam_width=n,
max_tokens=max_tokens,
temperature=temperature,
watermarking=self.watermarking,
length_penalty=self.length_penalty,
include_stop_str_in_output=self.include_stop_str_in_output,
)
def to_sampling_params(
self, default_max_tokens: int, default_sampling_params: dict | None = None
) -> SamplingParams:
max_tokens = default_max_tokens
if default_sampling_params is None:
default_sampling_params = {}
# Default parameters
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get(
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
)
if (top_p := self.top_p) is None:
top_p = default_sampling_params.get(
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
)
if (top_k := self.top_k) is None:
top_k = default_sampling_params.get(
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
)
if (min_p := self.min_p) is None:
min_p = default_sampling_params.get(
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
)
if (repetition_penalty := self.repetition_penalty) is None:
repetition_penalty = default_sampling_params.get(
"repetition_penalty",
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
)
return SamplingParams.from_optional(
temperature=temperature,
watermarking=self.watermarking,
max_tokens=max_tokens,
seed=self.seed,
top_p=top_p,
top_k=top_k,
min_p=min_p,
frequency_penalty=self.frequency_penalty,
repetition_penalty=repetition_penalty,
presence_penalty=self.presence_penalty,
output_kind=RequestOutputKind.DELTA
if self.stream
else RequestOutputKind.FINAL_ONLY,
extra_args=self.vllm_xargs,
skip_clone=True, # Created fresh per request, safe to skip clone
)
@model_validator(mode="before")
@classmethod
def validate_transcription_request(cls, data):
if not isinstance(data, dict):
return data
if isinstance(data.get("file"), str):
raise HTTPException(
status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
detail="Expected 'file' to be a file-like object, not 'str'.",
)
stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
stream = data.get("stream", False)
if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
# Find which specific stream option was set
invalid_param = next(
(so for so in stream_opts if data.get(so, False)),
"stream_include_usage",
)
raise VLLMValidationError(
"Stream options can only be defined when `stream=True`.",
parameter=invalid_param,
)
# Parse vllm_xargs from JSON string (form data sends it as a string)
xargs = data.get("vllm_xargs")
if isinstance(xargs, str):
try:
data["vllm_xargs"] = json.loads(xargs)
except json.JSONDecodeError as e:
raise VLLMValidationError(
f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
parameter="vllm_xargs",
) from e
return data
# Transcription response objects
class TranscriptionUsageAudio(OpenAIBaseModel):
type: Literal["duration"] = "duration"
seconds: int
class TranscriptionResponse(OpenAIBaseModel):
text: str
"""The transcribed text."""
usage: TranscriptionUsageAudio
class TranscriptionWord(OpenAIBaseModel):
end: float
"""End time of the word in seconds."""
start: float
"""Start time of the word in seconds."""
word: str
"""The text content of the word."""
class TranscriptionSegment(OpenAIBaseModel):
id: int
"""Unique identifier of the segment."""
avg_logprob: float
"""Average logprob of the segment.
If the value is lower than -1, consider the logprobs failed.
"""
compression_ratio: float
"""Compression ratio of the segment.
If the value is greater than 2.4, consider the compression failed.
"""
end: float
"""End time of the segment in seconds."""
no_speech_prob: float | None = None
"""Probability of no speech in the segment.
If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
this segment silent.
"""
seek: int
"""Seek offset of the segment."""
start: float
"""Start time of the segment in seconds."""
temperature: float
"""Temperature parameter used for generating the segment."""
text: str
"""Text content of the segment."""
tokens: list[int]
"""Array of token IDs for the text content."""
class TranscriptionResponseVerbose(OpenAIBaseModel):
duration: float
"""The duration of the input audio."""
language: str
"""The language of the input audio."""
text: str
"""The transcribed text."""
segments: list[TranscriptionSegment] | None = None
"""Segments of the transcribed text and their corresponding details."""
words: list[TranscriptionWord] | None = None
"""Extracted words and their corresponding timestamps."""
class TranscriptionDiarizedSegment(OpenAIBaseModel):
"""A speaker-attributed transcription segment."""
type: Literal["transcript.text.segment"] = "transcript.text.segment"
id: str
start: float
end: float
text: str
speaker: str
class TranscriptionResponseDiarized(OpenAIBaseModel):
"""OpenAI-compatible diarized transcription response."""
task: Literal["transcribe"] = "transcribe"
duration: float
text: str
segments: list[TranscriptionDiarizedSegment]
usage: TranscriptionUsageAudio
TranscriptionResponseVariant: TypeAlias = (
TranscriptionResponse | TranscriptionResponseVerbose | TranscriptionResponseDiarized
)
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncGenerator
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from ...serve.engine.protocol import ErrorResponse
from ..base.serving import SpeechToTextBaseServing
from .protocol import (
TranscriptionRequest,
TranscriptionResponse,
TranscriptionResponseDiarized,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionStreamResponse,
)
logger = init_logger(__name__)
class OpenAIServingTranscription(SpeechToTextBaseServing):
"""Handles transcription requests."""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_force_include_usage: bool = False,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
task_type="transcribe",
enable_force_include_usage=enable_force_include_usage,
)
async def create_transcription(
self,
audio_data: bytes,
request: TranscriptionRequest,
raw_request: Request | None = None,
) -> (
TranscriptionResponse
| TranscriptionResponseVerbose
| TranscriptionResponseDiarized
| AsyncGenerator[str, None]
| ErrorResponse
):
"""Transcription API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranscription
for the API specification. This API mimics the OpenAI transcription API.
"""
return await self._create_speech_to_text(
audio_data=audio_data,
request=request,
raw_request=raw_request,
response_class=(
TranscriptionResponseVerbose
if request.response_format == "verbose_json"
else TranscriptionResponseDiarized
if request.response_format == "diarized_json"
else TranscriptionResponse
),
stream_generator_method=self.transcription_stream_generator,
)
async def transcription_stream_generator(
self,
request: TranscriptionRequest,
result_generator: list[AsyncGenerator[RequestOutput, None]],
request_id: str,
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
separator: str,
) -> AsyncGenerator[str, None]:
generator = self._speech_to_text_stream_generator(
request=request,
list_result_generator=result_generator,
request_id=request_id,
request_metadata=request_metadata,
audio_duration_s=audio_duration_s,
chunk_object_type="transcription.chunk",
response_stream_choice_class=TranscriptionResponseStreamChoice,
stream_response_class=TranscriptionStreamResponse,
separator=separator,
)
async for chunk in generator:
yield chunk
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from typing import Annotated
from fastapi import APIRouter, Form, Request
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
with_cancellation,
)
from vllm.entrypoints.speech_to_text.base.utils import read_upload_with_limit
from vllm.logger import init_logger
from .protocol import TranslationRequest, TranslationResponseVariant
from .serving import OpenAIServingTranslation
logger = init_logger(__name__)
router = APIRouter()
def translation(request: Request) -> OpenAIServingTranslation:
return request.app.state.openai_serving_translation
@router.post(
"/v1/audio/translations",
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_translations(
request: Annotated[TranslationRequest, Form()], raw_request: Request
):
handler = translation(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Translations API")
audio_data = await read_upload_with_limit(request.file)
generator = await handler.create_translation(audio_data, request, raw_request)
if isinstance(generator, ErrorResponse):
return JSONResponse(
content=generator.model_dump(), status_code=generator.error.code
)
elif isinstance(generator, TranslationResponseVariant):
return JSONResponse(content=generator.model_dump())
return StreamingResponse(content=generator, media_type="text/event-stream")
@@ -1,386 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import time
from http import HTTPStatus
from typing import TYPE_CHECKING, Literal, TypeAlias
from fastapi import HTTPException, UploadFile
from pydantic import (
Field,
model_validator,
)
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import (
BeamSearchParams,
RequestOutputKind,
SamplingParams,
)
from vllm.utils import random_uuid
from ..base.protocol import _LONG_INFO, AudioResponseFormat
if TYPE_CHECKING:
import numpy as np
from vllm.config import ModelConfig, SpeechToTextConfig
logger = init_logger(__name__)
class TranslationResponseStreamChoice(OpenAIBaseModel):
delta: DeltaMessage
finish_reason: str | None = None
stop_reason: int | str | None = None
class TranslationStreamResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
object: Literal["translation.chunk"] = "translation.chunk"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
choices: list[TranslationResponseStreamChoice]
usage: UsageInfo | None = Field(default=None)
class TranslationRequest(OpenAIBaseModel):
# Ordered by official OpenAI API documentation
# https://platform.openai.com/docs/api-reference/audio/createTranslation
file: UploadFile
"""
The audio file object (not file name) to translate, in one of these
formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
"""
model: str | None = None
"""ID of the model to use.
"""
prompt: str = Field(default="")
"""An optional text to guide the model's style or continue a previous audio
segment.
The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
should match the audio language.
"""
response_format: AudioResponseFormat = Field(default="json")
"""
The format of the output, in one of these options: `json`, `text`, `srt`,
`verbose_json`, or `vtt`.
"""
# --8<-- [start:translation-sampling-params]
use_beam_search: bool = False
"""Whether or not beam search should be used."""
n: int = 1
"""The number of beams to be used in beam search."""
length_penalty: float = 1.0
"""Length penalty to be used for beam search."""
include_stop_str_in_output: bool = False
"""Whether to include the stop strings in output text."""
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
"""The seed to use for sampling."""
temperature: float = Field(default=0.0)
"""The sampling temperature, between 0 and 1.
Higher values like 0.8 will make the output more random, while lower values
like 0.2 will make it more focused / deterministic. If set to 0, the model
will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
to automatically increase the temperature until certain thresholds are hit.
"""
watermarking: bool | None = None
"""Whether to apply the engine's configured watermark to this request."""
top_p: float | None = None
"""Enables nucleus (top-p) sampling, where tokens are selected from the
smallest possible set whose cumulative probability exceeds `p`.
"""
top_k: int | None = None
"""Limits sampling to the `k` most probable tokens at each step."""
min_p: float | None = None
"""Filters out tokens with a probability lower than `min_p`, ensuring a
minimum likelihood threshold during sampling.
"""
frequency_penalty: float | None = 0.0
"""The frequency penalty to use for sampling."""
repetition_penalty: float | None = None
"""The repetition penalty to use for sampling."""
presence_penalty: float | None = 0.0
"""The presence penalty to use for sampling."""
# --8<-- [end:translation-sampling-params]
# --8<-- [start:translation-extra-params]
language: str | None = None
"""The language of the input audio we translate from.
Supplying the input language in
[ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
will improve accuracy.
"""
hotwords: str | None = None
"""
hotwords refers to a list of important words or phrases that the model
should pay extra attention to during transcription.
"""
to_language: str | None = None
"""The language of the input audio we translate to.
Please note that this is not supported by all models, refer to the specific
model documentation for more details.
For instance, Whisper only supports `to_language=en`.
"""
stream: bool | None = False
"""Custom field not present in the original OpenAI definition. When set,
it will enable output to be streamed in a similar fashion as the Chat
Completion endpoint.
"""
# Flattened stream option to simplify form data.
stream_include_usage: bool | None = False
stream_continuous_usage_stats: bool | None = False
max_completion_tokens: int | None = None
"""The maximum number of tokens to generate."""
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
default=None,
description=(
"Additional request parameters with (list of) string or "
"numeric values, used by custom extensions."
),
)
# --8<-- [end:translation-extra-params]
# Default sampling parameters for translation requests.
_DEFAULT_SAMPLING_PARAMS: dict = {
"repetition_penalty": 1.0,
"temperature": 0,
"top_p": 1.0,
"top_k": 0,
"min_p": 0.0,
}
def build_stt_params(
self,
audio: "np.ndarray",
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
task_type: str,
) -> SpeechToTextParams:
return SpeechToTextParams(
audio=audio,
stt_config=stt_config,
model_config=model_config,
language=self.language,
task_type=task_type,
request_prompt=self.prompt,
to_language=self.to_language,
hotwords=self.hotwords,
)
def to_beam_search_params(
self,
default_max_tokens: int,
default_sampling_params: dict | None = None,
) -> BeamSearchParams:
if default_sampling_params is None:
default_sampling_params = {}
max_tokens = default_max_tokens
n = self.n if self.n is not None else 1
# NOTE: Temp 0 is a different fallback than completions
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get("temperature", 0)
return BeamSearchParams(
beam_width=n,
max_tokens=max_tokens,
temperature=temperature,
watermarking=self.watermarking,
length_penalty=self.length_penalty,
include_stop_str_in_output=self.include_stop_str_in_output,
)
def to_sampling_params(
self, default_max_tokens: int, default_sampling_params: dict | None = None
) -> SamplingParams:
max_tokens = default_max_tokens
if default_sampling_params is None:
default_sampling_params = {}
# Default parameters
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get(
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
)
if (top_p := self.top_p) is None:
top_p = default_sampling_params.get(
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
)
if (top_k := self.top_k) is None:
top_k = default_sampling_params.get(
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
)
if (min_p := self.min_p) is None:
min_p = default_sampling_params.get(
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
)
if (repetition_penalty := self.repetition_penalty) is None:
repetition_penalty = default_sampling_params.get(
"repetition_penalty",
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
)
return SamplingParams.from_optional(
temperature=temperature,
watermarking=self.watermarking,
max_tokens=max_tokens,
seed=self.seed,
top_p=top_p,
top_k=top_k,
min_p=min_p,
frequency_penalty=self.frequency_penalty,
repetition_penalty=repetition_penalty,
presence_penalty=self.presence_penalty,
output_kind=RequestOutputKind.DELTA
if self.stream
else RequestOutputKind.FINAL_ONLY,
extra_args=self.vllm_xargs,
skip_clone=True, # Created fresh per request, safe to skip clone
)
@model_validator(mode="before")
@classmethod
def validate_stream_options(cls, data):
if not isinstance(data, dict):
return data
if isinstance(data.get("file"), str):
raise HTTPException(
status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
detail="Expected 'file' to be a file-like object, not 'str'.",
)
stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
stream = data.get("stream", False)
if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
# Find which specific stream option was set
invalid_param = next(
(so for so in stream_opts if data.get(so, False)),
"stream_include_usage",
)
raise VLLMValidationError(
"Stream options can only be defined when `stream=True`.",
parameter=invalid_param,
)
xargs = data.get("vllm_xargs")
if isinstance(xargs, str):
try:
data["vllm_xargs"] = json.loads(xargs)
except json.JSONDecodeError as e:
raise VLLMValidationError(
f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
parameter="vllm_xargs",
) from e
return data
# Translation response objects
class TranslationResponse(OpenAIBaseModel):
text: str
"""The translated text."""
class TranslationWord(OpenAIBaseModel):
end: float
"""End time of the word in seconds."""
start: float
"""Start time of the word in seconds."""
word: str
"""The text content of the word."""
class TranslationSegment(OpenAIBaseModel):
id: int
"""Unique identifier of the segment."""
avg_logprob: float
"""Average logprob of the segment.
If the value is lower than -1, consider the logprobs failed.
"""
compression_ratio: float
"""Compression ratio of the segment.
If the value is greater than 2.4, consider the compression failed.
"""
end: float
"""End time of the segment in seconds."""
no_speech_prob: float | None = None
"""Probability of no speech in the segment.
If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
this segment silent.
"""
seek: int
"""Seek offset of the segment."""
start: float
"""Start time of the segment in seconds."""
temperature: float
"""Temperature parameter used for generating the segment."""
text: str
"""Text content of the segment."""
tokens: list[int]
"""Array of token IDs for the text content."""
class TranslationResponseVerbose(OpenAIBaseModel):
duration: float
"""The duration of the input audio."""
language: str
"""The language of the input audio."""
text: str
"""The translated text."""
segments: list[TranslationSegment] | None = None
"""Segments of the translated text and their corresponding details."""
words: list[TranslationWord] | None = None
"""Extracted words and their corresponding timestamps."""
TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose
@@ -1,97 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncGenerator
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from ..base.serving import SpeechToTextBaseServing
from .protocol import (
TranslationRequest,
TranslationResponse,
TranslationResponseStreamChoice,
TranslationResponseVerbose,
TranslationStreamResponse,
)
logger = init_logger(__name__)
class OpenAIServingTranslation(SpeechToTextBaseServing):
"""Handles translation requests."""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_force_include_usage: bool = False,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
task_type="translate",
enable_force_include_usage=enable_force_include_usage,
)
async def create_translation(
self,
audio_data: bytes,
request: TranslationRequest,
raw_request: Request | None = None,
) -> (
TranslationResponse
| TranslationResponseVerbose
| AsyncGenerator[str, None]
| ErrorResponse
):
"""Translation API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranslation
for the API specification. This API mimics the OpenAI translation API.
"""
return await self._create_speech_to_text(
audio_data=audio_data,
request=request,
raw_request=raw_request,
response_class=(
TranslationResponseVerbose
if request.response_format == "verbose_json"
else TranslationResponse
),
stream_generator_method=self.translation_stream_generator,
)
async def translation_stream_generator(
self,
request: TranslationRequest,
result_generator: list[AsyncGenerator[RequestOutput, None]],
request_id: str,
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
separator: str,
) -> AsyncGenerator[str, None]:
generator = self._speech_to_text_stream_generator(
request=request,
list_result_generator=result_generator,
request_id=request_id,
request_metadata=request_metadata,
audio_duration_s=audio_duration_s,
chunk_object_type="translation.chunk",
response_stream_choice_class=TranslationResponseStreamChoice,
stream_response_class=TranslationStreamResponse,
separator=separator,
)
async for chunk in generator:
yield chunk