Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+15 -15
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pydantic models for Anthropic API protocol"""
"""Pydantic models for Anthropic API protocol."""
import time
from typing import Annotated, Any, Literal
@@ -11,21 +11,21 @@ import vllm.envs as envs
class AnthropicError(BaseModel):
"""Error structure for Anthropic API"""
"""Error structure for Anthropic API."""
type: str
message: str
class AnthropicErrorResponse(BaseModel):
"""Error response structure for Anthropic API"""
"""Error response structure for Anthropic API."""
type: Literal["error"] = "error"
error: AnthropicError
class AnthropicUsage(BaseModel):
"""Token usage information"""
"""Token usage information."""
input_tokens: int
output_tokens: int
@@ -34,7 +34,7 @@ class AnthropicUsage(BaseModel):
class AnthropicContentBlock(BaseModel):
"""Content block in message"""
"""Content block in message."""
type: Literal[
"text",
@@ -65,14 +65,14 @@ class AnthropicContentBlock(BaseModel):
class AnthropicMessage(BaseModel):
"""Message structure"""
"""Message structure."""
role: Literal["user", "assistant", "system"]
content: str | list[AnthropicContentBlock]
class AnthropicTool(BaseModel):
"""Tool definition"""
"""Tool definition."""
name: str
description: str | None = None
@@ -91,7 +91,7 @@ class AnthropicTool(BaseModel):
class AnthropicToolChoice(BaseModel):
"""Tool Choice definition"""
"""Tool Choice definition."""
type: Literal["auto", "any", "tool", "none"]
name: str | None = None
@@ -105,7 +105,7 @@ class AnthropicToolChoice(BaseModel):
class AnthropicJsonOutputFormat(BaseModel):
"""JSON output format configuration"""
"""JSON output format configuration."""
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
type: Literal["json_schema"] = "json_schema"
@@ -119,7 +119,7 @@ class AnthropicOutputConfig(BaseModel):
class AnthropicMessagesRequest(BaseModel):
"""Anthropic Messages API request"""
"""Anthropic Messages API request."""
model: str
messages: list[AnthropicMessage]
@@ -192,7 +192,7 @@ class AnthropicMessagesRequest(BaseModel):
class AnthropicDelta(BaseModel):
"""Delta for streaming responses"""
"""Delta for streaming responses."""
type: (
Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
@@ -211,7 +211,7 @@ class AnthropicDelta(BaseModel):
class AnthropicStreamEvent(BaseModel):
"""Streaming event"""
"""Streaming event."""
type: Literal[
"message_start",
@@ -232,7 +232,7 @@ class AnthropicStreamEvent(BaseModel):
class AnthropicMessagesResponse(BaseModel):
"""Anthropic Messages API response"""
"""Anthropic Messages API response."""
id: str
type: Literal["message"] = "message"
@@ -265,7 +265,7 @@ class AnthropicContextManagement(BaseModel):
class AnthropicCountTokensRequest(BaseModel):
"""Anthropic messages.count_tokens request"""
"""Anthropic messages.count_tokens request."""
model: str
messages: list[AnthropicMessage]
@@ -291,7 +291,7 @@ class AnthropicCountTokensRequest(BaseModel):
class AnthropicCountTokensResponse(BaseModel):
"""Anthropic messages.count_tokens response"""
"""Anthropic messages.count_tokens response."""
input_tokens: int
context_management: AnthropicContextManagement | None = None
+16 -17
View File
@@ -3,7 +3,7 @@
# Adapted from
# https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/chat_completion/serving.py
"""Anthropic Messages API serving handler"""
"""Anthropic Messages API serving handler."""
import json
import logging
@@ -96,7 +96,7 @@ def wrap_data_with_event(data: str, event: str):
class AnthropicServingMessages(OpenAIServingChat):
"""Handler for Anthropic Messages API requests"""
"""Handler for Anthropic Messages API requests."""
def __init__(
self,
@@ -196,7 +196,7 @@ class AnthropicServingMessages(OpenAIServingChat):
*,
merge_inline_system: bool = False,
) -> ChatCompletionRequest:
"""Convert Anthropic message format to OpenAI format"""
"""Convert Anthropic message format to OpenAI format."""
openai_messages: list[dict[str, Any]] = []
cls._convert_system_message(
@@ -224,7 +224,7 @@ class AnthropicServingMessages(OpenAIServingChat):
*,
merge_inline_system: bool = False,
) -> None:
"""Convert Anthropic system message to OpenAI format"""
"""Convert Anthropic system message to OpenAI format."""
system_parts: list[str] = []
# Top-level system field
@@ -278,7 +278,7 @@ class AnthropicServingMessages(OpenAIServingChat):
*,
merge_inline_system: bool = False,
) -> None:
"""Convert Anthropic messages to OpenAI format"""
"""Convert Anthropic messages to OpenAI format."""
for msg in messages:
# Handle system messages in-place: extract text, strip billing
# headers, and only emit if there is real content. This avoids
@@ -310,7 +310,7 @@ class AnthropicServingMessages(OpenAIServingChat):
openai_msg: dict[str, Any],
openai_messages: list[dict[str, Any]],
) -> None:
"""Convert complex message content blocks"""
"""Convert complex message content blocks."""
content_parts: list[dict[str, Any]] = []
tool_calls: list[dict[str, Any]] = []
reasoning_parts: list[str] = []
@@ -349,7 +349,7 @@ class AnthropicServingMessages(OpenAIServingChat):
reasoning_parts: list[str],
openai_messages: list[dict[str, Any]],
) -> None:
"""Convert individual content block"""
"""Convert individual content block."""
if block.type == "text" and block.text:
content_parts.append({"type": "text", "text": block.text})
elif block.type == "image" and block.source:
@@ -374,7 +374,7 @@ class AnthropicServingMessages(OpenAIServingChat):
@classmethod
def _convert_tool_use_block(cls, block, tool_calls: list[dict[str, Any]]) -> None:
"""Convert tool_use block to OpenAI function call format"""
"""Convert tool_use block to OpenAI function call format."""
tool_call = {
"id": block.id or f"call_{int(time.time())}",
"type": "function",
@@ -393,7 +393,7 @@ class AnthropicServingMessages(OpenAIServingChat):
openai_messages: list[dict[str, Any]],
content_parts: list[dict[str, Any]],
) -> None:
"""Convert tool_result block to OpenAI format"""
"""Convert tool_result block to OpenAI format."""
if role == "user":
cls._convert_user_tool_result(block, openai_messages)
else:
@@ -406,7 +406,7 @@ class AnthropicServingMessages(OpenAIServingChat):
def _convert_user_tool_result(
cls, block, openai_messages: list[dict[str, Any]]
) -> None:
"""Convert user tool_result with text and image support"""
"""Convert user tool_result with text and image support."""
tool_text = ""
tool_image_urls: list[str] = []
tool_reference: list[dict[str, Any]] = []
@@ -468,7 +468,7 @@ class AnthropicServingMessages(OpenAIServingChat):
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
openai_messages: list[dict[str, Any]],
) -> ChatCompletionRequest:
"""Build base ChatCompletionRequest"""
"""Build base ChatCompletionRequest."""
if isinstance(anthropic_request, AnthropicCountTokensRequest):
return ChatCompletionRequest(
model=anthropic_request.model,
@@ -498,7 +498,7 @@ class AnthropicServingMessages(OpenAIServingChat):
req: ChatCompletionRequest,
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
) -> None:
"""Handle output configuration such as output format and effort"""
"""Handle output configuration such as output format and effort."""
if isinstance(anthropic_request, AnthropicCountTokensRequest):
return
output_config: AnthropicOutputConfig | None = anthropic_request.output_config
@@ -519,7 +519,7 @@ class AnthropicServingMessages(OpenAIServingChat):
req: ChatCompletionRequest,
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
) -> None:
"""Handle streaming configuration"""
"""Handle streaming configuration."""
if isinstance(anthropic_request, AnthropicCountTokensRequest):
return
if anthropic_request.stream:
@@ -534,7 +534,7 @@ class AnthropicServingMessages(OpenAIServingChat):
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
req: ChatCompletionRequest,
) -> None:
"""Convert Anthropic tool_choice to OpenAI format"""
"""Convert Anthropic tool_choice to OpenAI format."""
if anthropic_request.tool_choice is None:
req.tool_choice = None
return
@@ -563,7 +563,7 @@ class AnthropicServingMessages(OpenAIServingChat):
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
req: ChatCompletionRequest,
) -> None:
"""Convert Anthropic tools to OpenAI format"""
"""Convert Anthropic tools to OpenAI format."""
if anthropic_request.tools is None:
return
@@ -593,8 +593,7 @@ class AnthropicServingMessages(OpenAIServingChat):
request: AnthropicMessagesRequest,
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | AnthropicMessagesResponse | ErrorResponse:
"""
Messages API similar to Anthropic's API.
"""Messages API similar to Anthropic's API.
See https://docs.anthropic.com/en/api/messages
for the API specification. This API mimics the Anthropic messages API.
+14 -12
View File
@@ -223,9 +223,7 @@ class ChatCompletionContentPartVideoParam(TypedDict, total=False):
class PILImage(BaseModel):
"""
A PIL.Image.Image object.
"""
"""A PIL.Image.Image object."""
image_pil: Image.Image
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -238,6 +236,7 @@ class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
{
"image_pil": ImageAsset('cherry_blossom').pil_image
}
"""
image_pil: PILImage | None
@@ -256,6 +255,7 @@ class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
{
"image_url": "https://example.com/image.jpg"
}
"""
image_url: str | None
@@ -273,6 +273,7 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
{
"audio_url": "https://example.com/audio.mp3"
}
"""
audio_url: str | None
@@ -285,6 +286,7 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
{
"video_url": "https://example.com/video.mp4"
}
"""
video_url: str | None
@@ -304,6 +306,7 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):
"closed": True,
"type": "thinking"
}
"""
thinking: Required[str]
@@ -324,6 +327,7 @@ class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False):
"name": "get_weather",
"type": "tool_reference"
}
"""
name: str
@@ -586,8 +590,7 @@ def _get_embeds_data(
class BaseMultiModalItemTracker(ABC, Generic[_T]):
"""
Tracks multi-modal items in a given request and ensures that the number
"""Tracks multi-modal items in a given request and ensures that the number
of multi-modal items in a given request does not exceed the configured
maximum per prompt.
"""
@@ -651,8 +654,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
return get_video_processor_cls_name(self.model_config)
def add(self, modality: ModalityStr, item: _T) -> str | None:
"""
Add a multi-modal item to the current prompt and returns the
"""Add a multi-modal item to the current prompt and returns the
placeholder string to use, if any.
An optional uuid can be added which serves as a unique identifier of the
@@ -663,6 +665,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
pre-computed embeddings that do not go through any HF processor, encoder,
or model-specific placeholder logic. The corresponding placeholder string is
managed by the parser via `_add_placeholder`, so we return None here.
"""
add_info = self._validate_add(modality)
if add_info is None:
@@ -791,13 +794,13 @@ def _resolve_items(
mm_processor: BaseMultiModalProcessor | None,
modality_order: dict[str, list[str]],
) -> tuple[MultiModalDataDict, MultiModalUUIDDict]:
"""
Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`.
"""Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`.
Note:
`mm_processor` is `None` for text-only models (no registered HF
processor) whose only modality is `prompt_embeds`. Every other
modality requires a processor, enforced by the guard below.
"""
if "image" in items_by_modality and "image_embeds" in items_by_modality:
raise VLLMValidationError(
@@ -1568,7 +1571,6 @@ def _get_full_multimodal_text_prompt(
multimodal_content_part_separator: str = "\n",
) -> str:
"""Combine multimodal prompts for a multimodal language model."""
# flatten storage to make it looks like
# {
# "<|image|>": 2,
@@ -1687,8 +1689,7 @@ def _collect_extra_fields(part: dict[str, Any]) -> dict[str, Any]:
def _parse_chat_message_content_mm_part(
part: ChatCompletionContentPartParam,
) -> tuple[str, _ContentPart]:
"""
Parses a given multi-modal content part based on its type.
"""Parses a given multi-modal content part based on its type.
Args:
part: A dict containing the content part, with a potential 'type' field.
@@ -1700,6 +1701,7 @@ def _parse_chat_message_content_mm_part(
Raises:
ValueError: If the 'type' field is missing and no direct URL is found.
"""
assert isinstance(
part, dict
+1
View File
@@ -22,5 +22,6 @@ class BenchmarkSubcommandBase(CLISubcommand):
Args:
args: The arguments to the command.
"""
raise NotImplementedError
+3 -2
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The CLI entrypoints of vLLM
"""The CLI entrypoints of vLLM.
Note that all future modules must be lazily loaded within main
to avoid certain eager import breakage."""
to avoid certain eager import breakage.
"""
import importlib.metadata
import sys
+4 -2
View File
@@ -179,6 +179,7 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
Args:
n: Number of sequences the request will occupy.
"""
if self.engine_client.errored:
raise self.engine_client.dead_error
@@ -236,7 +237,7 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
@staticmethod
def _get_data_parallel_rank(raw_request: Request | None) -> int | None:
"""Pulls the data parallel rank from a header, if provided"""
"""Pulls the data parallel rank from a header, if provided."""
if raw_request is None:
return None
@@ -294,7 +295,8 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
) -> _T:
"""Wrap a `create_*` coroutine so that, if it raises or returns an
ErrorResponse (i.e. the request never reached the engine), the KV
connector is notified to free any pinned remote-prefill blocks."""
connector is notified to free any pinned remote-prefill blocks.
"""
kv_transfer_params = self.has_kv_connector and request.kv_transfer_params
if not kv_transfer_params or not kv_transfer_params.get("do_remote_prefill"):
return await awaitable
@@ -53,7 +53,7 @@ def _bitmask_to_token_ids(bitmask_row: torch.Tensor, vocab_size: int) -> list[in
class BeamSearchOfflineMixin(OfflineInferenceMixin):
"""Offline inference for beam search"""
"""Offline inference for beam search."""
def beam_search(
self,
@@ -63,8 +63,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
use_tqdm: bool = False,
concurrency_limit: int | None = None,
) -> list[BeamSearchOutput]:
"""
Generate sequences using beam search.
"""Generate sequences using beam search.
Args:
prompts: A list of prompts. Each prompt can be a string or a list
@@ -74,6 +73,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
use_tqdm: Whether to use tqdm to display the progress bar.
concurrency_limit: The maximum number of concurrent requests.
If None, the number of concurrent requests is unlimited.
"""
# TODO: how does beam search work together with length penalty,
# frequency, penalty, and stopping criteria, etc.?
@@ -20,7 +20,7 @@ from .utils import BeamSearchSequence, create_sort_beams_key_function
class BeamSearchOnlineMixin(ABC):
"""online serving for beam search"""
"""online serving for beam search."""
renderer: BaseRenderer
engine_client: EngineClient
@@ -60,6 +60,7 @@ class GenerativeScoringRequest(OpenAIBaseModel):
the full vocab for those ids (False).
item_first: If True, prepend items to query. Otherwise append items to query.
add_special_tokens: Whether to add special tokens when tokenizing.
"""
model: str | None = None
@@ -112,6 +113,7 @@ class GenerativeScoringItemResult(OpenAIBaseModel):
index: The index of this item in the input items list.
object: Type of object, always "score".
score: The probability score for the first label token.
"""
index: int
@@ -129,6 +131,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
model: The model used for scoring.
data: List of scoring results, one per input item.
usage: Token usage information.
"""
id: str = Field(default="")
@@ -188,6 +191,7 @@ class ServingGenerativeScoring(BaseServing):
Returns:
GenerativeScoringResponse with probabilities for each item, or
ErrorResponse if an error occurred.
"""
# Check model
error_check_ret = await self._check_model(request) # type: ignore[arg-type]
@@ -401,6 +405,7 @@ class ServingGenerativeScoring(BaseServing):
Returns:
Tuple of (list of EngineInput, list of prompt token counts).
"""
# Tokenize query if it's a string
if isinstance(request.query, str):
@@ -455,6 +460,7 @@ class ServingGenerativeScoring(BaseServing):
Returns:
Dictionary mapping token_id to probability.
"""
if apply_softmax:
# Normalize over only the label tokens (subset softmax)
@@ -70,14 +70,12 @@ async def build_async_engine_client_from_engine_args(
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
client_config: dict[str, Any] | None = None,
) -> AsyncIterator[EngineClient]:
"""
Create EngineClient, either:
"""Create EngineClient, either:
- in-process using the AsyncLLMEngine Directly
- multiprocess using AsyncLLMEngine RPC
Returns the Client or None if the creation failed.
"""
# Create the EngineConfig (determines if we can use V1).
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
@@ -123,7 +121,6 @@ async def build_and_serve(
Returns the shutdown task for the caller to await.
"""
# Get uvicorn log config (from file or with endpoint filter)
log_config = get_uvicorn_log_config(args)
if log_config is not None:
@@ -162,7 +159,6 @@ async def build_and_serve(
async def run_server(args, **uvicorn_kwargs) -> None:
"""Run a single-worker API server."""
decorate_logs("APIServer", skip_if_decorated=True)
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
@@ -180,7 +176,6 @@ async def run_server_worker(
listen_address, sock, args, client_config=None, **uvicorn_kwargs
) -> None:
"""Run a single API server worker."""
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file contains the command line arguments for the vLLM's online server.
"""This file contains the command line arguments for the vLLM's online server.
It is kept in a separate file for documentation purposes.
"""
+8 -19
View File
@@ -182,8 +182,7 @@ async def _probe_endpoint(
conn_err_failure_threshold: int = 3,
conn_err_retry_delay: float = 5.0,
) -> bool:
"""
Probe /health endpoint for 200 status.
"""Probe /health endpoint for 200 status.
If there is a connection error, retry every N seconds.
"""
@@ -251,9 +250,7 @@ def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
"""
Entrypoint function for the vLLM DP Server.
"""
"""Entrypoint function for the vLLM DP Server."""
# Create a fresh process group for the vLLM DP Server,
# so that CTRL-C is propagated cleanly.
os.setpgrp()
@@ -320,8 +317,7 @@ class DPSupervisor:
await supervisor_server_task
async def _start_server(self) -> tuple[uvicorn.Server, asyncio.Task[None]]:
"""
Launch the DPSupervisor HTTP server.
"""Launch the DPSupervisor HTTP server.
Called only after the vLLM DP Servers are ready so that /health does
not return 503 to external probes while the engines are initializing.
@@ -366,8 +362,7 @@ class DPSupervisor:
return supervisor_server, supervisor_server_task
async def _wait_until_ready(self, monitor_task: asyncio.Task[None]) -> None:
"""
Block until the vLLM DP Servers are ready or shutdown is triggered.
"""Block until the vLLM DP Servers are ready or shutdown is triggered.
Returns early if monitoring stops (e.g. a DP Server dies during
startup), in which case the supervisor server is never started.
@@ -379,14 +374,12 @@ class DPSupervisor:
await asyncio.sleep(0.05)
def _handle_signal(self, signum: int) -> None:
"""
Signal handler that is added to the event loop.
"""Signal handler that is added to the event loop.
This catches the SIGTERM from K8s and begins graceful shutdown,
by setting the _shutdown_event(), which is watched by the main
coroutine monitoring the vLLM DP Servers.
"""
if self._shutdown_event.is_set():
return
@@ -400,9 +393,7 @@ class DPSupervisor:
self._is_ready = False
def _start_children(self) -> None:
"""
Launch vLLM DP Servers on separate GPUs.
"""
"""Launch vLLM DP Servers on separate GPUs."""
logger.info("Launching vLLM DP Servers")
context = multiprocessing.get_context("spawn")
for local_rank in range(self.args.data_parallel_size_local):
@@ -416,8 +407,7 @@ class DPSupervisor:
self._processes.append(process)
async def _probe_all_children(self) -> None:
"""
Background coroutine: probes all child endpoints on each interval.
"""Background coroutine: probes all child endpoints on each interval.
Exits when any server becomes unhealthy after being ready, signalling
_monitor_children to initiate shutdown.
@@ -474,8 +464,7 @@ class DPSupervisor:
)
async def _monitor_children(self) -> None:
"""
Main coroutine task that monitors the children vLLM servers.
"""Main coroutine task that monitors the children vLLM servers.
Before the vLLM servers are /ready:
- if the pid is dead, we will shut down
+4 -4
View File
@@ -2,8 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# mypy: ignore-errors
"""
vLLM gRPC Server
"""vLLM gRPC Server
Starts a gRPC server backed by AsyncLLM, using the VllmEngineServicer
from the smg-grpc-servicer package.
@@ -16,6 +15,7 @@ Example:
--model meta-llama/Llama-2-7b-hf \
--host 0.0.0.0 \
--port 50051
"""
import argparse
@@ -54,11 +54,11 @@ logger = init_logger(__name__)
async def serve_grpc(args: argparse.Namespace):
"""
Main gRPC serving function.
"""Main gRPC serving function.
Args:
args: Parsed command line arguments
"""
log_version_and_model(logger, VLLM_VERSION, args.model)
logger.info("vLLM gRPC server args: %s", args)
+3 -7
View File
@@ -53,8 +53,7 @@ async def serve_http(
enable_ssl_refresh: bool = False,
**uvicorn_kwargs: Any,
):
"""
Start a FastAPI app using Uvicorn, with support for custom Uvicorn config
"""Start a FastAPI app using Uvicorn, with support for custom Uvicorn config
options. Supports http header limits via h11_max_incomplete_event_size and
h11_max_header_count.
"""
@@ -185,8 +184,7 @@ async def serve_http(
async def watchdog_loop(server: uvicorn.Server, engine: EngineClient):
"""
# Watchdog task that runs in the background, checking
"""# Watchdog task that runs in the background, checking
# for error state in the engine. Needed to trigger shutdown
# if an exception arises is StreamingResponse() generator.
"""
@@ -197,8 +195,7 @@ async def watchdog_loop(server: uvicorn.Server, engine: EngineClient):
def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
"""
See discussions here on shutting down a uvicorn server
"""See discussions here on shutting down a uvicorn server
https://github.com/encode/uvicorn/discussions/1103
In this case we cannot await the server shutdown here
because handler must first return to close the connection
@@ -254,7 +251,6 @@ def validate_api_server_args(args):
@instrument(span_name="API server setup")
def setup_server(args, *, reuse_port: bool):
"""Validate API server args and create the server socket."""
log_version_and_model(logger, VLLM_VERSION, args.model)
log_non_default_args(args)
@@ -31,7 +31,6 @@ async def init_render_app_state(
preprocessing pipeline (renderer, input_processor)
directly from the :class:`~vllm.config.VllmConfig`.
"""
served_model_names = args.served_model_name or [args.model]
model_registry = OpenAIModelRegistry(
model_config=vllm_config.model_config,
@@ -30,7 +30,6 @@ async def build_and_serve_renderer(
Returns the shutdown task for the caller to await.
"""
# Get uvicorn log config (from file or with endpoint filter)
log_config = get_uvicorn_log_config(args)
if log_config is not None:
+15 -23
View File
@@ -88,8 +88,7 @@ def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
class BatchTranscriptionRequest(TranscriptionRequest):
"""
Batch transcription request that uses file_url instead of file.
"""Batch transcription request that uses file_url instead of file.
This class extends TranscriptionRequest but replaces the file field
with file_url to support batch processing from audio files written in JSON format.
@@ -119,8 +118,7 @@ class BatchTranscriptionRequest(TranscriptionRequest):
class BatchTranslationRequest(TranslationRequest):
"""
Batch translation request that uses file_url instead of file.
"""Batch translation request that uses file_url instead of file.
This class extends TranslationRequest but replaces the file field
with file_url to support batch processing from audio files written in JSON format.
@@ -160,8 +158,7 @@ BatchRequestInputBody: TypeAlias = (
class BatchRequestInput(OpenAIBaseModel):
"""
The per-line object of the batch input file.
"""The per-line object of the batch input file.
NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
"""
@@ -225,9 +222,7 @@ class BatchResponseData(OpenAIBaseModel):
class BatchRequestOutput(OpenAIBaseModel):
"""
The per-line object of the batch output and error files
"""
"""The per-line object of the batch output and error files"""
id: str
@@ -358,8 +353,7 @@ async def read_file(path_or_url: str) -> str:
async def write_local_file(
output_path: str, batch_outputs: list[BatchRequestOutput]
) -> None:
"""
Write the responses to a local file.
"""Write the responses to a local file.
output_path: The path to write the responses to.
batch_outputs: The list of batch outputs to write.
"""
@@ -371,8 +365,7 @@ async def write_local_file(
async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
"""
Upload a local file to a URL.
"""Upload a local file to a URL.
output_url: The URL to upload the file to.
data_or_file: Either the data to upload or the path to the file to upload.
from_file: If True, data_or_file is the path to the file to upload.
@@ -429,8 +422,7 @@ async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> No
async def write_file(
path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
) -> None:
"""
Write batch_outputs to a file or upload to a URL.
"""Write batch_outputs to a file or upload to a URL.
path_or_url: The path or URL to write batch_outputs to.
batch_outputs: The list of batch outputs to write.
output_tmp_dir: The directory to store the output file before uploading it
@@ -471,8 +463,7 @@ async def download_bytes_from_url(
url: str,
allowed_media_domains: list[str] | None = None,
) -> bytes:
"""
Download data from a URL or decode from a data URL.
"""Download data from a URL or decode from a data URL.
Args:
url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
@@ -482,6 +473,7 @@ async def download_bytes_from_url(
Returns:
Data as bytes
"""
parsed = urlparse(url)
@@ -615,8 +607,7 @@ def handle_endpoint_request(
handler_getter: Callable[[], Callable | None],
wrapper_fn: WrapperFn | None = None,
) -> Awaitable[BatchRequestOutput] | None:
"""
Generic handler for endpoint requests.
"""Generic handler for endpoint requests.
Args:
request: The batch request input
@@ -628,6 +619,7 @@ def handle_endpoint_request(
Returns:
Awaitable[BatchRequestOutput] if the request was handled,
None if URL didn't match
"""
if not url_matcher(request.url):
return None
@@ -649,8 +641,7 @@ def make_transcription_wrapper(
is_translation: bool,
allowed_media_domains: list[str] | None = None,
) -> WrapperFn:
"""
Factory function to create a wrapper for transcription/translation handlers.
"""Factory function to create a wrapper for transcription/translation handlers.
The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest
to TranscriptionRequest or TranslationRequest and calls the appropriate handler.
@@ -662,6 +653,7 @@ def make_transcription_wrapper(
Returns:
A function that takes a handler and returns a wrapped handler
"""
def wrapper(handler_fn: Callable):
@@ -723,8 +715,7 @@ async def build_endpoint_registry(
engine_client: EngineClient,
args: Namespace,
) -> dict[str, dict[str, Any]]:
"""
Build the endpoint registry with all serving objects and handler configurations.
"""Build the endpoint registry with all serving objects and handler configurations.
Args:
engine_client: The engine client
@@ -732,6 +723,7 @@ async def build_endpoint_registry(
Returns:
Dictionary mapping endpoint keys to their configurations
"""
supported_tasks = await engine_client.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
@@ -29,8 +29,7 @@ def load_log_config(log_config_file: str | None) -> dict | None:
def get_uvicorn_log_config(args: Namespace) -> dict | None:
"""
Get the uvicorn log config based on the provided arguments.
"""Get the uvicorn log config based on the provided arguments.
Priority:
1. If log_config_file is specified, use it
+22 -16
View File
@@ -174,6 +174,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Note:
This class is intended to be used for offline inference. For online
serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead.
"""
def __init__(
@@ -224,7 +225,6 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
**kwargs: Any,
) -> None:
"""LLM constructor."""
if "disable_log_stats" not in kwargs:
kwargs["disable_log_stats"] = True
@@ -398,6 +398,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Returns:
The world size (tensor_parallel_size * pipeline_parallel_size),
optionally multiplied by data_parallel_size if include_dp is True.
"""
parallel_config = self.llm_engine.vllm_config.parallel_config
if include_dp:
@@ -457,6 +458,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Returns:
A list of `RequestOutput` objects containing the
generated completions in the same order as the input prompts.
"""
runner_type = self.model_config.runner_type
if runner_type != "generate":
@@ -507,6 +509,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Returns:
A list of request IDs for the enqueued requests.
"""
runner_type = self.model_config.runner_type
if runner_type != "generate":
@@ -558,6 +561,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Returns:
A list of output objects for all completed requests.
"""
if output_type is None:
output_type = (RequestOutput, PoolingRequestOutput)
@@ -571,8 +575,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
args: tuple = (),
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
"""
Execute an RPC call on all workers.
"""Execute an RPC call on all workers.
Args:
method: Name of the worker method to execute, or a callable that
@@ -592,13 +595,12 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Note:
It is recommended to use this API to only pass control messages,
and set up data-plane communication to pass data.
"""
"""
return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
"""
Run a function directly on the model inside each worker,
"""Run a function directly on the model inside each worker,
returning the result for each of them.
!!! warning
@@ -625,8 +627,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[RequestOutput]:
"""
Generate responses for a chat conversation.
"""Generate responses for a chat conversation.
The chat conversation is converted into a text prompt using the
tokenizer and calls the [generate][vllm.LLM.generate] method to generate
@@ -670,10 +671,12 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
template.
tokenization_kwargs: Overrides for `tokenizer.encode`.
mm_processor_kwargs: Overrides for `processor.__call__`.
tools: Tools to make available to the model, if any.
Returns:
A list of `RequestOutput` objects containing the generated
responses in the same order as the input messages.
"""
model_config = self.model_config
runner_type = model_config.runner_type
@@ -751,6 +754,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Returns:
A list of request IDs for the enqueued requests.
"""
model_config = self.model_config
runner_type = model_config.runner_type
@@ -787,6 +791,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
profile_prefix: Optional prefix for the trace file names. If provided,
trace files will be named as "<prefix>_dp<X>_pp<Y>_tp<Z>".
If not provided, default naming will be used.
"""
self.llm_engine.start_profile(profile_prefix)
@@ -801,8 +806,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
)
def sleep(self, level: int = 1, mode: PauseMode = "abort"):
"""
Put the engine to sleep. The engine should not process any requests.
"""Put the engine to sleep. The engine should not process any requests.
The caller should guarantee that no requests are being processed
during the sleep period, before `wake_up` is called.
@@ -822,12 +826,12 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
CPU memory pressure.
mode: How to handle any existing requests, can be "abort", "wait",
or "keep".
"""
self.llm_engine.sleep(level=level, mode=mode)
def wake_up(self, tags: list[str] | None = None):
"""
Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
"""Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
method for more details.
Args:
@@ -837,6 +841,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
is reallocated. wake_up should be called with all tags
(or None) before the engine is used again.
Use tags=["scheduling"] to resume from level 0 sleep.
"""
self.llm_engine.wake_up(tags)
@@ -849,17 +854,18 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
Note:
This method is only available with the V1 LLM engine.
"""
return self.llm_engine.get_metrics()
def init_weight_transfer_engine(
self, request: WeightTransferInitRequest | dict
) -> None:
"""
Initialize weight transfer for RL training.
"""Initialize weight transfer for RL training.
Args:
request: Weight transfer initialization request with backend-specific info
"""
init_info_dict = (
request["init_info"] if isinstance(request, dict) else request.init_info
@@ -878,11 +884,11 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
self.llm_engine.collective_rpc("start_draft_weight_update")
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
"""
Update the weights of the model.
"""Update the weights of the model.
Args:
request: Weight update request with backend-specific update info
"""
update_info_dict = (
request["update_info"] if isinstance(request, dict) else request.update_info
+2 -4
View File
@@ -23,8 +23,7 @@ MIN_GPT_OSS_VERSION = "0.0.7"
def validate_gpt_oss_install():
"""
Check if the gpt-oss is installed and its version is at least 0.0.7.
"""Check if the gpt-oss is installed and its version is at least 0.0.7.
If not, raise an ImportError.
"""
from importlib.metadata import PackageNotFoundError, version
@@ -151,8 +150,7 @@ class HarmonyPythonTool(Tool):
return tool_output_msgs
async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
"""
This function converts parsable context types to harmony and
"""This function converts parsable context types to harmony and
back so we can use GPTOSS demo python tool
"""
from vllm.entrypoints.openai.responses.context import ParsableContext
+3 -8
View File
@@ -76,17 +76,14 @@ def post_process_tools_description(
class ToolServer(ABC):
@abstractmethod
def has_tool(self, tool_name: str) -> bool:
"""
Return True if the tool is supported, False otherwise.
"""
"""Return True if the tool is supported, False otherwise."""
pass
@abstractmethod
def get_tool_description(
self, tool_name: str, allowed_tools: list[str] | None = None
) -> ToolNamespaceConfig | None:
"""
Return the tool description for the given tool name.
"""Return the tool description for the given tool name.
If the tool is not supported, return None.
"""
pass
@@ -95,9 +92,7 @@ class ToolServer(ABC):
def new_session(
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
) -> AbstractAsyncContextManager[Any]:
"""
Create a session for the tool.
"""
"""Create a session for the tool."""
...
+5 -5
View File
@@ -48,7 +48,7 @@ _R = TypeVar("_R", default=Any)
class OfflineInferenceMixin:
"""Offline inference utils"""
"""Offline inference utils."""
request_counter: Counter
renderer: BaseRenderer
@@ -113,14 +113,14 @@ class OfflineInferenceMixin:
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
"""
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
"""Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
a format that can be passed to `_add_request`.
Refer to [LLM.generate][] for a complete description of the arguments.
Returns:
A list of `EngineInput` objects ready to be passed into LLMEngine.
"""
renderer = self.renderer
model_config = self.model_config
@@ -168,14 +168,14 @@ class OfflineInferenceMixin:
tokenization_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
"""
Convert a list of conversations into prompts so that they can then
"""Convert a list of conversations into prompts so that they can then
be used as input for other LLM APIs.
Refer to [LLM.chat][] for a complete description of the arguments.
Returns:
A list of `EngineInput` objects ready to be passed into LLMEngine.
"""
renderer = self.renderer
@@ -50,6 +50,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
Returns:
A tuple of (all_conversations, engine_prompts) on success — one
entry per conversation — or an ErrorResponse on failure.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
@@ -220,8 +220,7 @@ class OpenAIServingChat(GenerateBaseServing):
self,
request: ChatCompletionRequest,
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
"""
Validate the model and preprocess a chat completion request.
"""Validate the model and preprocess a chat completion request.
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
@@ -229,6 +228,7 @@ class OpenAIServingChat(GenerateBaseServing):
Returns:
A tuple of (conversation, engine_inputs) on success,
or an ErrorResponse on failure.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
@@ -244,8 +244,7 @@ class OpenAIServingChat(GenerateBaseServing):
request: ChatCompletionRequest,
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
"""
Chat Completion API similar to OpenAI's API.
"""Chat Completion API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/chat/create
for the API specification. This API mimics the OpenAI
@@ -89,14 +89,14 @@ class OpenAIServingCompletion(GenerateBaseServing):
self,
request: CompletionRequest,
) -> list[EngineInput] | ErrorResponse:
"""
Validate the model and preprocess a completion request.
"""Validate the model and preprocess a completion request.
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
Returns:
A list of engine_inputs on success, or an ErrorResponse on failure.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
+5 -2
View File
@@ -123,7 +123,8 @@ class OpenAIServingModels:
async def init_static_loras(self):
"""Loads all static LoRA modules.
Raises if any fail to load"""
Raises if any fail to load
"""
if self.static_lora_modules is None:
return
for lora in self.static_lora_modules:
@@ -148,7 +149,8 @@ class OpenAIServingModels:
async def show_available_models(self) -> ModelList:
"""Show available models. This includes the base model and all
adapters."""
adapters.
"""
model_list = await self.registry.show_available_models()
lora_cards = [
ModelCard(
@@ -289,6 +291,7 @@ class OpenAIServingModels:
LoRARequest if found and loaded successfully.
ErrorResponse (404) if no resolver finds the adapter.
ErrorResponse (400) if adapter(s) are found but none load.
"""
async with self.lora_resolver_lock[lora_name]:
# First check if this LoRA is already loaded
@@ -88,8 +88,7 @@ MCP_BUILTIN_TOOLS: set[str] = set(BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())
def has_custom_tools(tool_types: set[str]) -> bool:
"""
Checks if the given tool types are custom tools
"""Checks if the given tool types are custom tools
(i.e. any tool other than MCP builtin tools)
"""
return not tool_types.issubset(MCP_BUILTIN_TOOLS)
@@ -209,8 +208,7 @@ def get_system_or_developer_message(role: str, instructions: str) -> Message:
def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
"""
Parse a list of messages from request.messages in the Chat Completion API to
"""Parse a list of messages from request.messages in the Chat Completion API to
Harmony messages.
"""
msgs: list[Message] = []
@@ -230,9 +228,8 @@ def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]:
"""
Harmony models expect the analysis messages (representing raw chain of thought) to
be dropped after an assistant message to the final channel is produced from the
"""Harmony models expect the analysis messages (representing raw chain of thought)
to be dropped after an assistant message to the final channel is produced from the
reasoning of those messages.
The openai-harmony library does this if the very last assistant message is to the
@@ -262,8 +259,7 @@ def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]:
def flatten_input_text_content(content: Any) -> str | None:
"""
Extract text parts from a Chat Completion or Responses API content field and
"""Extract text parts from a Chat Completion or Responses API content field and
flatten them into a single string. Returns None if no text content is found.
"""
if content is None or isinstance(content, str):
@@ -286,8 +282,7 @@ def flatten_input_text_content(content: Any) -> str | None:
def extract_instructions_from_messages(
messages: Sequence[Any],
) -> tuple[str | None, list[Any]]:
"""
Peel a leading system/developer Chat Completion or Responses message and
"""Peel a leading system/developer Chat Completion or Responses message and
flatten its instruction text.
"""
remaining_messages = list(messages)
@@ -327,9 +322,7 @@ def build_harmony_preamble(
container_description: str | None = None,
with_custom_tools: bool = False,
) -> list[Message]:
"""
Build the standard Harmony system/developer prefix for a request.
"""
"""Build the standard Harmony system/developer prefix for a request."""
developer_instructions = system_instructions = None
if envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
system_instructions = instructions
@@ -359,8 +352,7 @@ def build_harmony_preamble(
def parse_chat_input_to_harmony_message(
chat_msg, tool_id_names: dict[str, str] | None = None
) -> list[Message]:
"""
Parse a message from request.messages in the Chat Completion API to
"""Parse a message from request.messages in the Chat Completion API to
Harmony messages.
"""
tool_id_names = tool_id_names or {}
@@ -34,7 +34,7 @@ def responses(request: Request) -> OpenAIServingResponses | None:
async def _convert_stream_to_sse_events(
generator: AsyncGenerator[StreamingResponsesResponse, None],
) -> AsyncGenerator[str, None]:
"""Convert the generator to a stream of events in SSE format"""
"""Convert the generator to a stream of events in SSE format."""
async for event in generator:
event_type = getattr(event, "type", "unknown")
# https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
+10 -11
View File
@@ -141,9 +141,7 @@ class ConversationContext(ABC):
def _create_json_parse_error_messages(
last_msg: Message, e: json.JSONDecodeError
) -> list[Message]:
"""
Creates an error message when json parse failed.
"""
"""Creates an error message when json parse failed."""
error_msg = (
f"Error parsing tool arguments as JSON: {str(e)}. "
"Please ensure the tool call arguments are valid JSON and try again."
@@ -161,7 +159,7 @@ def _create_json_parse_error_messages(
class SimpleContext(ConversationContext):
"""This is a context that cannot handle MCP tool calls"""
"""This is a context that cannot handle MCP tool calls."""
def __init__(
self,
@@ -410,7 +408,8 @@ class ParsableContext(ConversationContext):
def need_builtin_tool_call(self) -> bool:
"""Return true if the last message is a builtin tool call
that the request has enabled."""
that the request has enabled.
"""
last_message = self.response_messages[-1]
if last_message.type != "function_call":
return False
@@ -474,8 +473,7 @@ class ParsableContext(ConversationContext):
async def call_container_tool(
self, tool_session: Union["ClientSession", Tool], last_msg: Message
) -> list[Message]:
"""
Call container tool. Expect this to be run in a stateful docker
"""Call container tool. Expect this to be run in a stateful docker
with command line terminal.
The official container tool would at least
expect the following format:
@@ -583,7 +581,7 @@ class ParsableContext(ConversationContext):
exit_stack.push_async_exit(self.cleanup_session)
async def cleanup_session(self, *args, **kwargs) -> None:
"""Can be used as coro to used in __aexit__"""
"""Can be used as coro to used in __aexit__."""
async def cleanup_tool_session(tool_session):
if not isinstance(tool_session, Tool):
@@ -693,6 +691,7 @@ class HarmonyContext(ConversationContext):
Args:
output: The RequestOutput containing prompt token information
"""
if output.prompt_token_ids is not None:
this_turn_input_tokens = len(output.prompt_token_ids)
@@ -757,6 +756,7 @@ class HarmonyContext(ConversationContext):
Returns:
int: Number of output tokens processed in this call
"""
updated_output_token_count = 0
if output.outputs:
@@ -884,8 +884,7 @@ class HarmonyContext(ConversationContext):
async def call_container_tool(
self, tool_session: Union["ClientSession", Tool], last_msg: Message
) -> list[Message]:
"""
Call container tool. Expect this to be run in a stateful docker
"""Call container tool. Expect this to be run in a stateful docker
with command line terminal.
The official container tool would at least
expect the following format:
@@ -925,7 +924,7 @@ class HarmonyContext(ConversationContext):
]
async def cleanup_session(self, *args, **kwargs) -> None:
"""Can be used as coro to used in __aexit__"""
"""Can be used as coro to used in __aexit__."""
async def cleanup_tool_session(tool_session):
if not isinstance(tool_session, Tool):
+3 -3
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Harmony ↔ Responses API conversion utilities.
"""Harmony ↔ Responses API conversion utilities.
Handles two directions:
1. Response Input → Harmony Messages (input parsing)
@@ -53,7 +52,8 @@ logger = init_logger(__name__)
def _parse_harmony_format_message(chat_msg: dict) -> Message:
"""Reconstruct a Message from Harmony-format dict,
preserving channel, recipient, and content_type."""
preserving channel, recipient, and content_type.
"""
author_dict = chat_msg["author"]
role = author_dict.get("role")
name = author_dict.get("name")
@@ -100,9 +100,7 @@ class ResponseUsage(OpenAIBaseModel):
def serialize_message(msg):
"""
Serializes a single message
"""
"""Serializes a single message."""
if isinstance(msg, dict):
return msg
elif hasattr(msg, "to_dict"):
@@ -113,15 +111,14 @@ def serialize_message(msg):
def serialize_messages(msgs):
"""
Serializes multiple messages
"""
"""Serializes multiple messages."""
return [serialize_message(msg) for msg in msgs] if msgs else None
class ResponseRawMessageAndToken(OpenAIBaseModel):
"""Class to show the raw message.
If message / tokens diverge, tokens is the source of truth"""
If message / tokens diverge, tokens is the source of truth
"""
message: str
tokens: list[int]
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Streaming SSE event builders for the Responses API.
"""Streaming SSE event builders for the Responses API.
Pure functions that translate streaming state + delta data into
OpenAI Response API SSE events. Used by the streaming event
@@ -125,8 +124,7 @@ def is_mcp_tool_by_namespace(
recipient: str | None,
allowed_function_tool_names: frozenset[str] | None = None,
) -> bool:
"""
Determine if a tool call is an MCP tool based on recipient prefix.
"""Determine if a tool call is an MCP tool based on recipient prefix.
Inverse of :func:`is_function_recipient` — everything that is not
a function call is an MCP tool.
@@ -1145,8 +1143,7 @@ def split_delta(delta: DeltaMessage) -> list[DeltaMessage]:
class SimpleStreamingEventProcessor:
"""
State-machine processor for the simple (non-Harmony) streaming path.
"""State-machine processor for the simple (non-Harmony) streaming path.
Core flow:
1. Resolve the target state from the delta_message
@@ -1188,8 +1185,7 @@ class SimpleStreamingEventProcessor:
def resolve_target_state(
self, delta_message: DeltaMessage
) -> tuple[_StateType, Any]:
"""
Decide which state the next delta belongs to.
"""Decide which state the next delta belongs to.
Priority: TOOL_CALL > REASONING > CONTENT, fallback to NONE.
For TOOL_CALL the first tool_call object is also returned so
@@ -1207,8 +1203,7 @@ class SimpleStreamingEventProcessor:
return _StateType.NONE, None
def needs_transition(self, target_state: _StateType, tool_call: Any) -> bool:
"""
Return True when we must close the current state and open a new one.
"""Return True when we must close the current state and open a new one.
Two cases trigger a transition:
1. The target state differs from the current state
+5 -9
View File
@@ -115,8 +115,7 @@ def build_response_output_items(
def should_continue_final_message(
request_input: str | list[ResponseInputOutputItem],
) -> bool:
"""
Determine if the last input message is a partial assistant message
"""Determine if the last input message is a partial assistant message
that should be continued rather than starting a new generation.
This enables partial message completion similar to Anthropic's Messages API,
@@ -132,6 +131,7 @@ def should_continue_final_message(
Returns:
True if the final message should be continued, False otherwise
"""
if isinstance(request_input, str):
# Simple string input is always a user message
@@ -226,8 +226,7 @@ def _construct_message_from_response_item(
item: ResponseInputOutputItem,
prev_msg: ChatCompletionMessageParam | None = None,
) -> ChatCompletionMessageParam | None:
"""
Returns a new message or None. If `None`, `prev_msg` might be updated.
"""Returns a new message or None. If `None`, `prev_msg` might be updated.
If `prev_msg` is `None`, a new message is always returned.
"""
prev_assistant_msg = (
@@ -360,9 +359,7 @@ def extract_function_tool_names(tools: list[Tool]) -> frozenset[str]:
def extract_tool_types(tools: list[Tool]) -> set[str]:
"""
Extracts the tool types from the given tools.
"""
"""Extracts the tool types from the given tools."""
tool_types: set[str] = set()
for tool in tools:
if tool.type == "mcp":
@@ -379,8 +376,7 @@ def extract_tool_types(tools: list[Tool]) -> set[str]:
def convert_tool_responses_to_completions_format(
tool: dict,
) -> ChatCompletionToolsParam:
"""
Convert a flat Responses tool schema:
"""Convert a flat Responses tool schema:
{"type": "function", "name": "...", "description": "...", "parameters": {...}}
into a Chat Completions tool param for chat-template rendering.
"""
+8 -9
View File
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
class PoolingOfflineMixin(OfflineInferenceMixin):
"""Offline inference for pooling models"""
"""Offline inference for pooling models."""
runner_type: str
chat_template: str | None
@@ -93,8 +93,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
Returns:
A list of `PoolingRequestOutput` objects containing the
pooled hidden states in the same order as the input prompts.
"""
"""
if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
raise ValueError(
"The 'data' field is only supported for the 'plugin' pooling task."
@@ -203,8 +203,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[EmbeddingRequestOutput]:
"""
Generate an embedding vector for each prompt.
"""Generate an embedding vector for each prompt.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
@@ -226,8 +225,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
Returns:
A list of `EmbeddingRequestOutput` objects containing the
embedding vectors in the same order as the input prompts.
"""
"""
items = self.encode(
prompts,
use_tqdm=use_tqdm,
@@ -248,8 +247,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[ClassificationRequestOutput]:
"""
Generate class logits for each prompt.
"""Generate class logits for each prompt.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
@@ -271,8 +269,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
Returns:
A list of `ClassificationRequestOutput` objects containing the
embedding vectors in the same order as the input prompts.
"""
"""
items = self.encode(
prompts,
use_tqdm=use_tqdm,
@@ -330,11 +328,12 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
chat_template: The chat template to use for the scoring. If None, we
use the model's default chat template.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `ScoringRequestOutput` objects containing the
generated scores in the same order as the input prompts.
"""
"""
if self.runner_type != "pooling":
raise ValueError(
"LLM.score() is only supported for pooling models. "
@@ -44,7 +44,8 @@ class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
class PluginWithIOProcessorPlugins(PoolingIOProcessor):
"""IO Processor plugins are a feature that allows pre- and post-processing
of the model input and output for pooling models."""
of the model input and output for pooling models.
"""
name = "plugin"
+1 -2
View File
@@ -21,8 +21,7 @@ ScoreContentPartParam: TypeAlias = (
class ScoreMultiModalParam(TypedDict, total=False):
"""
A specialized parameter type for scoring multimodal content
"""A specialized parameter type for scoring multimodal content.
The reasons why don't reuse `CustomChatCompletionMessageParam` directly:
1. Score tasks don't need the 'role' field (user/assistant/system) that's required in chat completions
+3 -4
View File
@@ -58,8 +58,7 @@ def truncate_text_to_tokens(
def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor:
"""
Compute ColBERT MaxSim score.
"""Compute ColBERT MaxSim score.
Args:
q_emb: Query token embeddings [query_len, dim]
@@ -67,6 +66,7 @@ def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tens
Returns:
MaxSim score (sum over query tokens of max similarity to any doc token)
"""
# compute in float32 for numerical stability
# [query_len, doc_len]
@@ -249,8 +249,7 @@ def parse_score_data(
def compress_token_type_ids(token_type_ids: list[int]) -> int:
"""
Return position of the first 1 or the length of the list
"""Return position of the first 1 or the length of the list
if not found.
"""
first_one = len(token_type_ids)
@@ -78,8 +78,7 @@ def attach_router(app: FastAPI):
@router.post("/abort_requests")
async def abort_requests(raw_request: Request):
"""
Abort one or more requests. To be used in a
"""Abort one or more requests. To be used in a
Disaggregated Everything setup.
"""
try:
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encode/decode utilities for multimodal tensors and field metadata
over JSON/HTTP, used by the disaggregated generate endpoint."""
over JSON/HTTP, used by the disaggregated generate endpoint.
"""
from __future__ import annotations
+4 -6
View File
@@ -23,8 +23,7 @@ async def reset_prefix_cache(
reset_running_requests: bool = Query(default=False),
reset_external: bool = Query(default=False),
):
"""
Reset the local prefix cache.
"""Reset the local prefix cache.
Optionally, if the query parameter `reset_external=true`
also resets the external (connector-managed) prefix cache.
@@ -35,6 +34,7 @@ async def reset_prefix_cache(
Example:
POST /reset_prefix_cache?reset_external=true
"""
logger.info("Resetting prefix cache...")
@@ -46,8 +46,7 @@ async def reset_prefix_cache(
@router.post("/reset_mm_cache")
async def reset_mm_cache(raw_request: Request):
"""
Reset the multi-modal cache. Note that we currently do not check if the
"""Reset the multi-modal cache. Note that we currently do not check if the
multi-modal cache is successfully reset in the API server.
"""
logger.info("Resetting multi-modal cache...")
@@ -57,8 +56,7 @@ async def reset_mm_cache(raw_request: Request):
@router.post("/reset_encoder_cache")
async def reset_encoder_cache(raw_request: Request):
"""
Reset the encoder cache. Note that we currently do not check if the
"""Reset the encoder cache. Note that we currently do not check if the
encoder cache is successfully reset in the API server.
"""
logger.info("Resetting encoder cache...")
@@ -36,6 +36,8 @@ async def pause_generation(
"""Pause generation requests to allow weight updates.
Args:
raw_request: The incoming FastAPI request, used to reach the engine
client on the app state.
mode: How to handle in-flight requests:
- ``"abort"``: Abort all in-flight requests immediately (default).
- ``"wait"``: Wait for in-flight requests to complete.
@@ -43,8 +45,8 @@ async def pause_generation(
wait_for_inflight_requests: DEPRECATED. Use ``mode="wait"`` instead.
clear_cache: DEPRECATED. Whether to clear KV/prefix caches after
draining. Ignored when mode="keep".
"""
"""
engine = engine_client(raw_request)
try:
@@ -74,7 +76,6 @@ async def pause_generation(
@router.post("/resume")
async def resume_generation(raw_request: Request) -> JSONResponse:
"""Resume generation after a pause."""
engine = engine_client(raw_request)
try:
@@ -97,7 +98,6 @@ async def abort_requests(raw_request: Request) -> JSONResponse:
Empty/missing ``request_ids`` aborts all in-flight requests.
"""
engine = engine_client(raw_request)
try:
@@ -139,7 +139,6 @@ async def abort_requests(raw_request: Request) -> JSONResponse:
@router.get("/is_paused")
async def is_paused(raw_request: Request) -> JSONResponse:
"""Return the current pause status."""
engine = engine_client(raw_request)
try:
@@ -234,9 +233,12 @@ async def get_world_size(
"""Get the world size from the parallel config.
Args:
raw_request: The incoming FastAPI request, used to reach the engine
client on the app state.
include_dp: If True (default), returns the world size including
data parallelism (TP * PP * DP). If False, returns the world
size without data parallelism (TP * PP).
"""
parallel_config = engine_client(raw_request).vllm_config.parallel_config
if include_dp:
@@ -20,8 +20,7 @@ def set_scaling_elastic_ep(value):
class ScalingMiddleware:
"""
Middleware that checks if the model is currently scaling and
"""Middleware that checks if the model is currently scaling and
returns a 503 Service Unavailable response if it is.
This middleware applies to all HTTP requests and prevents
+1 -1
View File
@@ -117,7 +117,7 @@ class BaseServing:
def _base_request_id(
raw_request: Request | None, default: str | None = None
) -> str | None:
"""Pulls the request id to use from a header, if provided"""
"""Pulls the request id to use from a header, if provided."""
if raw_request is not None and (
(req_id := raw_request.headers.get("X-Request-Id")) is not None
):
@@ -126,7 +126,8 @@ def _format_error(err: object) -> str:
def _is_internal_loc_segment(segment: str) -> bool:
"""True if `segment` is a Pydantic-internal wrapper/union-branch
marker rather than a user-meaningful field name or list index."""
marker rather than a user-meaningful field name or list index.
"""
if _BRACKETED_INTERNAL_RE.search(segment):
return True
return segment.lower() in _INTERNAL_LOC_MARKERS
@@ -28,8 +28,7 @@ async def vllm_error_handler(req: Request, exc: VLLMError):
async def engine_error_handler(
req: Request, exc: EngineDeadError | EngineGenerateError
):
"""
VLLM V1 AsyncLLM catches exceptions and returns
"""VLLM V1 AsyncLLM catches exceptions and returns
only two types: EngineGenerateError and EngineDeadError.
EngineGenerateError is raised by the per request generate()
@@ -51,7 +50,6 @@ async def engine_error_handler(
will not automatically shut down. Instead, we use the watchdog
background task for check for errored state.
"""
if req.app.state.args.log_error_stack:
logger.exception(
"Engine Exception caught. Request id: %s",
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
vLLM Exception handlers are registered in four layers:
"""vLLM Exception handlers are registered in four layers:
1. framework errors raised by FastAPI/Starlette
2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
@@ -55,7 +55,6 @@ class PrometheusResponse(Response):
def attach_router(app: FastAPI):
"""Mount prometheus metrics to a FastAPI app."""
registry = get_prometheus_registry()
# `response_class=PrometheusResponse` is needed to return an HTTP response
+4 -2
View File
@@ -63,7 +63,8 @@ def _attach_router(app: FastAPI):
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
"""Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
serving engine."""
serving engine.
"""
handler: OpenAIServingModels = models(raw_request)
response = await handler.load_lora_adapter(request)
if isinstance(response, ErrorResponse):
@@ -85,7 +86,8 @@ def _attach_router(app: FastAPI):
request: UnloadLoRAAdapterRequest, raw_request: Request
):
"""Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
the serving engine."""
the serving engine.
"""
handler: OpenAIServingModels = models(raw_request)
response = await handler.unload_lora_adapter(request)
if isinstance(response, ErrorResponse):
@@ -12,8 +12,7 @@ GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
class AuthenticationMiddleware:
"""
Pure ASGI middleware that authenticates each request by checking
"""Pure ASGI middleware that authenticates each request by checking
if the Authorization Bearer token exists and equals anyof "{api_key}".
Notes
@@ -21,6 +20,7 @@ class AuthenticationMiddleware:
There are two cases in which authentication is skipped:
1. The HTTP method is OPTIONS.
2. The request path doesn't start with GUARDED_PREFIX (e.g. /health).
"""
def __init__(self, app: ASGIApp, tokens: list[str]) -> None:
@@ -94,7 +94,6 @@ def _extract_content_from_chunk(chunk_data: dict) -> str:
def _log_streaming_response(response, response_body: list) -> None:
"""Log streaming response with robust SSE parsing."""
sse_decoder = SSEDecoder()
chunk_count = 0
@@ -8,8 +8,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
class XRequestIdMiddleware:
"""
Middleware the set's the X-Request-Id header for each response
"""Middleware the set's the X-Request-Id header for each response
to a random uuid4 (hex) value if the header isn't already
present in the request, otherwise use the provided request id.
"""
@@ -25,8 +24,7 @@ class XRequestIdMiddleware:
request_headers = Headers(scope=scope)
async def send_with_request_id(message: Message) -> None:
"""
Custom send function to mutate the response headers
"""Custom send function to mutate the response headers
and append X-Request-Id to it.
"""
if message["type"] == "http.response.start":
@@ -30,7 +30,8 @@ EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
"""Snapshot handler levels for loggers that may be affected by third-party
logging configuration side effects (e.g. model_hosting_container_standards
calling configure_root_logger() at import time)."""
calling configure_root_logger() at import time).
"""
loggers = [logging.getLogger(), logging.getLogger("vllm")]
seen: set[int] = set()
snapshot: list[tuple[logging.Handler, int]] = []
@@ -73,7 +74,8 @@ def _attach_router(
model_config: ModelConfig | None = None,
):
"""Register the SageMaker hosting routes (/ping, /invocations) on the
app."""
app.
"""
import model_hosting_container_standards.sagemaker as sagemaker_standards
router = APIRouter()
@@ -92,7 +94,7 @@ def _attach_router(
@router.get("/ping", response_class=Response)
@sagemaker_standards.register_ping_handler
async def ping(raw_request: Request) -> Response:
"""Ping check. Endpoint required for SageMaker"""
"""Ping check. Endpoint required for SageMaker."""
return await health(raw_request)
@router.post(
+1 -2
View File
@@ -181,8 +181,7 @@ class DetokenizeResponse(OpenAIBaseModel):
class TokenizerInfoResponse(OpenAIBaseModel):
"""
Response containing tokenizer configuration
"""Response containing tokenizer configuration
equivalent to tokenizer_config.json
"""
+1 -1
View File
@@ -34,7 +34,7 @@ VLLM_SUBCMD_PARSER_EPILOG = (
async def listen_for_disconnect(request: Request) -> None:
"""Returns if a disconnect message is received"""
"""Returns if a disconnect message is received."""
while True:
message = await request.receive()
if message["type"] == "http.disconnect":
+4 -2
View File
@@ -33,7 +33,8 @@ def set_default_fingerprint_mode(
custom_value: str | None = None,
) -> None:
"""Configure the fingerprint mode for subsequent ``get_system_fingerprint``
calls. Called once at server startup."""
calls. Called once at server startup.
"""
global _DEFAULT_MODE, _CUSTOM_VALUE
_DEFAULT_MODE = mode
_CUSTOM_VALUE = custom_value
@@ -41,7 +42,8 @@ def set_default_fingerprint_mode(
def get_system_fingerprint(vllm_config: Any) -> str | None:
"""Return the fingerprint for ``vllm_config`` using the mode configured by
``set_default_fingerprint_mode``."""
``set_default_fingerprint_mode``.
"""
return build_system_fingerprint(vllm_config, _DEFAULT_MODE, _CUSTOM_VALUE)
+17 -18
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Utility functions that create ORCA endpoint load report response headers.
"""
"""Utility functions that create ORCA endpoint load report response headers."""
import json
from collections.abc import Mapping
@@ -16,23 +14,22 @@ logger = init_logger(__name__)
def create_orca_header(
metrics_format: str, named_metrics: list[tuple[str, float]]
) -> Mapping[str, str] | None:
"""
Creates ORCA headers named 'endpoint-load-metrics' in the specified format
"""Creates ORCA headers named 'endpoint-load-metrics' in the specified format
and adds custom metrics to named_metrics.
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
Parameters:
- metrics_format (str): The format of the header ('TEXT', 'JSON').
- named_metrics (List[Tuple[str, float]]): List of tuples with metric names
and their corresponding double values.
Args:
metrics_format (str): The format of the header ('TEXT', 'JSON').
named_metrics (List[Tuple[str, float]]): List of tuples with metric names
and their corresponding double values.
Returns:
- Optional[Mapping[str,str]]: A dictionary with header key as
'endpoint-load-metrics' and values as the ORCA header strings with
format prefix and data in with named_metrics in.
"""
"""
if metrics_format.lower() not in ["text", "json"]:
logger.warning(
"Warning: `%s` format is not supported in the ORCA response header",
@@ -71,15 +68,17 @@ def create_orca_header(
def get_named_metrics_from_prometheus() -> list[tuple[str, float]]:
"""
Collects current metrics from Prometheus and returns some of them
"""Collects current metrics from Prometheus and returns some of them
in the form of the `named_metrics` list for `create_orca_header()`.
Parameters:
Parameters
----------
- None
Returns:
Returns
-------
- list[tuple[str, float]]: List of tuples of metric names and their values.
"""
named_metrics: list[tuple[str, float]] = []
# Map from prometheus metric names to ORCA named metrics.
@@ -98,20 +97,20 @@ def get_named_metrics_from_prometheus() -> list[tuple[str, float]]:
def metrics_header(metrics_format: str) -> Mapping[str, str] | None:
"""
Creates ORCA headers named 'endpoint-load-metrics' in the specified format.
"""Creates ORCA headers named 'endpoint-load-metrics' in the specified format.
Metrics are collected from Prometheus using `get_named_metrics_from_prometheus()`.
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
Parameters:
- metrics_format (str): The format of the header ('TEXT', 'JSON').
Args:
metrics_format (str): The format of the header ('TEXT', 'JSON').
Returns:
- Optional[Mapping[str,str]]: A dictionary with header key as
'endpoint-load-metrics' and values as the ORCA header strings with
format prefix and data in with named_metrics in.
"""
if not metrics_format:
return None
@@ -20,7 +20,6 @@ def maybe_filter_parallel_tool_calls(
choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
) -> _ChatCompletionResponseChoiceT:
"""Filter to first tool call only when parallel_tool_calls is explicitly False."""
if request.parallel_tool_calls is not False:
return choice
@@ -92,7 +92,8 @@ def asr_inter_chunk_separator(
class SpeechToTextBaseServing(GenerateBaseServing):
"""Base class for speech-to-text operations like transcription and
translation."""
translation.
"""
def __init__(
self,
@@ -358,8 +359,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
segment_class: type[SpeechToTextSegment],
start_time: float = 0,
) -> list[SpeechToTextSegment]:
"""
Convert tokens to verbose segments.
"""Convert tokens to verbose segments.
This method expects the model to produce
timestamps as tokens (similar to Whisper).
@@ -427,7 +427,8 @@ class SpeechToTextBaseServing(GenerateBaseServing):
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
) -> T | V | AsyncGenerator[str, None] | ErrorResponse:
"""Base method for speech-to-text operations like transcription and
translation."""
translation.
"""
if request.stream and request.use_beam_search:
return self.create_error_response(
"Streaming is not currently supported with beam search"
@@ -33,6 +33,7 @@ async def read_upload_with_limit(
Raises:
VLLMValidationError: If the file exceeds the configured size limit.
"""
if max_size_mb is None:
max_size_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
@@ -13,14 +13,14 @@ from vllm.utils import random_uuid
class InputAudioBufferAppend(OpenAIBaseModel):
"""Append audio chunk to buffer"""
"""Append audio chunk to buffer."""
type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append"
audio: str # base64-encoded PCM16 @ 16kHz
class InputAudioBufferCommit(OpenAIBaseModel):
"""Process accumulated audio buffer"""
"""Process accumulated audio buffer."""
type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit"
final: bool = False
@@ -28,14 +28,14 @@ class InputAudioBufferCommit(OpenAIBaseModel):
# Server -> Client Events
class SessionUpdate(OpenAIBaseModel):
"""Configure session parameters"""
"""Configure session parameters."""
type: Literal["session.update"] = "session.update"
model: str | None = None
class SessionCreated(OpenAIBaseModel):
"""Connection established notification"""
"""Connection established notification."""
type: Literal["session.created"] = "session.created"
id: str = Field(default_factory=lambda: f"sess-{random_uuid()}")
@@ -43,14 +43,14 @@ class SessionCreated(OpenAIBaseModel):
class TranscriptionDelta(OpenAIBaseModel):
"""Incremental transcription text"""
"""Incremental transcription text."""
type: Literal["transcription.delta"] = "transcription.delta"
delta: str # Incremental text
class TranscriptionDone(OpenAIBaseModel):
"""Final transcription with usage stats"""
"""Final transcription with usage stats."""
type: Literal["transcription.done"] = "transcription.done"
text: str # Complete transcription
@@ -58,7 +58,7 @@ class TranscriptionDone(OpenAIBaseModel):
class ErrorEvent(OpenAIBaseModel):
"""Error notification"""
"""Error notification."""
type: Literal["error"] = "error"
error: str
@@ -68,6 +68,7 @@ class OpenAIServingRealtime(GenerateBaseServing):
Yields:
StreamingInput objects containing audio prompts for the engine
"""
model_config = self.model_config
renderer = self.renderer