Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pydantic models for Anthropic API protocol"""
|
||||
"""Pydantic models for Anthropic API protocol."""
|
||||
|
||||
import time
|
||||
from typing import Annotated, Any, Literal
|
||||
@@ -11,21 +11,21 @@ import vllm.envs as envs
|
||||
|
||||
|
||||
class AnthropicError(BaseModel):
|
||||
"""Error structure for Anthropic API"""
|
||||
"""Error structure for Anthropic API."""
|
||||
|
||||
type: str
|
||||
message: str
|
||||
|
||||
|
||||
class AnthropicErrorResponse(BaseModel):
|
||||
"""Error response structure for Anthropic API"""
|
||||
"""Error response structure for Anthropic API."""
|
||||
|
||||
type: Literal["error"] = "error"
|
||||
error: AnthropicError
|
||||
|
||||
|
||||
class AnthropicUsage(BaseModel):
|
||||
"""Token usage information"""
|
||||
"""Token usage information."""
|
||||
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
@@ -34,7 +34,7 @@ class AnthropicUsage(BaseModel):
|
||||
|
||||
|
||||
class AnthropicContentBlock(BaseModel):
|
||||
"""Content block in message"""
|
||||
"""Content block in message."""
|
||||
|
||||
type: Literal[
|
||||
"text",
|
||||
@@ -65,14 +65,14 @@ class AnthropicContentBlock(BaseModel):
|
||||
|
||||
|
||||
class AnthropicMessage(BaseModel):
|
||||
"""Message structure"""
|
||||
"""Message structure."""
|
||||
|
||||
role: Literal["user", "assistant", "system"]
|
||||
content: str | list[AnthropicContentBlock]
|
||||
|
||||
|
||||
class AnthropicTool(BaseModel):
|
||||
"""Tool definition"""
|
||||
"""Tool definition."""
|
||||
|
||||
name: str
|
||||
description: str | None = None
|
||||
@@ -91,7 +91,7 @@ class AnthropicTool(BaseModel):
|
||||
|
||||
|
||||
class AnthropicToolChoice(BaseModel):
|
||||
"""Tool Choice definition"""
|
||||
"""Tool Choice definition."""
|
||||
|
||||
type: Literal["auto", "any", "tool", "none"]
|
||||
name: str | None = None
|
||||
@@ -105,7 +105,7 @@ class AnthropicToolChoice(BaseModel):
|
||||
|
||||
|
||||
class AnthropicJsonOutputFormat(BaseModel):
|
||||
"""JSON output format configuration"""
|
||||
"""JSON output format configuration."""
|
||||
|
||||
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
|
||||
type: Literal["json_schema"] = "json_schema"
|
||||
@@ -119,7 +119,7 @@ class AnthropicOutputConfig(BaseModel):
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(BaseModel):
|
||||
"""Anthropic Messages API request"""
|
||||
"""Anthropic Messages API request."""
|
||||
|
||||
model: str
|
||||
messages: list[AnthropicMessage]
|
||||
@@ -192,7 +192,7 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
|
||||
|
||||
class AnthropicDelta(BaseModel):
|
||||
"""Delta for streaming responses"""
|
||||
"""Delta for streaming responses."""
|
||||
|
||||
type: (
|
||||
Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
|
||||
@@ -211,7 +211,7 @@ class AnthropicDelta(BaseModel):
|
||||
|
||||
|
||||
class AnthropicStreamEvent(BaseModel):
|
||||
"""Streaming event"""
|
||||
"""Streaming event."""
|
||||
|
||||
type: Literal[
|
||||
"message_start",
|
||||
@@ -232,7 +232,7 @@ class AnthropicStreamEvent(BaseModel):
|
||||
|
||||
|
||||
class AnthropicMessagesResponse(BaseModel):
|
||||
"""Anthropic Messages API response"""
|
||||
"""Anthropic Messages API response."""
|
||||
|
||||
id: str
|
||||
type: Literal["message"] = "message"
|
||||
@@ -265,7 +265,7 @@ class AnthropicContextManagement(BaseModel):
|
||||
|
||||
|
||||
class AnthropicCountTokensRequest(BaseModel):
|
||||
"""Anthropic messages.count_tokens request"""
|
||||
"""Anthropic messages.count_tokens request."""
|
||||
|
||||
model: str
|
||||
messages: list[AnthropicMessage]
|
||||
@@ -291,7 +291,7 @@ class AnthropicCountTokensRequest(BaseModel):
|
||||
|
||||
|
||||
class AnthropicCountTokensResponse(BaseModel):
|
||||
"""Anthropic messages.count_tokens response"""
|
||||
"""Anthropic messages.count_tokens response."""
|
||||
|
||||
input_tokens: int
|
||||
context_management: AnthropicContextManagement | None = None
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# Adapted from
|
||||
# https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/chat_completion/serving.py
|
||||
|
||||
"""Anthropic Messages API serving handler"""
|
||||
"""Anthropic Messages API serving handler."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
@@ -96,7 +96,7 @@ def wrap_data_with_event(data: str, event: str):
|
||||
|
||||
|
||||
class AnthropicServingMessages(OpenAIServingChat):
|
||||
"""Handler for Anthropic Messages API requests"""
|
||||
"""Handler for Anthropic Messages API requests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -196,7 +196,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
*,
|
||||
merge_inline_system: bool = False,
|
||||
) -> ChatCompletionRequest:
|
||||
"""Convert Anthropic message format to OpenAI format"""
|
||||
"""Convert Anthropic message format to OpenAI format."""
|
||||
openai_messages: list[dict[str, Any]] = []
|
||||
|
||||
cls._convert_system_message(
|
||||
@@ -224,7 +224,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
*,
|
||||
merge_inline_system: bool = False,
|
||||
) -> None:
|
||||
"""Convert Anthropic system message to OpenAI format"""
|
||||
"""Convert Anthropic system message to OpenAI format."""
|
||||
system_parts: list[str] = []
|
||||
|
||||
# Top-level system field
|
||||
@@ -278,7 +278,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
*,
|
||||
merge_inline_system: bool = False,
|
||||
) -> None:
|
||||
"""Convert Anthropic messages to OpenAI format"""
|
||||
"""Convert Anthropic messages to OpenAI format."""
|
||||
for msg in messages:
|
||||
# Handle system messages in-place: extract text, strip billing
|
||||
# headers, and only emit if there is real content. This avoids
|
||||
@@ -310,7 +310,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
openai_msg: dict[str, Any],
|
||||
openai_messages: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Convert complex message content blocks"""
|
||||
"""Convert complex message content blocks."""
|
||||
content_parts: list[dict[str, Any]] = []
|
||||
tool_calls: list[dict[str, Any]] = []
|
||||
reasoning_parts: list[str] = []
|
||||
@@ -349,7 +349,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
reasoning_parts: list[str],
|
||||
openai_messages: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Convert individual content block"""
|
||||
"""Convert individual content block."""
|
||||
if block.type == "text" and block.text:
|
||||
content_parts.append({"type": "text", "text": block.text})
|
||||
elif block.type == "image" and block.source:
|
||||
@@ -374,7 +374,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
|
||||
@classmethod
|
||||
def _convert_tool_use_block(cls, block, tool_calls: list[dict[str, Any]]) -> None:
|
||||
"""Convert tool_use block to OpenAI function call format"""
|
||||
"""Convert tool_use block to OpenAI function call format."""
|
||||
tool_call = {
|
||||
"id": block.id or f"call_{int(time.time())}",
|
||||
"type": "function",
|
||||
@@ -393,7 +393,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
openai_messages: list[dict[str, Any]],
|
||||
content_parts: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Convert tool_result block to OpenAI format"""
|
||||
"""Convert tool_result block to OpenAI format."""
|
||||
if role == "user":
|
||||
cls._convert_user_tool_result(block, openai_messages)
|
||||
else:
|
||||
@@ -406,7 +406,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
def _convert_user_tool_result(
|
||||
cls, block, openai_messages: list[dict[str, Any]]
|
||||
) -> None:
|
||||
"""Convert user tool_result with text and image support"""
|
||||
"""Convert user tool_result with text and image support."""
|
||||
tool_text = ""
|
||||
tool_image_urls: list[str] = []
|
||||
tool_reference: list[dict[str, Any]] = []
|
||||
@@ -468,7 +468,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
openai_messages: list[dict[str, Any]],
|
||||
) -> ChatCompletionRequest:
|
||||
"""Build base ChatCompletionRequest"""
|
||||
"""Build base ChatCompletionRequest."""
|
||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
||||
return ChatCompletionRequest(
|
||||
model=anthropic_request.model,
|
||||
@@ -498,7 +498,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
req: ChatCompletionRequest,
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
) -> None:
|
||||
"""Handle output configuration such as output format and effort"""
|
||||
"""Handle output configuration such as output format and effort."""
|
||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
||||
return
|
||||
output_config: AnthropicOutputConfig | None = anthropic_request.output_config
|
||||
@@ -519,7 +519,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
req: ChatCompletionRequest,
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
) -> None:
|
||||
"""Handle streaming configuration"""
|
||||
"""Handle streaming configuration."""
|
||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
||||
return
|
||||
if anthropic_request.stream:
|
||||
@@ -534,7 +534,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
req: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Convert Anthropic tool_choice to OpenAI format"""
|
||||
"""Convert Anthropic tool_choice to OpenAI format."""
|
||||
if anthropic_request.tool_choice is None:
|
||||
req.tool_choice = None
|
||||
return
|
||||
@@ -563,7 +563,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
req: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Convert Anthropic tools to OpenAI format"""
|
||||
"""Convert Anthropic tools to OpenAI format."""
|
||||
if anthropic_request.tools is None:
|
||||
return
|
||||
|
||||
@@ -593,8 +593,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
request: AnthropicMessagesRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> AsyncGenerator[str, None] | AnthropicMessagesResponse | ErrorResponse:
|
||||
"""
|
||||
Messages API similar to Anthropic's API.
|
||||
"""Messages API similar to Anthropic's API.
|
||||
|
||||
See https://docs.anthropic.com/en/api/messages
|
||||
for the API specification. This API mimics the Anthropic messages API.
|
||||
|
||||
@@ -223,9 +223,7 @@ class ChatCompletionContentPartVideoParam(TypedDict, total=False):
|
||||
|
||||
|
||||
class PILImage(BaseModel):
|
||||
"""
|
||||
A PIL.Image.Image object.
|
||||
"""
|
||||
"""A PIL.Image.Image object."""
|
||||
|
||||
image_pil: Image.Image
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
@@ -238,6 +236,7 @@ class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
|
||||
{
|
||||
"image_pil": ImageAsset('cherry_blossom').pil_image
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
image_pil: PILImage | None
|
||||
@@ -256,6 +255,7 @@ class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
|
||||
{
|
||||
"image_url": "https://example.com/image.jpg"
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
image_url: str | None
|
||||
@@ -273,6 +273,7 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
|
||||
{
|
||||
"audio_url": "https://example.com/audio.mp3"
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
audio_url: str | None
|
||||
@@ -285,6 +286,7 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
|
||||
{
|
||||
"video_url": "https://example.com/video.mp4"
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
video_url: str | None
|
||||
@@ -304,6 +306,7 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):
|
||||
"closed": True,
|
||||
"type": "thinking"
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
thinking: Required[str]
|
||||
@@ -324,6 +327,7 @@ class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False):
|
||||
"name": "get_weather",
|
||||
"type": "tool_reference"
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -586,8 +590,7 @@ def _get_embeds_data(
|
||||
|
||||
|
||||
class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
"""
|
||||
Tracks multi-modal items in a given request and ensures that the number
|
||||
"""Tracks multi-modal items in a given request and ensures that the number
|
||||
of multi-modal items in a given request does not exceed the configured
|
||||
maximum per prompt.
|
||||
"""
|
||||
@@ -651,8 +654,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
return get_video_processor_cls_name(self.model_config)
|
||||
|
||||
def add(self, modality: ModalityStr, item: _T) -> str | None:
|
||||
"""
|
||||
Add a multi-modal item to the current prompt and returns the
|
||||
"""Add a multi-modal item to the current prompt and returns the
|
||||
placeholder string to use, if any.
|
||||
|
||||
An optional uuid can be added which serves as a unique identifier of the
|
||||
@@ -663,6 +665,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
pre-computed embeddings that do not go through any HF processor, encoder,
|
||||
or model-specific placeholder logic. The corresponding placeholder string is
|
||||
managed by the parser via `_add_placeholder`, so we return None here.
|
||||
|
||||
"""
|
||||
add_info = self._validate_add(modality)
|
||||
if add_info is None:
|
||||
@@ -791,13 +794,13 @@ def _resolve_items(
|
||||
mm_processor: BaseMultiModalProcessor | None,
|
||||
modality_order: dict[str, list[str]],
|
||||
) -> tuple[MultiModalDataDict, MultiModalUUIDDict]:
|
||||
"""
|
||||
Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`.
|
||||
"""Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`.
|
||||
|
||||
Note:
|
||||
`mm_processor` is `None` for text-only models (no registered HF
|
||||
processor) whose only modality is `prompt_embeds`. Every other
|
||||
modality requires a processor, enforced by the guard below.
|
||||
|
||||
"""
|
||||
if "image" in items_by_modality and "image_embeds" in items_by_modality:
|
||||
raise VLLMValidationError(
|
||||
@@ -1568,7 +1571,6 @@ def _get_full_multimodal_text_prompt(
|
||||
multimodal_content_part_separator: str = "\n",
|
||||
) -> str:
|
||||
"""Combine multimodal prompts for a multimodal language model."""
|
||||
|
||||
# flatten storage to make it looks like
|
||||
# {
|
||||
# "<|image|>": 2,
|
||||
@@ -1687,8 +1689,7 @@ def _collect_extra_fields(part: dict[str, Any]) -> dict[str, Any]:
|
||||
def _parse_chat_message_content_mm_part(
|
||||
part: ChatCompletionContentPartParam,
|
||||
) -> tuple[str, _ContentPart]:
|
||||
"""
|
||||
Parses a given multi-modal content part based on its type.
|
||||
"""Parses a given multi-modal content part based on its type.
|
||||
|
||||
Args:
|
||||
part: A dict containing the content part, with a potential 'type' field.
|
||||
@@ -1700,6 +1701,7 @@ def _parse_chat_message_content_mm_part(
|
||||
|
||||
Raises:
|
||||
ValueError: If the 'type' field is missing and no direct URL is found.
|
||||
|
||||
"""
|
||||
assert isinstance(
|
||||
part, dict
|
||||
|
||||
@@ -22,5 +22,6 @@ class BenchmarkSubcommandBase(CLISubcommand):
|
||||
|
||||
Args:
|
||||
args: The arguments to the command.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The CLI entrypoints of vLLM
|
||||
"""The CLI entrypoints of vLLM.
|
||||
|
||||
Note that all future modules must be lazily loaded within main
|
||||
to avoid certain eager import breakage."""
|
||||
to avoid certain eager import breakage.
|
||||
"""
|
||||
|
||||
import importlib.metadata
|
||||
import sys
|
||||
|
||||
@@ -179,6 +179,7 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
|
||||
"""
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
@@ -236,7 +237,7 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
|
||||
@staticmethod
|
||||
def _get_data_parallel_rank(raw_request: Request | None) -> int | None:
|
||||
"""Pulls the data parallel rank from a header, if provided"""
|
||||
"""Pulls the data parallel rank from a header, if provided."""
|
||||
if raw_request is None:
|
||||
return None
|
||||
|
||||
@@ -294,7 +295,8 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
) -> _T:
|
||||
"""Wrap a `create_*` coroutine so that, if it raises or returns an
|
||||
ErrorResponse (i.e. the request never reached the engine), the KV
|
||||
connector is notified to free any pinned remote-prefill blocks."""
|
||||
connector is notified to free any pinned remote-prefill blocks.
|
||||
"""
|
||||
kv_transfer_params = self.has_kv_connector and request.kv_transfer_params
|
||||
if not kv_transfer_params or not kv_transfer_params.get("do_remote_prefill"):
|
||||
return await awaitable
|
||||
|
||||
@@ -53,7 +53,7 @@ def _bitmask_to_token_ids(bitmask_row: torch.Tensor, vocab_size: int) -> list[in
|
||||
|
||||
|
||||
class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
"""Offline inference for beam search"""
|
||||
"""Offline inference for beam search."""
|
||||
|
||||
def beam_search(
|
||||
self,
|
||||
@@ -63,8 +63,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
use_tqdm: bool = False,
|
||||
concurrency_limit: int | None = None,
|
||||
) -> list[BeamSearchOutput]:
|
||||
"""
|
||||
Generate sequences using beam search.
|
||||
"""Generate sequences using beam search.
|
||||
|
||||
Args:
|
||||
prompts: A list of prompts. Each prompt can be a string or a list
|
||||
@@ -74,6 +73,7 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
use_tqdm: Whether to use tqdm to display the progress bar.
|
||||
concurrency_limit: The maximum number of concurrent requests.
|
||||
If None, the number of concurrent requests is unlimited.
|
||||
|
||||
"""
|
||||
# TODO: how does beam search work together with length penalty,
|
||||
# frequency, penalty, and stopping criteria, etc.?
|
||||
|
||||
@@ -20,7 +20,7 @@ from .utils import BeamSearchSequence, create_sort_beams_key_function
|
||||
|
||||
|
||||
class BeamSearchOnlineMixin(ABC):
|
||||
"""online serving for beam search"""
|
||||
"""online serving for beam search."""
|
||||
|
||||
renderer: BaseRenderer
|
||||
engine_client: EngineClient
|
||||
|
||||
@@ -60,6 +60,7 @@ class GenerativeScoringRequest(OpenAIBaseModel):
|
||||
the full vocab for those ids (False).
|
||||
item_first: If True, prepend items to query. Otherwise append items to query.
|
||||
add_special_tokens: Whether to add special tokens when tokenizing.
|
||||
|
||||
"""
|
||||
|
||||
model: str | None = None
|
||||
@@ -112,6 +113,7 @@ class GenerativeScoringItemResult(OpenAIBaseModel):
|
||||
index: The index of this item in the input items list.
|
||||
object: Type of object, always "score".
|
||||
score: The probability score for the first label token.
|
||||
|
||||
"""
|
||||
|
||||
index: int
|
||||
@@ -129,6 +131,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
|
||||
model: The model used for scoring.
|
||||
data: List of scoring results, one per input item.
|
||||
usage: Token usage information.
|
||||
|
||||
"""
|
||||
|
||||
id: str = Field(default="")
|
||||
@@ -188,6 +191,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
Returns:
|
||||
GenerativeScoringResponse with probabilities for each item, or
|
||||
ErrorResponse if an error occurred.
|
||||
|
||||
"""
|
||||
# Check model
|
||||
error_check_ret = await self._check_model(request) # type: ignore[arg-type]
|
||||
@@ -401,6 +405,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
|
||||
Returns:
|
||||
Tuple of (list of EngineInput, list of prompt token counts).
|
||||
|
||||
"""
|
||||
# Tokenize query if it's a string
|
||||
if isinstance(request.query, str):
|
||||
@@ -455,6 +460,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
|
||||
Returns:
|
||||
Dictionary mapping token_id to probability.
|
||||
|
||||
"""
|
||||
if apply_softmax:
|
||||
# Normalize over only the label tokens (subset softmax)
|
||||
|
||||
@@ -70,14 +70,12 @@ async def build_async_engine_client_from_engine_args(
|
||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
"""
|
||||
Create EngineClient, either:
|
||||
"""Create EngineClient, either:
|
||||
- in-process using the AsyncLLMEngine Directly
|
||||
- multiprocess using AsyncLLMEngine RPC
|
||||
|
||||
Returns the Client or None if the creation failed.
|
||||
"""
|
||||
|
||||
# Create the EngineConfig (determines if we can use V1).
|
||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||
|
||||
@@ -123,7 +121,6 @@ async def build_and_serve(
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
@@ -162,7 +159,6 @@ async def build_and_serve(
|
||||
|
||||
async def run_server(args, **uvicorn_kwargs) -> None:
|
||||
"""Run a single-worker API server."""
|
||||
|
||||
decorate_logs("APIServer", skip_if_decorated=True)
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
|
||||
@@ -180,7 +176,6 @@ async def run_server_worker(
|
||||
listen_address, sock, args, client_config=None, **uvicorn_kwargs
|
||||
) -> None:
|
||||
"""Run a single API server worker."""
|
||||
|
||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file contains the command line arguments for the vLLM's online server.
|
||||
"""This file contains the command line arguments for the vLLM's online server.
|
||||
It is kept in a separate file for documentation purposes.
|
||||
"""
|
||||
|
||||
|
||||
@@ -182,8 +182,7 @@ async def _probe_endpoint(
|
||||
conn_err_failure_threshold: int = 3,
|
||||
conn_err_retry_delay: float = 5.0,
|
||||
) -> bool:
|
||||
"""
|
||||
Probe /health endpoint for 200 status.
|
||||
"""Probe /health endpoint for 200 status.
|
||||
|
||||
If there is a connection error, retry every N seconds.
|
||||
"""
|
||||
@@ -251,9 +250,7 @@ def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
|
||||
|
||||
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
"""
|
||||
Entrypoint function for the vLLM DP Server.
|
||||
"""
|
||||
"""Entrypoint function for the vLLM DP Server."""
|
||||
# Create a fresh process group for the vLLM DP Server,
|
||||
# so that CTRL-C is propagated cleanly.
|
||||
os.setpgrp()
|
||||
@@ -320,8 +317,7 @@ class DPSupervisor:
|
||||
await supervisor_server_task
|
||||
|
||||
async def _start_server(self) -> tuple[uvicorn.Server, asyncio.Task[None]]:
|
||||
"""
|
||||
Launch the DPSupervisor HTTP server.
|
||||
"""Launch the DPSupervisor HTTP server.
|
||||
|
||||
Called only after the vLLM DP Servers are ready so that /health does
|
||||
not return 503 to external probes while the engines are initializing.
|
||||
@@ -366,8 +362,7 @@ class DPSupervisor:
|
||||
return supervisor_server, supervisor_server_task
|
||||
|
||||
async def _wait_until_ready(self, monitor_task: asyncio.Task[None]) -> None:
|
||||
"""
|
||||
Block until the vLLM DP Servers are ready or shutdown is triggered.
|
||||
"""Block until the vLLM DP Servers are ready or shutdown is triggered.
|
||||
|
||||
Returns early if monitoring stops (e.g. a DP Server dies during
|
||||
startup), in which case the supervisor server is never started.
|
||||
@@ -379,14 +374,12 @@ class DPSupervisor:
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
def _handle_signal(self, signum: int) -> None:
|
||||
"""
|
||||
Signal handler that is added to the event loop.
|
||||
"""Signal handler that is added to the event loop.
|
||||
|
||||
This catches the SIGTERM from K8s and begins graceful shutdown,
|
||||
by setting the _shutdown_event(), which is watched by the main
|
||||
coroutine monitoring the vLLM DP Servers.
|
||||
"""
|
||||
|
||||
if self._shutdown_event.is_set():
|
||||
return
|
||||
|
||||
@@ -400,9 +393,7 @@ class DPSupervisor:
|
||||
self._is_ready = False
|
||||
|
||||
def _start_children(self) -> None:
|
||||
"""
|
||||
Launch vLLM DP Servers on separate GPUs.
|
||||
"""
|
||||
"""Launch vLLM DP Servers on separate GPUs."""
|
||||
logger.info("Launching vLLM DP Servers")
|
||||
context = multiprocessing.get_context("spawn")
|
||||
for local_rank in range(self.args.data_parallel_size_local):
|
||||
@@ -416,8 +407,7 @@ class DPSupervisor:
|
||||
self._processes.append(process)
|
||||
|
||||
async def _probe_all_children(self) -> None:
|
||||
"""
|
||||
Background coroutine: probes all child endpoints on each interval.
|
||||
"""Background coroutine: probes all child endpoints on each interval.
|
||||
|
||||
Exits when any server becomes unhealthy after being ready, signalling
|
||||
_monitor_children to initiate shutdown.
|
||||
@@ -474,8 +464,7 @@ class DPSupervisor:
|
||||
)
|
||||
|
||||
async def _monitor_children(self) -> None:
|
||||
"""
|
||||
Main coroutine task that monitors the children vLLM servers.
|
||||
"""Main coroutine task that monitors the children vLLM servers.
|
||||
|
||||
Before the vLLM servers are /ready:
|
||||
- if the pid is dead, we will shut down
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# mypy: ignore-errors
|
||||
"""
|
||||
vLLM gRPC Server
|
||||
"""vLLM gRPC Server
|
||||
|
||||
Starts a gRPC server backed by AsyncLLM, using the VllmEngineServicer
|
||||
from the smg-grpc-servicer package.
|
||||
@@ -16,6 +15,7 @@ Example:
|
||||
--model meta-llama/Llama-2-7b-hf \
|
||||
--host 0.0.0.0 \
|
||||
--port 50051
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -54,11 +54,11 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
async def serve_grpc(args: argparse.Namespace):
|
||||
"""
|
||||
Main gRPC serving function.
|
||||
"""Main gRPC serving function.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
|
||||
"""
|
||||
log_version_and_model(logger, VLLM_VERSION, args.model)
|
||||
logger.info("vLLM gRPC server args: %s", args)
|
||||
|
||||
@@ -53,8 +53,7 @@ async def serve_http(
|
||||
enable_ssl_refresh: bool = False,
|
||||
**uvicorn_kwargs: Any,
|
||||
):
|
||||
"""
|
||||
Start a FastAPI app using Uvicorn, with support for custom Uvicorn config
|
||||
"""Start a FastAPI app using Uvicorn, with support for custom Uvicorn config
|
||||
options. Supports http header limits via h11_max_incomplete_event_size and
|
||||
h11_max_header_count.
|
||||
"""
|
||||
@@ -185,8 +184,7 @@ async def serve_http(
|
||||
|
||||
|
||||
async def watchdog_loop(server: uvicorn.Server, engine: EngineClient):
|
||||
"""
|
||||
# Watchdog task that runs in the background, checking
|
||||
"""# Watchdog task that runs in the background, checking
|
||||
# for error state in the engine. Needed to trigger shutdown
|
||||
# if an exception arises is StreamingResponse() generator.
|
||||
"""
|
||||
@@ -197,8 +195,7 @@ async def watchdog_loop(server: uvicorn.Server, engine: EngineClient):
|
||||
|
||||
|
||||
def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
|
||||
"""
|
||||
See discussions here on shutting down a uvicorn server
|
||||
"""See discussions here on shutting down a uvicorn server
|
||||
https://github.com/encode/uvicorn/discussions/1103
|
||||
In this case we cannot await the server shutdown here
|
||||
because handler must first return to close the connection
|
||||
@@ -254,7 +251,6 @@ def validate_api_server_args(args):
|
||||
@instrument(span_name="API server setup")
|
||||
def setup_server(args, *, reuse_port: bool):
|
||||
"""Validate API server args and create the server socket."""
|
||||
|
||||
log_version_and_model(logger, VLLM_VERSION, args.model)
|
||||
log_non_default_args(args)
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ async def init_render_app_state(
|
||||
preprocessing pipeline (renderer, input_processor)
|
||||
directly from the :class:`~vllm.config.VllmConfig`.
|
||||
"""
|
||||
|
||||
served_model_names = args.served_model_name or [args.model]
|
||||
model_registry = OpenAIModelRegistry(
|
||||
model_config=vllm_config.model_config,
|
||||
|
||||
@@ -30,7 +30,6 @@ async def build_and_serve_renderer(
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
|
||||
@@ -88,8 +88,7 @@ def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
|
||||
|
||||
|
||||
class BatchTranscriptionRequest(TranscriptionRequest):
|
||||
"""
|
||||
Batch transcription request that uses file_url instead of file.
|
||||
"""Batch transcription request that uses file_url instead of file.
|
||||
|
||||
This class extends TranscriptionRequest but replaces the file field
|
||||
with file_url to support batch processing from audio files written in JSON format.
|
||||
@@ -119,8 +118,7 @@ class BatchTranscriptionRequest(TranscriptionRequest):
|
||||
|
||||
|
||||
class BatchTranslationRequest(TranslationRequest):
|
||||
"""
|
||||
Batch translation request that uses file_url instead of file.
|
||||
"""Batch translation request that uses file_url instead of file.
|
||||
|
||||
This class extends TranslationRequest but replaces the file field
|
||||
with file_url to support batch processing from audio files written in JSON format.
|
||||
@@ -160,8 +158,7 @@ BatchRequestInputBody: TypeAlias = (
|
||||
|
||||
|
||||
class BatchRequestInput(OpenAIBaseModel):
|
||||
"""
|
||||
The per-line object of the batch input file.
|
||||
"""The per-line object of the batch input file.
|
||||
|
||||
NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
|
||||
"""
|
||||
@@ -225,9 +222,7 @@ class BatchResponseData(OpenAIBaseModel):
|
||||
|
||||
|
||||
class BatchRequestOutput(OpenAIBaseModel):
|
||||
"""
|
||||
The per-line object of the batch output and error files
|
||||
"""
|
||||
"""The per-line object of the batch output and error files"""
|
||||
|
||||
id: str
|
||||
|
||||
@@ -358,8 +353,7 @@ async def read_file(path_or_url: str) -> str:
|
||||
async def write_local_file(
|
||||
output_path: str, batch_outputs: list[BatchRequestOutput]
|
||||
) -> None:
|
||||
"""
|
||||
Write the responses to a local file.
|
||||
"""Write the responses to a local file.
|
||||
output_path: The path to write the responses to.
|
||||
batch_outputs: The list of batch outputs to write.
|
||||
"""
|
||||
@@ -371,8 +365,7 @@ async def write_local_file(
|
||||
|
||||
|
||||
async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
|
||||
"""
|
||||
Upload a local file to a URL.
|
||||
"""Upload a local file to a URL.
|
||||
output_url: The URL to upload the file to.
|
||||
data_or_file: Either the data to upload or the path to the file to upload.
|
||||
from_file: If True, data_or_file is the path to the file to upload.
|
||||
@@ -429,8 +422,7 @@ async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> No
|
||||
async def write_file(
|
||||
path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
|
||||
) -> None:
|
||||
"""
|
||||
Write batch_outputs to a file or upload to a URL.
|
||||
"""Write batch_outputs to a file or upload to a URL.
|
||||
path_or_url: The path or URL to write batch_outputs to.
|
||||
batch_outputs: The list of batch outputs to write.
|
||||
output_tmp_dir: The directory to store the output file before uploading it
|
||||
@@ -471,8 +463,7 @@ async def download_bytes_from_url(
|
||||
url: str,
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
) -> bytes:
|
||||
"""
|
||||
Download data from a URL or decode from a data URL.
|
||||
"""Download data from a URL or decode from a data URL.
|
||||
|
||||
Args:
|
||||
url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
|
||||
@@ -482,6 +473,7 @@ async def download_bytes_from_url(
|
||||
|
||||
Returns:
|
||||
Data as bytes
|
||||
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
|
||||
@@ -615,8 +607,7 @@ def handle_endpoint_request(
|
||||
handler_getter: Callable[[], Callable | None],
|
||||
wrapper_fn: WrapperFn | None = None,
|
||||
) -> Awaitable[BatchRequestOutput] | None:
|
||||
"""
|
||||
Generic handler for endpoint requests.
|
||||
"""Generic handler for endpoint requests.
|
||||
|
||||
Args:
|
||||
request: The batch request input
|
||||
@@ -628,6 +619,7 @@ def handle_endpoint_request(
|
||||
Returns:
|
||||
Awaitable[BatchRequestOutput] if the request was handled,
|
||||
None if URL didn't match
|
||||
|
||||
"""
|
||||
if not url_matcher(request.url):
|
||||
return None
|
||||
@@ -649,8 +641,7 @@ def make_transcription_wrapper(
|
||||
is_translation: bool,
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
) -> WrapperFn:
|
||||
"""
|
||||
Factory function to create a wrapper for transcription/translation handlers.
|
||||
"""Factory function to create a wrapper for transcription/translation handlers.
|
||||
The wrapper converts BatchTranscriptionRequest or BatchTranslationRequest
|
||||
to TranscriptionRequest or TranslationRequest and calls the appropriate handler.
|
||||
|
||||
@@ -662,6 +653,7 @@ def make_transcription_wrapper(
|
||||
|
||||
Returns:
|
||||
A function that takes a handler and returns a wrapped handler
|
||||
|
||||
"""
|
||||
|
||||
def wrapper(handler_fn: Callable):
|
||||
@@ -723,8 +715,7 @@ async def build_endpoint_registry(
|
||||
engine_client: EngineClient,
|
||||
args: Namespace,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""
|
||||
Build the endpoint registry with all serving objects and handler configurations.
|
||||
"""Build the endpoint registry with all serving objects and handler configurations.
|
||||
|
||||
Args:
|
||||
engine_client: The engine client
|
||||
@@ -732,6 +723,7 @@ async def build_endpoint_registry(
|
||||
|
||||
Returns:
|
||||
Dictionary mapping endpoint keys to their configurations
|
||||
|
||||
"""
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
|
||||
@@ -29,8 +29,7 @@ def load_log_config(log_config_file: str | None) -> dict | None:
|
||||
|
||||
|
||||
def get_uvicorn_log_config(args: Namespace) -> dict | None:
|
||||
"""
|
||||
Get the uvicorn log config based on the provided arguments.
|
||||
"""Get the uvicorn log config based on the provided arguments.
|
||||
|
||||
Priority:
|
||||
1. If log_config_file is specified, use it
|
||||
|
||||
+22
-16
@@ -174,6 +174,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
Note:
|
||||
This class is intended to be used for offline inference. For online
|
||||
serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -224,7 +225,6 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""LLM constructor."""
|
||||
|
||||
if "disable_log_stats" not in kwargs:
|
||||
kwargs["disable_log_stats"] = True
|
||||
|
||||
@@ -398,6 +398,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
Returns:
|
||||
The world size (tensor_parallel_size * pipeline_parallel_size),
|
||||
optionally multiplied by data_parallel_size if include_dp is True.
|
||||
|
||||
"""
|
||||
parallel_config = self.llm_engine.vllm_config.parallel_config
|
||||
if include_dp:
|
||||
@@ -457,6 +458,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
Returns:
|
||||
A list of `RequestOutput` objects containing the
|
||||
generated completions in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
runner_type = self.model_config.runner_type
|
||||
if runner_type != "generate":
|
||||
@@ -507,6 +509,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
|
||||
Returns:
|
||||
A list of request IDs for the enqueued requests.
|
||||
|
||||
"""
|
||||
runner_type = self.model_config.runner_type
|
||||
if runner_type != "generate":
|
||||
@@ -558,6 +561,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
|
||||
Returns:
|
||||
A list of output objects for all completed requests.
|
||||
|
||||
"""
|
||||
if output_type is None:
|
||||
output_type = (RequestOutput, PoolingRequestOutput)
|
||||
@@ -571,8 +575,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
args: tuple = (),
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
"""
|
||||
Execute an RPC call on all workers.
|
||||
"""Execute an RPC call on all workers.
|
||||
|
||||
Args:
|
||||
method: Name of the worker method to execute, or a callable that
|
||||
@@ -592,13 +595,12 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
Note:
|
||||
It is recommended to use this API to only pass control messages,
|
||||
and set up data-plane communication to pass data.
|
||||
"""
|
||||
|
||||
"""
|
||||
return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
|
||||
|
||||
def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
|
||||
"""
|
||||
Run a function directly on the model inside each worker,
|
||||
"""Run a function directly on the model inside each worker,
|
||||
returning the result for each of them.
|
||||
|
||||
!!! warning
|
||||
@@ -625,8 +627,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[RequestOutput]:
|
||||
"""
|
||||
Generate responses for a chat conversation.
|
||||
"""Generate responses for a chat conversation.
|
||||
|
||||
The chat conversation is converted into a text prompt using the
|
||||
tokenizer and calls the [generate][vllm.LLM.generate] method to generate
|
||||
@@ -670,10 +671,12 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
template.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
mm_processor_kwargs: Overrides for `processor.__call__`.
|
||||
tools: Tools to make available to the model, if any.
|
||||
|
||||
Returns:
|
||||
A list of `RequestOutput` objects containing the generated
|
||||
responses in the same order as the input messages.
|
||||
|
||||
"""
|
||||
model_config = self.model_config
|
||||
runner_type = model_config.runner_type
|
||||
@@ -751,6 +754,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
|
||||
Returns:
|
||||
A list of request IDs for the enqueued requests.
|
||||
|
||||
"""
|
||||
model_config = self.model_config
|
||||
runner_type = model_config.runner_type
|
||||
@@ -787,6 +791,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
profile_prefix: Optional prefix for the trace file names. If provided,
|
||||
trace files will be named as "<prefix>_dp<X>_pp<Y>_tp<Z>".
|
||||
If not provided, default naming will be used.
|
||||
|
||||
"""
|
||||
self.llm_engine.start_profile(profile_prefix)
|
||||
|
||||
@@ -801,8 +806,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
)
|
||||
|
||||
def sleep(self, level: int = 1, mode: PauseMode = "abort"):
|
||||
"""
|
||||
Put the engine to sleep. The engine should not process any requests.
|
||||
"""Put the engine to sleep. The engine should not process any requests.
|
||||
The caller should guarantee that no requests are being processed
|
||||
during the sleep period, before `wake_up` is called.
|
||||
|
||||
@@ -822,12 +826,12 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
CPU memory pressure.
|
||||
mode: How to handle any existing requests, can be "abort", "wait",
|
||||
or "keep".
|
||||
|
||||
"""
|
||||
self.llm_engine.sleep(level=level, mode=mode)
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
"""
|
||||
Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
|
||||
"""Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
|
||||
method for more details.
|
||||
|
||||
Args:
|
||||
@@ -837,6 +841,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
is reallocated. wake_up should be called with all tags
|
||||
(or None) before the engine is used again.
|
||||
Use tags=["scheduling"] to resume from level 0 sleep.
|
||||
|
||||
"""
|
||||
self.llm_engine.wake_up(tags)
|
||||
|
||||
@@ -849,17 +854,18 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
|
||||
Note:
|
||||
This method is only available with the V1 LLM engine.
|
||||
|
||||
"""
|
||||
return self.llm_engine.get_metrics()
|
||||
|
||||
def init_weight_transfer_engine(
|
||||
self, request: WeightTransferInitRequest | dict
|
||||
) -> None:
|
||||
"""
|
||||
Initialize weight transfer for RL training.
|
||||
"""Initialize weight transfer for RL training.
|
||||
|
||||
Args:
|
||||
request: Weight transfer initialization request with backend-specific info
|
||||
|
||||
"""
|
||||
init_info_dict = (
|
||||
request["init_info"] if isinstance(request, dict) else request.init_info
|
||||
@@ -878,11 +884,11 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
self.llm_engine.collective_rpc("start_draft_weight_update")
|
||||
|
||||
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
|
||||
"""
|
||||
Update the weights of the model.
|
||||
"""Update the weights of the model.
|
||||
|
||||
Args:
|
||||
request: Weight update request with backend-specific update info
|
||||
|
||||
"""
|
||||
update_info_dict = (
|
||||
request["update_info"] if isinstance(request, dict) else request.update_info
|
||||
|
||||
@@ -23,8 +23,7 @@ MIN_GPT_OSS_VERSION = "0.0.7"
|
||||
|
||||
|
||||
def validate_gpt_oss_install():
|
||||
"""
|
||||
Check if the gpt-oss is installed and its version is at least 0.0.7.
|
||||
"""Check if the gpt-oss is installed and its version is at least 0.0.7.
|
||||
If not, raise an ImportError.
|
||||
"""
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
@@ -151,8 +150,7 @@ class HarmonyPythonTool(Tool):
|
||||
return tool_output_msgs
|
||||
|
||||
async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
|
||||
"""
|
||||
This function converts parsable context types to harmony and
|
||||
"""This function converts parsable context types to harmony and
|
||||
back so we can use GPTOSS demo python tool
|
||||
"""
|
||||
from vllm.entrypoints.openai.responses.context import ParsableContext
|
||||
|
||||
@@ -76,17 +76,14 @@ def post_process_tools_description(
|
||||
class ToolServer(ABC):
|
||||
@abstractmethod
|
||||
def has_tool(self, tool_name: str) -> bool:
|
||||
"""
|
||||
Return True if the tool is supported, False otherwise.
|
||||
"""
|
||||
"""Return True if the tool is supported, False otherwise."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_tool_description(
|
||||
self, tool_name: str, allowed_tools: list[str] | None = None
|
||||
) -> ToolNamespaceConfig | None:
|
||||
"""
|
||||
Return the tool description for the given tool name.
|
||||
"""Return the tool description for the given tool name.
|
||||
If the tool is not supported, return None.
|
||||
"""
|
||||
pass
|
||||
@@ -95,9 +92,7 @@ class ToolServer(ABC):
|
||||
def new_session(
|
||||
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
|
||||
) -> AbstractAsyncContextManager[Any]:
|
||||
"""
|
||||
Create a session for the tool.
|
||||
"""
|
||||
"""Create a session for the tool."""
|
||||
...
|
||||
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ _R = TypeVar("_R", default=Any)
|
||||
|
||||
|
||||
class OfflineInferenceMixin:
|
||||
"""Offline inference utils"""
|
||||
"""Offline inference utils."""
|
||||
|
||||
request_counter: Counter
|
||||
renderer: BaseRenderer
|
||||
@@ -113,14 +113,14 @@ class OfflineInferenceMixin:
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
"""
|
||||
Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
|
||||
"""Convert prompt inputs from LLM APIs (other than [LLM.chat][]) into
|
||||
a format that can be passed to `_add_request`.
|
||||
|
||||
Refer to [LLM.generate][] for a complete description of the arguments.
|
||||
|
||||
Returns:
|
||||
A list of `EngineInput` objects ready to be passed into LLMEngine.
|
||||
|
||||
"""
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
@@ -168,14 +168,14 @@ class OfflineInferenceMixin:
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
"""
|
||||
Convert a list of conversations into prompts so that they can then
|
||||
"""Convert a list of conversations into prompts so that they can then
|
||||
be used as input for other LLM APIs.
|
||||
|
||||
Refer to [LLM.chat][] for a complete description of the arguments.
|
||||
|
||||
Returns:
|
||||
A list of `EngineInput` objects ready to be passed into LLMEngine.
|
||||
|
||||
"""
|
||||
renderer = self.renderer
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
Returns:
|
||||
A tuple of (all_conversations, engine_prompts) on success — one
|
||||
entry per conversation — or an ErrorResponse on failure.
|
||||
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
|
||||
@@ -220,8 +220,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
||||
"""
|
||||
Validate the model and preprocess a chat completion request.
|
||||
"""Validate the model and preprocess a chat completion request.
|
||||
|
||||
Delegates preprocessing logic to OnlineRenderer, adding the
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
@@ -229,6 +228,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
Returns:
|
||||
A tuple of (conversation, engine_inputs) on success,
|
||||
or an ErrorResponse on failure.
|
||||
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
@@ -244,8 +244,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
request: ChatCompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
|
||||
"""
|
||||
Chat Completion API similar to OpenAI's API.
|
||||
"""Chat Completion API similar to OpenAI's API.
|
||||
|
||||
See https://platform.openai.com/docs/api-reference/chat/create
|
||||
for the API specification. This API mimics the OpenAI
|
||||
|
||||
@@ -89,14 +89,14 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
) -> list[EngineInput] | ErrorResponse:
|
||||
"""
|
||||
Validate the model and preprocess a completion request.
|
||||
"""Validate the model and preprocess a completion request.
|
||||
|
||||
Delegates preprocessing logic to OnlineRenderer, adding the
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
|
||||
Returns:
|
||||
A list of engine_inputs on success, or an ErrorResponse on failure.
|
||||
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
|
||||
@@ -123,7 +123,8 @@ class OpenAIServingModels:
|
||||
|
||||
async def init_static_loras(self):
|
||||
"""Loads all static LoRA modules.
|
||||
Raises if any fail to load"""
|
||||
Raises if any fail to load
|
||||
"""
|
||||
if self.static_lora_modules is None:
|
||||
return
|
||||
for lora in self.static_lora_modules:
|
||||
@@ -148,7 +149,8 @@ class OpenAIServingModels:
|
||||
|
||||
async def show_available_models(self) -> ModelList:
|
||||
"""Show available models. This includes the base model and all
|
||||
adapters."""
|
||||
adapters.
|
||||
"""
|
||||
model_list = await self.registry.show_available_models()
|
||||
lora_cards = [
|
||||
ModelCard(
|
||||
@@ -289,6 +291,7 @@ class OpenAIServingModels:
|
||||
LoRARequest if found and loaded successfully.
|
||||
ErrorResponse (404) if no resolver finds the adapter.
|
||||
ErrorResponse (400) if adapter(s) are found but none load.
|
||||
|
||||
"""
|
||||
async with self.lora_resolver_lock[lora_name]:
|
||||
# First check if this LoRA is already loaded
|
||||
|
||||
@@ -88,8 +88,7 @@ MCP_BUILTIN_TOOLS: set[str] = set(BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())
|
||||
|
||||
|
||||
def has_custom_tools(tool_types: set[str]) -> bool:
|
||||
"""
|
||||
Checks if the given tool types are custom tools
|
||||
"""Checks if the given tool types are custom tools
|
||||
(i.e. any tool other than MCP builtin tools)
|
||||
"""
|
||||
return not tool_types.issubset(MCP_BUILTIN_TOOLS)
|
||||
@@ -209,8 +208,7 @@ def get_system_or_developer_message(role: str, instructions: str) -> Message:
|
||||
|
||||
|
||||
def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
|
||||
"""
|
||||
Parse a list of messages from request.messages in the Chat Completion API to
|
||||
"""Parse a list of messages from request.messages in the Chat Completion API to
|
||||
Harmony messages.
|
||||
"""
|
||||
msgs: list[Message] = []
|
||||
@@ -230,9 +228,8 @@ def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
|
||||
|
||||
|
||||
def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]:
|
||||
"""
|
||||
Harmony models expect the analysis messages (representing raw chain of thought) to
|
||||
be dropped after an assistant message to the final channel is produced from the
|
||||
"""Harmony models expect the analysis messages (representing raw chain of thought)
|
||||
to be dropped after an assistant message to the final channel is produced from the
|
||||
reasoning of those messages.
|
||||
|
||||
The openai-harmony library does this if the very last assistant message is to the
|
||||
@@ -262,8 +259,7 @@ def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]:
|
||||
|
||||
|
||||
def flatten_input_text_content(content: Any) -> str | None:
|
||||
"""
|
||||
Extract text parts from a Chat Completion or Responses API content field and
|
||||
"""Extract text parts from a Chat Completion or Responses API content field and
|
||||
flatten them into a single string. Returns None if no text content is found.
|
||||
"""
|
||||
if content is None or isinstance(content, str):
|
||||
@@ -286,8 +282,7 @@ def flatten_input_text_content(content: Any) -> str | None:
|
||||
def extract_instructions_from_messages(
|
||||
messages: Sequence[Any],
|
||||
) -> tuple[str | None, list[Any]]:
|
||||
"""
|
||||
Peel a leading system/developer Chat Completion or Responses message and
|
||||
"""Peel a leading system/developer Chat Completion or Responses message and
|
||||
flatten its instruction text.
|
||||
"""
|
||||
remaining_messages = list(messages)
|
||||
@@ -327,9 +322,7 @@ def build_harmony_preamble(
|
||||
container_description: str | None = None,
|
||||
with_custom_tools: bool = False,
|
||||
) -> list[Message]:
|
||||
"""
|
||||
Build the standard Harmony system/developer prefix for a request.
|
||||
"""
|
||||
"""Build the standard Harmony system/developer prefix for a request."""
|
||||
developer_instructions = system_instructions = None
|
||||
if envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
|
||||
system_instructions = instructions
|
||||
@@ -359,8 +352,7 @@ def build_harmony_preamble(
|
||||
def parse_chat_input_to_harmony_message(
|
||||
chat_msg, tool_id_names: dict[str, str] | None = None
|
||||
) -> list[Message]:
|
||||
"""
|
||||
Parse a message from request.messages in the Chat Completion API to
|
||||
"""Parse a message from request.messages in the Chat Completion API to
|
||||
Harmony messages.
|
||||
"""
|
||||
tool_id_names = tool_id_names or {}
|
||||
|
||||
@@ -34,7 +34,7 @@ def responses(request: Request) -> OpenAIServingResponses | None:
|
||||
async def _convert_stream_to_sse_events(
|
||||
generator: AsyncGenerator[StreamingResponsesResponse, None],
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Convert the generator to a stream of events in SSE format"""
|
||||
"""Convert the generator to a stream of events in SSE format."""
|
||||
async for event in generator:
|
||||
event_type = getattr(event, "type", "unknown")
|
||||
# https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
|
||||
|
||||
@@ -141,9 +141,7 @@ class ConversationContext(ABC):
|
||||
def _create_json_parse_error_messages(
|
||||
last_msg: Message, e: json.JSONDecodeError
|
||||
) -> list[Message]:
|
||||
"""
|
||||
Creates an error message when json parse failed.
|
||||
"""
|
||||
"""Creates an error message when json parse failed."""
|
||||
error_msg = (
|
||||
f"Error parsing tool arguments as JSON: {str(e)}. "
|
||||
"Please ensure the tool call arguments are valid JSON and try again."
|
||||
@@ -161,7 +159,7 @@ def _create_json_parse_error_messages(
|
||||
|
||||
|
||||
class SimpleContext(ConversationContext):
|
||||
"""This is a context that cannot handle MCP tool calls"""
|
||||
"""This is a context that cannot handle MCP tool calls."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -410,7 +408,8 @@ class ParsableContext(ConversationContext):
|
||||
|
||||
def need_builtin_tool_call(self) -> bool:
|
||||
"""Return true if the last message is a builtin tool call
|
||||
that the request has enabled."""
|
||||
that the request has enabled.
|
||||
"""
|
||||
last_message = self.response_messages[-1]
|
||||
if last_message.type != "function_call":
|
||||
return False
|
||||
@@ -474,8 +473,7 @@ class ParsableContext(ConversationContext):
|
||||
async def call_container_tool(
|
||||
self, tool_session: Union["ClientSession", Tool], last_msg: Message
|
||||
) -> list[Message]:
|
||||
"""
|
||||
Call container tool. Expect this to be run in a stateful docker
|
||||
"""Call container tool. Expect this to be run in a stateful docker
|
||||
with command line terminal.
|
||||
The official container tool would at least
|
||||
expect the following format:
|
||||
@@ -583,7 +581,7 @@ class ParsableContext(ConversationContext):
|
||||
exit_stack.push_async_exit(self.cleanup_session)
|
||||
|
||||
async def cleanup_session(self, *args, **kwargs) -> None:
|
||||
"""Can be used as coro to used in __aexit__"""
|
||||
"""Can be used as coro to used in __aexit__."""
|
||||
|
||||
async def cleanup_tool_session(tool_session):
|
||||
if not isinstance(tool_session, Tool):
|
||||
@@ -693,6 +691,7 @@ class HarmonyContext(ConversationContext):
|
||||
|
||||
Args:
|
||||
output: The RequestOutput containing prompt token information
|
||||
|
||||
"""
|
||||
if output.prompt_token_ids is not None:
|
||||
this_turn_input_tokens = len(output.prompt_token_ids)
|
||||
@@ -757,6 +756,7 @@ class HarmonyContext(ConversationContext):
|
||||
|
||||
Returns:
|
||||
int: Number of output tokens processed in this call
|
||||
|
||||
"""
|
||||
updated_output_token_count = 0
|
||||
if output.outputs:
|
||||
@@ -884,8 +884,7 @@ class HarmonyContext(ConversationContext):
|
||||
async def call_container_tool(
|
||||
self, tool_session: Union["ClientSession", Tool], last_msg: Message
|
||||
) -> list[Message]:
|
||||
"""
|
||||
Call container tool. Expect this to be run in a stateful docker
|
||||
"""Call container tool. Expect this to be run in a stateful docker
|
||||
with command line terminal.
|
||||
The official container tool would at least
|
||||
expect the following format:
|
||||
@@ -925,7 +924,7 @@ class HarmonyContext(ConversationContext):
|
||||
]
|
||||
|
||||
async def cleanup_session(self, *args, **kwargs) -> None:
|
||||
"""Can be used as coro to used in __aexit__"""
|
||||
"""Can be used as coro to used in __aexit__."""
|
||||
|
||||
async def cleanup_tool_session(tool_session):
|
||||
if not isinstance(tool_session, Tool):
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Harmony ↔ Responses API conversion utilities.
|
||||
"""Harmony ↔ Responses API conversion utilities.
|
||||
|
||||
Handles two directions:
|
||||
1. Response Input → Harmony Messages (input parsing)
|
||||
@@ -53,7 +52,8 @@ logger = init_logger(__name__)
|
||||
|
||||
def _parse_harmony_format_message(chat_msg: dict) -> Message:
|
||||
"""Reconstruct a Message from Harmony-format dict,
|
||||
preserving channel, recipient, and content_type."""
|
||||
preserving channel, recipient, and content_type.
|
||||
"""
|
||||
author_dict = chat_msg["author"]
|
||||
role = author_dict.get("role")
|
||||
name = author_dict.get("name")
|
||||
|
||||
@@ -100,9 +100,7 @@ class ResponseUsage(OpenAIBaseModel):
|
||||
|
||||
|
||||
def serialize_message(msg):
|
||||
"""
|
||||
Serializes a single message
|
||||
"""
|
||||
"""Serializes a single message."""
|
||||
if isinstance(msg, dict):
|
||||
return msg
|
||||
elif hasattr(msg, "to_dict"):
|
||||
@@ -113,15 +111,14 @@ def serialize_message(msg):
|
||||
|
||||
|
||||
def serialize_messages(msgs):
|
||||
"""
|
||||
Serializes multiple messages
|
||||
"""
|
||||
"""Serializes multiple messages."""
|
||||
return [serialize_message(msg) for msg in msgs] if msgs else None
|
||||
|
||||
|
||||
class ResponseRawMessageAndToken(OpenAIBaseModel):
|
||||
"""Class to show the raw message.
|
||||
If message / tokens diverge, tokens is the source of truth"""
|
||||
If message / tokens diverge, tokens is the source of truth
|
||||
"""
|
||||
|
||||
message: str
|
||||
tokens: list[int]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Streaming SSE event builders for the Responses API.
|
||||
"""Streaming SSE event builders for the Responses API.
|
||||
|
||||
Pure functions that translate streaming state + delta data into
|
||||
OpenAI Response API SSE events. Used by the streaming event
|
||||
@@ -125,8 +124,7 @@ def is_mcp_tool_by_namespace(
|
||||
recipient: str | None,
|
||||
allowed_function_tool_names: frozenset[str] | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Determine if a tool call is an MCP tool based on recipient prefix.
|
||||
"""Determine if a tool call is an MCP tool based on recipient prefix.
|
||||
|
||||
Inverse of :func:`is_function_recipient` — everything that is not
|
||||
a function call is an MCP tool.
|
||||
@@ -1145,8 +1143,7 @@ def split_delta(delta: DeltaMessage) -> list[DeltaMessage]:
|
||||
|
||||
|
||||
class SimpleStreamingEventProcessor:
|
||||
"""
|
||||
State-machine processor for the simple (non-Harmony) streaming path.
|
||||
"""State-machine processor for the simple (non-Harmony) streaming path.
|
||||
|
||||
Core flow:
|
||||
1. Resolve the target state from the delta_message
|
||||
@@ -1188,8 +1185,7 @@ class SimpleStreamingEventProcessor:
|
||||
def resolve_target_state(
|
||||
self, delta_message: DeltaMessage
|
||||
) -> tuple[_StateType, Any]:
|
||||
"""
|
||||
Decide which state the next delta belongs to.
|
||||
"""Decide which state the next delta belongs to.
|
||||
|
||||
Priority: TOOL_CALL > REASONING > CONTENT, fallback to NONE.
|
||||
For TOOL_CALL the first tool_call object is also returned so
|
||||
@@ -1207,8 +1203,7 @@ class SimpleStreamingEventProcessor:
|
||||
return _StateType.NONE, None
|
||||
|
||||
def needs_transition(self, target_state: _StateType, tool_call: Any) -> bool:
|
||||
"""
|
||||
Return True when we must close the current state and open a new one.
|
||||
"""Return True when we must close the current state and open a new one.
|
||||
|
||||
Two cases trigger a transition:
|
||||
1. The target state differs from the current state
|
||||
|
||||
@@ -115,8 +115,7 @@ def build_response_output_items(
|
||||
def should_continue_final_message(
|
||||
request_input: str | list[ResponseInputOutputItem],
|
||||
) -> bool:
|
||||
"""
|
||||
Determine if the last input message is a partial assistant message
|
||||
"""Determine if the last input message is a partial assistant message
|
||||
that should be continued rather than starting a new generation.
|
||||
|
||||
This enables partial message completion similar to Anthropic's Messages API,
|
||||
@@ -132,6 +131,7 @@ def should_continue_final_message(
|
||||
|
||||
Returns:
|
||||
True if the final message should be continued, False otherwise
|
||||
|
||||
"""
|
||||
if isinstance(request_input, str):
|
||||
# Simple string input is always a user message
|
||||
@@ -226,8 +226,7 @@ def _construct_message_from_response_item(
|
||||
item: ResponseInputOutputItem,
|
||||
prev_msg: ChatCompletionMessageParam | None = None,
|
||||
) -> ChatCompletionMessageParam | None:
|
||||
"""
|
||||
Returns a new message or None. If `None`, `prev_msg` might be updated.
|
||||
"""Returns a new message or None. If `None`, `prev_msg` might be updated.
|
||||
If `prev_msg` is `None`, a new message is always returned.
|
||||
"""
|
||||
prev_assistant_msg = (
|
||||
@@ -360,9 +359,7 @@ def extract_function_tool_names(tools: list[Tool]) -> frozenset[str]:
|
||||
|
||||
|
||||
def extract_tool_types(tools: list[Tool]) -> set[str]:
|
||||
"""
|
||||
Extracts the tool types from the given tools.
|
||||
"""
|
||||
"""Extracts the tool types from the given tools."""
|
||||
tool_types: set[str] = set()
|
||||
for tool in tools:
|
||||
if tool.type == "mcp":
|
||||
@@ -379,8 +376,7 @@ def extract_tool_types(tools: list[Tool]) -> set[str]:
|
||||
def convert_tool_responses_to_completions_format(
|
||||
tool: dict,
|
||||
) -> ChatCompletionToolsParam:
|
||||
"""
|
||||
Convert a flat Responses tool schema:
|
||||
"""Convert a flat Responses tool schema:
|
||||
{"type": "function", "name": "...", "description": "...", "parameters": {...}}
|
||||
into a Chat Completions tool param for chat-template rendering.
|
||||
"""
|
||||
|
||||
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
"""Offline inference for pooling models"""
|
||||
"""Offline inference for pooling models."""
|
||||
|
||||
runner_type: str
|
||||
chat_template: str | None
|
||||
@@ -93,8 +93,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
Returns:
|
||||
A list of `PoolingRequestOutput` objects containing the
|
||||
pooled hidden states in the same order as the input prompts.
|
||||
"""
|
||||
|
||||
"""
|
||||
if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
|
||||
raise ValueError(
|
||||
"The 'data' field is only supported for the 'plugin' pooling task."
|
||||
@@ -203,8 +203,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[EmbeddingRequestOutput]:
|
||||
"""
|
||||
Generate an embedding vector for each prompt.
|
||||
"""Generate an embedding vector for each prompt.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
@@ -226,8 +225,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
Returns:
|
||||
A list of `EmbeddingRequestOutput` objects containing the
|
||||
embedding vectors in the same order as the input prompts.
|
||||
"""
|
||||
|
||||
"""
|
||||
items = self.encode(
|
||||
prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
@@ -248,8 +247,7 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[ClassificationRequestOutput]:
|
||||
"""
|
||||
Generate class logits for each prompt.
|
||||
"""Generate class logits for each prompt.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
@@ -271,8 +269,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
Returns:
|
||||
A list of `ClassificationRequestOutput` objects containing the
|
||||
embedding vectors in the same order as the input prompts.
|
||||
"""
|
||||
|
||||
"""
|
||||
items = self.encode(
|
||||
prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
@@ -330,11 +328,12 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
chat_template: The chat template to use for the scoring. If None, we
|
||||
use the model's default chat template.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `ScoringRequestOutput` objects containing the
|
||||
generated scores in the same order as the input prompts.
|
||||
"""
|
||||
|
||||
"""
|
||||
if self.runner_type != "pooling":
|
||||
raise ValueError(
|
||||
"LLM.score() is only supported for pooling models. "
|
||||
|
||||
@@ -44,7 +44,8 @@ class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
|
||||
|
||||
class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
"""IO Processor plugins are a feature that allows pre- and post-processing
|
||||
of the model input and output for pooling models."""
|
||||
of the model input and output for pooling models.
|
||||
"""
|
||||
|
||||
name = "plugin"
|
||||
|
||||
|
||||
@@ -21,8 +21,7 @@ ScoreContentPartParam: TypeAlias = (
|
||||
|
||||
|
||||
class ScoreMultiModalParam(TypedDict, total=False):
|
||||
"""
|
||||
A specialized parameter type for scoring multimodal content
|
||||
"""A specialized parameter type for scoring multimodal content.
|
||||
|
||||
The reasons why don't reuse `CustomChatCompletionMessageParam` directly:
|
||||
1. Score tasks don't need the 'role' field (user/assistant/system) that's required in chat completions
|
||||
|
||||
@@ -58,8 +58,7 @@ def truncate_text_to_tokens(
|
||||
|
||||
|
||||
def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Compute ColBERT MaxSim score.
|
||||
"""Compute ColBERT MaxSim score.
|
||||
|
||||
Args:
|
||||
q_emb: Query token embeddings [query_len, dim]
|
||||
@@ -67,6 +66,7 @@ def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tens
|
||||
|
||||
Returns:
|
||||
MaxSim score (sum over query tokens of max similarity to any doc token)
|
||||
|
||||
"""
|
||||
# compute in float32 for numerical stability
|
||||
# [query_len, doc_len]
|
||||
@@ -249,8 +249,7 @@ def parse_score_data(
|
||||
|
||||
|
||||
def compress_token_type_ids(token_type_ids: list[int]) -> int:
|
||||
"""
|
||||
Return position of the first 1 or the length of the list
|
||||
"""Return position of the first 1 or the length of the list
|
||||
if not found.
|
||||
"""
|
||||
first_one = len(token_type_ids)
|
||||
|
||||
@@ -78,8 +78,7 @@ def attach_router(app: FastAPI):
|
||||
|
||||
@router.post("/abort_requests")
|
||||
async def abort_requests(raw_request: Request):
|
||||
"""
|
||||
Abort one or more requests. To be used in a
|
||||
"""Abort one or more requests. To be used in a
|
||||
Disaggregated Everything setup.
|
||||
"""
|
||||
try:
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Encode/decode utilities for multimodal tensors and field metadata
|
||||
over JSON/HTTP, used by the disaggregated generate endpoint."""
|
||||
over JSON/HTTP, used by the disaggregated generate endpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
+4
-6
@@ -23,8 +23,7 @@ async def reset_prefix_cache(
|
||||
reset_running_requests: bool = Query(default=False),
|
||||
reset_external: bool = Query(default=False),
|
||||
):
|
||||
"""
|
||||
Reset the local prefix cache.
|
||||
"""Reset the local prefix cache.
|
||||
|
||||
Optionally, if the query parameter `reset_external=true`
|
||||
also resets the external (connector-managed) prefix cache.
|
||||
@@ -35,6 +34,7 @@ async def reset_prefix_cache(
|
||||
|
||||
Example:
|
||||
POST /reset_prefix_cache?reset_external=true
|
||||
|
||||
"""
|
||||
logger.info("Resetting prefix cache...")
|
||||
|
||||
@@ -46,8 +46,7 @@ async def reset_prefix_cache(
|
||||
|
||||
@router.post("/reset_mm_cache")
|
||||
async def reset_mm_cache(raw_request: Request):
|
||||
"""
|
||||
Reset the multi-modal cache. Note that we currently do not check if the
|
||||
"""Reset the multi-modal cache. Note that we currently do not check if the
|
||||
multi-modal cache is successfully reset in the API server.
|
||||
"""
|
||||
logger.info("Resetting multi-modal cache...")
|
||||
@@ -57,8 +56,7 @@ async def reset_mm_cache(raw_request: Request):
|
||||
|
||||
@router.post("/reset_encoder_cache")
|
||||
async def reset_encoder_cache(raw_request: Request):
|
||||
"""
|
||||
Reset the encoder cache. Note that we currently do not check if the
|
||||
"""Reset the encoder cache. Note that we currently do not check if the
|
||||
encoder cache is successfully reset in the API server.
|
||||
"""
|
||||
logger.info("Resetting encoder cache...")
|
||||
|
||||
@@ -36,6 +36,8 @@ async def pause_generation(
|
||||
"""Pause generation requests to allow weight updates.
|
||||
|
||||
Args:
|
||||
raw_request: The incoming FastAPI request, used to reach the engine
|
||||
client on the app state.
|
||||
mode: How to handle in-flight requests:
|
||||
- ``"abort"``: Abort all in-flight requests immediately (default).
|
||||
- ``"wait"``: Wait for in-flight requests to complete.
|
||||
@@ -43,8 +45,8 @@ async def pause_generation(
|
||||
wait_for_inflight_requests: DEPRECATED. Use ``mode="wait"`` instead.
|
||||
clear_cache: DEPRECATED. Whether to clear KV/prefix caches after
|
||||
draining. Ignored when mode="keep".
|
||||
"""
|
||||
|
||||
"""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
@@ -74,7 +76,6 @@ async def pause_generation(
|
||||
@router.post("/resume")
|
||||
async def resume_generation(raw_request: Request) -> JSONResponse:
|
||||
"""Resume generation after a pause."""
|
||||
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
@@ -97,7 +98,6 @@ async def abort_requests(raw_request: Request) -> JSONResponse:
|
||||
|
||||
Empty/missing ``request_ids`` aborts all in-flight requests.
|
||||
"""
|
||||
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
@@ -139,7 +139,6 @@ async def abort_requests(raw_request: Request) -> JSONResponse:
|
||||
@router.get("/is_paused")
|
||||
async def is_paused(raw_request: Request) -> JSONResponse:
|
||||
"""Return the current pause status."""
|
||||
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
@@ -234,9 +233,12 @@ async def get_world_size(
|
||||
"""Get the world size from the parallel config.
|
||||
|
||||
Args:
|
||||
raw_request: The incoming FastAPI request, used to reach the engine
|
||||
client on the app state.
|
||||
include_dp: If True (default), returns the world size including
|
||||
data parallelism (TP * PP * DP). If False, returns the world
|
||||
size without data parallelism (TP * PP).
|
||||
|
||||
"""
|
||||
parallel_config = engine_client(raw_request).vllm_config.parallel_config
|
||||
if include_dp:
|
||||
|
||||
@@ -20,8 +20,7 @@ def set_scaling_elastic_ep(value):
|
||||
|
||||
|
||||
class ScalingMiddleware:
|
||||
"""
|
||||
Middleware that checks if the model is currently scaling and
|
||||
"""Middleware that checks if the model is currently scaling and
|
||||
returns a 503 Service Unavailable response if it is.
|
||||
|
||||
This middleware applies to all HTTP requests and prevents
|
||||
|
||||
@@ -117,7 +117,7 @@ class BaseServing:
|
||||
def _base_request_id(
|
||||
raw_request: Request | None, default: str | None = None
|
||||
) -> str | None:
|
||||
"""Pulls the request id to use from a header, if provided"""
|
||||
"""Pulls the request id to use from a header, if provided."""
|
||||
if raw_request is not None and (
|
||||
(req_id := raw_request.headers.get("X-Request-Id")) is not None
|
||||
):
|
||||
|
||||
@@ -126,7 +126,8 @@ def _format_error(err: object) -> str:
|
||||
|
||||
def _is_internal_loc_segment(segment: str) -> bool:
|
||||
"""True if `segment` is a Pydantic-internal wrapper/union-branch
|
||||
marker rather than a user-meaningful field name or list index."""
|
||||
marker rather than a user-meaningful field name or list index.
|
||||
"""
|
||||
if _BRACKETED_INTERNAL_RE.search(segment):
|
||||
return True
|
||||
return segment.lower() in _INTERNAL_LOC_MARKERS
|
||||
|
||||
@@ -28,8 +28,7 @@ async def vllm_error_handler(req: Request, exc: VLLMError):
|
||||
async def engine_error_handler(
|
||||
req: Request, exc: EngineDeadError | EngineGenerateError
|
||||
):
|
||||
"""
|
||||
VLLM V1 AsyncLLM catches exceptions and returns
|
||||
"""VLLM V1 AsyncLLM catches exceptions and returns
|
||||
only two types: EngineGenerateError and EngineDeadError.
|
||||
|
||||
EngineGenerateError is raised by the per request generate()
|
||||
@@ -51,7 +50,6 @@ async def engine_error_handler(
|
||||
will not automatically shut down. Instead, we use the watchdog
|
||||
background task for check for errored state.
|
||||
"""
|
||||
|
||||
if req.app.state.args.log_error_stack:
|
||||
logger.exception(
|
||||
"Engine Exception caught. Request id: %s",
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
vLLM Exception handlers are registered in four layers:
|
||||
"""vLLM Exception handlers are registered in four layers:
|
||||
1. framework errors raised by FastAPI/Starlette
|
||||
2. vLLM-specific errors dispatched via a single ``VLLMError`` handler
|
||||
3. fallback handlers for raw exceptions not yet migrated to ``VLLMError``
|
||||
|
||||
@@ -55,7 +55,6 @@ class PrometheusResponse(Response):
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
"""Mount prometheus metrics to a FastAPI app."""
|
||||
|
||||
registry = get_prometheus_registry()
|
||||
|
||||
# `response_class=PrometheusResponse` is needed to return an HTTP response
|
||||
|
||||
@@ -63,7 +63,8 @@ def _attach_router(app: FastAPI):
|
||||
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
|
||||
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
|
||||
"""Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
|
||||
serving engine."""
|
||||
serving engine.
|
||||
"""
|
||||
handler: OpenAIServingModels = models(raw_request)
|
||||
response = await handler.load_lora_adapter(request)
|
||||
if isinstance(response, ErrorResponse):
|
||||
@@ -85,7 +86,8 @@ def _attach_router(app: FastAPI):
|
||||
request: UnloadLoRAAdapterRequest, raw_request: Request
|
||||
):
|
||||
"""Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
|
||||
the serving engine."""
|
||||
the serving engine.
|
||||
"""
|
||||
handler: OpenAIServingModels = models(raw_request)
|
||||
response = await handler.unload_lora_adapter(request)
|
||||
if isinstance(response, ErrorResponse):
|
||||
|
||||
@@ -12,8 +12,7 @@ GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
"""
|
||||
Pure ASGI middleware that authenticates each request by checking
|
||||
"""Pure ASGI middleware that authenticates each request by checking
|
||||
if the Authorization Bearer token exists and equals anyof "{api_key}".
|
||||
|
||||
Notes
|
||||
@@ -21,6 +20,7 @@ class AuthenticationMiddleware:
|
||||
There are two cases in which authentication is skipped:
|
||||
1. The HTTP method is OPTIONS.
|
||||
2. The request path doesn't start with GUARDED_PREFIX (e.g. /health).
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp, tokens: list[str]) -> None:
|
||||
|
||||
@@ -94,7 +94,6 @@ def _extract_content_from_chunk(chunk_data: dict) -> str:
|
||||
|
||||
def _log_streaming_response(response, response_body: list) -> None:
|
||||
"""Log streaming response with robust SSE parsing."""
|
||||
|
||||
sse_decoder = SSEDecoder()
|
||||
chunk_count = 0
|
||||
|
||||
|
||||
@@ -8,8 +8,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
|
||||
class XRequestIdMiddleware:
|
||||
"""
|
||||
Middleware the set's the X-Request-Id header for each response
|
||||
"""Middleware the set's the X-Request-Id header for each response
|
||||
to a random uuid4 (hex) value if the header isn't already
|
||||
present in the request, otherwise use the provided request id.
|
||||
"""
|
||||
@@ -25,8 +24,7 @@ class XRequestIdMiddleware:
|
||||
request_headers = Headers(scope=scope)
|
||||
|
||||
async def send_with_request_id(message: Message) -> None:
|
||||
"""
|
||||
Custom send function to mutate the response headers
|
||||
"""Custom send function to mutate the response headers
|
||||
and append X-Request-Id to it.
|
||||
"""
|
||||
if message["type"] == "http.response.start":
|
||||
|
||||
@@ -30,7 +30,8 @@ EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
|
||||
"""Snapshot handler levels for loggers that may be affected by third-party
|
||||
logging configuration side effects (e.g. model_hosting_container_standards
|
||||
calling configure_root_logger() at import time)."""
|
||||
calling configure_root_logger() at import time).
|
||||
"""
|
||||
loggers = [logging.getLogger(), logging.getLogger("vllm")]
|
||||
seen: set[int] = set()
|
||||
snapshot: list[tuple[logging.Handler, int]] = []
|
||||
@@ -73,7 +74,8 @@ def _attach_router(
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Register the SageMaker hosting routes (/ping, /invocations) on the
|
||||
app."""
|
||||
app.
|
||||
"""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
router = APIRouter()
|
||||
@@ -92,7 +94,7 @@ def _attach_router(
|
||||
@router.get("/ping", response_class=Response)
|
||||
@sagemaker_standards.register_ping_handler
|
||||
async def ping(raw_request: Request) -> Response:
|
||||
"""Ping check. Endpoint required for SageMaker"""
|
||||
"""Ping check. Endpoint required for SageMaker."""
|
||||
return await health(raw_request)
|
||||
|
||||
@router.post(
|
||||
|
||||
@@ -181,8 +181,7 @@ class DetokenizeResponse(OpenAIBaseModel):
|
||||
|
||||
|
||||
class TokenizerInfoResponse(OpenAIBaseModel):
|
||||
"""
|
||||
Response containing tokenizer configuration
|
||||
"""Response containing tokenizer configuration
|
||||
equivalent to tokenizer_config.json
|
||||
"""
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ VLLM_SUBCMD_PARSER_EPILOG = (
|
||||
|
||||
|
||||
async def listen_for_disconnect(request: Request) -> None:
|
||||
"""Returns if a disconnect message is received"""
|
||||
"""Returns if a disconnect message is received."""
|
||||
while True:
|
||||
message = await request.receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
|
||||
@@ -33,7 +33,8 @@ def set_default_fingerprint_mode(
|
||||
custom_value: str | None = None,
|
||||
) -> None:
|
||||
"""Configure the fingerprint mode for subsequent ``get_system_fingerprint``
|
||||
calls. Called once at server startup."""
|
||||
calls. Called once at server startup.
|
||||
"""
|
||||
global _DEFAULT_MODE, _CUSTOM_VALUE
|
||||
_DEFAULT_MODE = mode
|
||||
_CUSTOM_VALUE = custom_value
|
||||
@@ -41,7 +42,8 @@ def set_default_fingerprint_mode(
|
||||
|
||||
def get_system_fingerprint(vllm_config: Any) -> str | None:
|
||||
"""Return the fingerprint for ``vllm_config`` using the mode configured by
|
||||
``set_default_fingerprint_mode``."""
|
||||
``set_default_fingerprint_mode``.
|
||||
"""
|
||||
return build_system_fingerprint(vllm_config, _DEFAULT_MODE, _CUSTOM_VALUE)
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utility functions that create ORCA endpoint load report response headers.
|
||||
"""
|
||||
"""Utility functions that create ORCA endpoint load report response headers."""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
@@ -16,23 +14,22 @@ logger = init_logger(__name__)
|
||||
def create_orca_header(
|
||||
metrics_format: str, named_metrics: list[tuple[str, float]]
|
||||
) -> Mapping[str, str] | None:
|
||||
"""
|
||||
Creates ORCA headers named 'endpoint-load-metrics' in the specified format
|
||||
"""Creates ORCA headers named 'endpoint-load-metrics' in the specified format
|
||||
and adds custom metrics to named_metrics.
|
||||
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
|
||||
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
|
||||
|
||||
Parameters:
|
||||
- metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
- named_metrics (List[Tuple[str, float]]): List of tuples with metric names
|
||||
and their corresponding double values.
|
||||
Args:
|
||||
metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
named_metrics (List[Tuple[str, float]]): List of tuples with metric names
|
||||
and their corresponding double values.
|
||||
|
||||
Returns:
|
||||
- Optional[Mapping[str,str]]: A dictionary with header key as
|
||||
'endpoint-load-metrics' and values as the ORCA header strings with
|
||||
format prefix and data in with named_metrics in.
|
||||
"""
|
||||
|
||||
"""
|
||||
if metrics_format.lower() not in ["text", "json"]:
|
||||
logger.warning(
|
||||
"Warning: `%s` format is not supported in the ORCA response header",
|
||||
@@ -71,15 +68,17 @@ def create_orca_header(
|
||||
|
||||
|
||||
def get_named_metrics_from_prometheus() -> list[tuple[str, float]]:
|
||||
"""
|
||||
Collects current metrics from Prometheus and returns some of them
|
||||
"""Collects current metrics from Prometheus and returns some of them
|
||||
in the form of the `named_metrics` list for `create_orca_header()`.
|
||||
|
||||
Parameters:
|
||||
Parameters
|
||||
----------
|
||||
- None
|
||||
|
||||
Returns:
|
||||
Returns
|
||||
-------
|
||||
- list[tuple[str, float]]: List of tuples of metric names and their values.
|
||||
|
||||
"""
|
||||
named_metrics: list[tuple[str, float]] = []
|
||||
# Map from prometheus metric names to ORCA named metrics.
|
||||
@@ -98,20 +97,20 @@ def get_named_metrics_from_prometheus() -> list[tuple[str, float]]:
|
||||
|
||||
|
||||
def metrics_header(metrics_format: str) -> Mapping[str, str] | None:
|
||||
"""
|
||||
Creates ORCA headers named 'endpoint-load-metrics' in the specified format.
|
||||
"""Creates ORCA headers named 'endpoint-load-metrics' in the specified format.
|
||||
Metrics are collected from Prometheus using `get_named_metrics_from_prometheus()`.
|
||||
|
||||
ORCA headers format description: https://docs.google.com/document/d/1C1ybMmDKJIVlrbOLbywhu9iRYo4rilR-cT50OTtOFTs/edit?tab=t.0
|
||||
ORCA proto https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto
|
||||
|
||||
Parameters:
|
||||
- metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
Args:
|
||||
metrics_format (str): The format of the header ('TEXT', 'JSON').
|
||||
|
||||
Returns:
|
||||
- Optional[Mapping[str,str]]: A dictionary with header key as
|
||||
'endpoint-load-metrics' and values as the ORCA header strings with
|
||||
format prefix and data in with named_metrics in.
|
||||
|
||||
"""
|
||||
if not metrics_format:
|
||||
return None
|
||||
|
||||
@@ -20,7 +20,6 @@ def maybe_filter_parallel_tool_calls(
|
||||
choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
|
||||
) -> _ChatCompletionResponseChoiceT:
|
||||
"""Filter to first tool call only when parallel_tool_calls is explicitly False."""
|
||||
|
||||
if request.parallel_tool_calls is not False:
|
||||
return choice
|
||||
|
||||
|
||||
@@ -92,7 +92,8 @@ def asr_inter_chunk_separator(
|
||||
|
||||
class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
"""Base class for speech-to-text operations like transcription and
|
||||
translation."""
|
||||
translation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -358,8 +359,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
segment_class: type[SpeechToTextSegment],
|
||||
start_time: float = 0,
|
||||
) -> list[SpeechToTextSegment]:
|
||||
"""
|
||||
Convert tokens to verbose segments.
|
||||
"""Convert tokens to verbose segments.
|
||||
|
||||
This method expects the model to produce
|
||||
timestamps as tokens (similar to Whisper).
|
||||
@@ -427,7 +427,8 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
|
||||
) -> T | V | AsyncGenerator[str, None] | ErrorResponse:
|
||||
"""Base method for speech-to-text operations like transcription and
|
||||
translation."""
|
||||
translation.
|
||||
"""
|
||||
if request.stream and request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Streaming is not currently supported with beam search"
|
||||
|
||||
@@ -33,6 +33,7 @@ async def read_upload_with_limit(
|
||||
|
||||
Raises:
|
||||
VLLMValidationError: If the file exceeds the configured size limit.
|
||||
|
||||
"""
|
||||
if max_size_mb is None:
|
||||
max_size_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
|
||||
|
||||
@@ -13,14 +13,14 @@ from vllm.utils import random_uuid
|
||||
|
||||
|
||||
class InputAudioBufferAppend(OpenAIBaseModel):
|
||||
"""Append audio chunk to buffer"""
|
||||
"""Append audio chunk to buffer."""
|
||||
|
||||
type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append"
|
||||
audio: str # base64-encoded PCM16 @ 16kHz
|
||||
|
||||
|
||||
class InputAudioBufferCommit(OpenAIBaseModel):
|
||||
"""Process accumulated audio buffer"""
|
||||
"""Process accumulated audio buffer."""
|
||||
|
||||
type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit"
|
||||
final: bool = False
|
||||
@@ -28,14 +28,14 @@ class InputAudioBufferCommit(OpenAIBaseModel):
|
||||
|
||||
# Server -> Client Events
|
||||
class SessionUpdate(OpenAIBaseModel):
|
||||
"""Configure session parameters"""
|
||||
"""Configure session parameters."""
|
||||
|
||||
type: Literal["session.update"] = "session.update"
|
||||
model: str | None = None
|
||||
|
||||
|
||||
class SessionCreated(OpenAIBaseModel):
|
||||
"""Connection established notification"""
|
||||
"""Connection established notification."""
|
||||
|
||||
type: Literal["session.created"] = "session.created"
|
||||
id: str = Field(default_factory=lambda: f"sess-{random_uuid()}")
|
||||
@@ -43,14 +43,14 @@ class SessionCreated(OpenAIBaseModel):
|
||||
|
||||
|
||||
class TranscriptionDelta(OpenAIBaseModel):
|
||||
"""Incremental transcription text"""
|
||||
"""Incremental transcription text."""
|
||||
|
||||
type: Literal["transcription.delta"] = "transcription.delta"
|
||||
delta: str # Incremental text
|
||||
|
||||
|
||||
class TranscriptionDone(OpenAIBaseModel):
|
||||
"""Final transcription with usage stats"""
|
||||
"""Final transcription with usage stats."""
|
||||
|
||||
type: Literal["transcription.done"] = "transcription.done"
|
||||
text: str # Complete transcription
|
||||
@@ -58,7 +58,7 @@ class TranscriptionDone(OpenAIBaseModel):
|
||||
|
||||
|
||||
class ErrorEvent(OpenAIBaseModel):
|
||||
"""Error notification"""
|
||||
"""Error notification."""
|
||||
|
||||
type: Literal["error"] = "error"
|
||||
error: str
|
||||
|
||||
@@ -68,6 +68,7 @@ class OpenAIServingRealtime(GenerateBaseServing):
|
||||
|
||||
Yields:
|
||||
StreamingInput objects containing audio prompts for the engine
|
||||
|
||||
"""
|
||||
model_config = self.model_config
|
||||
renderer = self.renderer
|
||||
|
||||
Reference in New Issue
Block a user