[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicCountTokensResponse,
|
||||
AnthropicError,
|
||||
AnthropicErrorResponse,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def messages(request: Request) -> AnthropicServingMessages:
|
||||
return request.app.state.anthropic_serving_messages
|
||||
|
||||
|
||||
def translate_error_response(response: ErrorResponse) -> JSONResponse:
|
||||
anthropic_error = AnthropicErrorResponse(
|
||||
error=AnthropicError(
|
||||
type=response.error.type,
|
||||
message=response.error.message,
|
||||
)
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=response.error.code, content=anthropic_error.model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
|
||||
handler = messages(raw_request)
|
||||
if handler is None:
|
||||
base_server = raw_request.app.state.serving_tokenization
|
||||
error = base_server.create_error_response(
|
||||
NotImplementedError("The model does not support Messages API")
|
||||
)
|
||||
return translate_error_response(error)
|
||||
|
||||
try:
|
||||
generator = await handler.create_messages(request, raw_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in create_messages: %s", e)
|
||||
return translate_error_response(create_error_response(e))
|
||||
|
||||
if isinstance(generator, ErrorResponse):
|
||||
return translate_error_response(generator)
|
||||
|
||||
elif isinstance(generator, AnthropicMessagesResponse):
|
||||
resp = generator.model_dump(exclude_none=True)
|
||||
logger.debug("Anthropic Messages Response: %s", resp)
|
||||
return JSONResponse(content=resp)
|
||||
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages/count_tokens",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"model": AnthropicCountTokensResponse},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request):
|
||||
handler = messages(raw_request)
|
||||
if handler is None:
|
||||
base_server = raw_request.app.state.serving_tokenization
|
||||
error = base_server.create_error_response(
|
||||
NotImplementedError("The model does not support Messages API")
|
||||
)
|
||||
return translate_error_response(error)
|
||||
|
||||
try:
|
||||
response = await handler.count_tokens(request, raw_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in count_tokens: %s", e)
|
||||
return translate_error_response(create_error_response(e))
|
||||
|
||||
if isinstance(response, ErrorResponse):
|
||||
return translate_error_response(response)
|
||||
|
||||
return JSONResponse(content=response.model_dump(exclude_none=True))
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,451 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pydantic models for Anthropic API protocol."""
|
||||
|
||||
import time
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
MMPayload,
|
||||
resolve_return_mm_kwargs_alias,
|
||||
)
|
||||
|
||||
|
||||
class AnthropicError(BaseModel):
|
||||
"""Error structure for Anthropic API."""
|
||||
|
||||
type: str
|
||||
message: str
|
||||
|
||||
|
||||
class AnthropicErrorResponse(BaseModel):
|
||||
"""Error response structure for Anthropic API."""
|
||||
|
||||
type: Literal["error"] = "error"
|
||||
error: AnthropicError
|
||||
|
||||
|
||||
class AnthropicUsage(BaseModel):
|
||||
"""Token usage information."""
|
||||
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int | None = None
|
||||
cache_read_input_tokens: int | None = None
|
||||
|
||||
|
||||
class AnthropicContentBlock(BaseModel):
|
||||
"""Content block in message."""
|
||||
|
||||
type: Literal[
|
||||
"text",
|
||||
"image",
|
||||
"tool_use",
|
||||
"tool_result",
|
||||
"tool_reference",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
"tool_addition",
|
||||
"tool_removal",
|
||||
]
|
||||
text: str | None = None
|
||||
# For image content
|
||||
source: dict[str, Any] | None = None
|
||||
# For tool use/result
|
||||
id: str | None = None
|
||||
tool_use_id: str | None = None
|
||||
name: str | None = None
|
||||
input: dict[str, Any] | None = None
|
||||
content: str | list[dict[str, Any]] | None = None
|
||||
is_error: bool | None = None
|
||||
# For tool_reference content
|
||||
tool_name: str | None = None
|
||||
# For thinking content
|
||||
thinking: str | None = None
|
||||
signature: str | None = None
|
||||
# For redacted thinking content (safety-filtered by the API)
|
||||
data: str | None = None
|
||||
# For tool_addition/tool_removal content
|
||||
tool: "AnthropicToolChange | None" = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_tool_change(self) -> "AnthropicContentBlock":
|
||||
if self.type not in ("tool_addition", "tool_removal"):
|
||||
return self
|
||||
if self.tool is None:
|
||||
raise ValueError(f"tool is required for {self.type} blocks")
|
||||
if self.type == "tool_removal" and self.tool.type != "tool_reference":
|
||||
raise ValueError("tool_removal only accepts a tool_reference")
|
||||
return self
|
||||
|
||||
|
||||
def _reject_tool_changes(content: str | list[AnthropicContentBlock] | None) -> None:
|
||||
if content is None or isinstance(content, str):
|
||||
return
|
||||
for block in content:
|
||||
if block.type in ("tool_addition", "tool_removal"):
|
||||
raise ValueError(
|
||||
f'{block.type} blocks are only allowed in messages with role "system"'
|
||||
)
|
||||
|
||||
|
||||
class AnthropicMessage(BaseModel):
|
||||
"""Message structure."""
|
||||
|
||||
role: Literal["user", "assistant", "system"]
|
||||
content: str | list[AnthropicContentBlock]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_tool_change_role(self) -> "AnthropicMessage":
|
||||
if self.role != "system":
|
||||
_reject_tool_changes(self.content)
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicTool(BaseModel):
|
||||
"""Tool definition."""
|
||||
|
||||
name: str
|
||||
description: str | None = None
|
||||
input_schema: dict[str, Any]
|
||||
strict: bool | None = None
|
||||
defer_loading: bool | None = None
|
||||
|
||||
@field_validator("input_schema")
|
||||
@classmethod
|
||||
def validate_input_schema(cls, v):
|
||||
if not isinstance(v, dict):
|
||||
raise ValueError("input_schema must be a dictionary")
|
||||
if "type" not in v:
|
||||
v["type"] = "object" # Default to object type
|
||||
return v
|
||||
|
||||
|
||||
class AnthropicToolChangeReference(BaseModel):
|
||||
"""A tool named by a tool_addition or tool_removal block."""
|
||||
|
||||
type: Literal["tool_reference"]
|
||||
name: str
|
||||
|
||||
|
||||
class AnthropicToolChangeDefinition(BaseModel):
|
||||
"""A tool defined by value in a tool_addition block."""
|
||||
|
||||
type: Literal["tool_definition"]
|
||||
definition: AnthropicTool
|
||||
|
||||
|
||||
AnthropicToolChange = Annotated[
|
||||
AnthropicToolChangeReference | AnthropicToolChangeDefinition,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class AnthropicToolChoice(BaseModel):
|
||||
"""Tool Choice definition."""
|
||||
|
||||
type: Literal["auto", "any", "tool", "none"]
|
||||
name: str | None = None
|
||||
disable_parallel_tool_use: bool | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_name_required_for_tool(self) -> "AnthropicToolChoice":
|
||||
if self.type == "tool" and not self.name:
|
||||
raise ValueError("tool_choice.name is required when type is 'tool'")
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicJsonOutputFormat(BaseModel):
|
||||
"""JSON output format configuration."""
|
||||
|
||||
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
|
||||
type: Literal["json_schema"] = "json_schema"
|
||||
|
||||
|
||||
AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"]
|
||||
# reasoning_effort sent to the chat template for thinking.type=disabled.
|
||||
AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort]
|
||||
AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort]
|
||||
AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"]
|
||||
|
||||
|
||||
class AnthropicOutputConfig(BaseModel):
|
||||
"""Configuration options for the model's output, such as the output format."""
|
||||
|
||||
effort: AnthropicEffort | None = None
|
||||
format: AnthropicJsonOutputFormat | None = None
|
||||
|
||||
|
||||
class AnthropicThinkingConfigEnabled(BaseModel):
|
||||
"""Extended thinking with a fixed token budget.
|
||||
|
||||
``display`` is accepted but ignored: reasoning is always returned.
|
||||
"""
|
||||
|
||||
type: Literal["enabled"]
|
||||
budget_tokens: int = Field(ge=1024)
|
||||
display: AnthropicThinkingDisplay | None = None
|
||||
|
||||
|
||||
class AnthropicThinkingConfigDisabled(BaseModel):
|
||||
type: Literal["disabled"]
|
||||
|
||||
|
||||
class AnthropicThinkingConfigAdaptive(BaseModel):
|
||||
"""Extended thinking whose depth the model chooses.
|
||||
|
||||
``display`` is accepted but ignored: reasoning is always returned.
|
||||
"""
|
||||
|
||||
type: Literal["adaptive"]
|
||||
display: AnthropicThinkingDisplay | None = None
|
||||
|
||||
|
||||
AnthropicThinkingConfig = Annotated[
|
||||
AnthropicThinkingConfigEnabled
|
||||
| AnthropicThinkingConfigDisabled
|
||||
| AnthropicThinkingConfigAdaptive,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(BaseModel):
|
||||
"""Anthropic Messages API request."""
|
||||
|
||||
model: str
|
||||
messages: list[AnthropicMessage]
|
||||
max_tokens: int
|
||||
metadata: dict[str, Any] | None = None
|
||||
output_config: AnthropicOutputConfig | None = None
|
||||
thinking: AnthropicThinkingConfig | None = None
|
||||
stop_sequences: (
|
||||
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
) = None
|
||||
stream: bool | None = False
|
||||
system: str | list[AnthropicContentBlock] | None = None
|
||||
temperature: float | None = None
|
||||
tool_choice: AnthropicToolChoice | None = None
|
||||
tools: list[AnthropicTool] | None = None
|
||||
top_k: int | None = None
|
||||
top_p: float | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
mm_payload: MMPayload = Field(
|
||||
default="tensors",
|
||||
description=(
|
||||
"What the render response's `features` carry for multimodal "
|
||||
"items. 'tensors' serializes the processed tensors into "
|
||||
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
|
||||
"callers that need only the token layout and item hashes, such as "
|
||||
"cache-aware routers. Do not send a 'none' response to "
|
||||
"`/inference/v1/generate` which reads a null `kwargs_data` as "
|
||||
"every item being cached. Only supported on the render endpoints. "
|
||||
"Ignored on regular generation endpoints."
|
||||
),
|
||||
)
|
||||
return_mm_kwargs: bool | None = Field(
|
||||
default=None,
|
||||
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
|
||||
description=(
|
||||
"Deprecated alias: false is the same as `mm_payload='none'`. "
|
||||
"Ignored when `mm_payload` is set."
|
||||
),
|
||||
)
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
watermarking: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Whether to apply the engine's configured watermark to this request."
|
||||
),
|
||||
)
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"ECTransfer parameters used for encoder-cache disaggregated serving."
|
||||
),
|
||||
)
|
||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional request parameters with (list of) string or "
|
||||
"numeric values, used by custom extensions."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def validate_model(cls, v):
|
||||
if not v:
|
||||
raise ValueError("Model is required")
|
||||
return v
|
||||
|
||||
@field_validator("max_tokens")
|
||||
@classmethod
|
||||
def validate_max_tokens(cls, v):
|
||||
if v <= 0:
|
||||
raise ValueError("max_tokens must be positive")
|
||||
return v
|
||||
|
||||
@field_validator("system")
|
||||
@classmethod
|
||||
def validate_system(cls, v):
|
||||
_reject_tool_changes(v)
|
||||
return v
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
|
||||
return resolve_return_mm_kwargs_alias(data)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
|
||||
# P/D prefill legs are sent with max_tokens=1 and never decode; the
|
||||
# decode leg carries the client's max_tokens and is still checked.
|
||||
if self.kv_transfer_params and self.kv_transfer_params.get("do_remote_decode"):
|
||||
return self
|
||||
if (
|
||||
isinstance(self.thinking, AnthropicThinkingConfigEnabled)
|
||||
and self.thinking.budget_tokens >= self.max_tokens
|
||||
):
|
||||
raise ValueError("thinking.budget_tokens must be less than max_tokens")
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicDelta(BaseModel):
|
||||
"""Delta for streaming responses."""
|
||||
|
||||
type: (
|
||||
Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
|
||||
| None
|
||||
) = None
|
||||
text: str | None = None
|
||||
thinking: str | None = None
|
||||
partial_json: str | None = None
|
||||
signature: str | None = None
|
||||
|
||||
# Message delta
|
||||
stop_reason: (
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
|
||||
) = None
|
||||
stop_sequence: str | None = None
|
||||
|
||||
|
||||
class AnthropicStreamEvent(BaseModel):
|
||||
"""Streaming event."""
|
||||
|
||||
type: Literal[
|
||||
"message_start",
|
||||
"message_delta",
|
||||
"message_stop",
|
||||
"content_block_start",
|
||||
"content_block_delta",
|
||||
"content_block_stop",
|
||||
"ping",
|
||||
"error",
|
||||
]
|
||||
message: "AnthropicMessagesResponse | None" = None
|
||||
delta: AnthropicDelta | None = None
|
||||
content_block: AnthropicContentBlock | None = None
|
||||
index: int | None = None
|
||||
error: AnthropicError | None = None
|
||||
usage: AnthropicUsage | None = None
|
||||
|
||||
|
||||
class AnthropicMessagesResponse(BaseModel):
|
||||
"""Anthropic Messages API response."""
|
||||
|
||||
id: str
|
||||
type: Literal["message"] = "message"
|
||||
role: Literal["assistant"] = "assistant"
|
||||
content: list[AnthropicContentBlock]
|
||||
model: str
|
||||
stop_reason: (
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
|
||||
) = None
|
||||
stop_sequence: str | None = None
|
||||
usage: AnthropicUsage | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
ec_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="ECTransfer parameters."
|
||||
)
|
||||
|
||||
def model_post_init(self, __context):
|
||||
if not self.id:
|
||||
self.id = f"msg_{int(time.time() * 1000)}"
|
||||
|
||||
|
||||
class AnthropicContextManagement(BaseModel):
|
||||
"""Context management information for token counting."""
|
||||
|
||||
original_input_tokens: int
|
||||
|
||||
|
||||
class AnthropicCountTokensRequest(BaseModel):
|
||||
"""Anthropic messages.count_tokens request."""
|
||||
|
||||
model: str
|
||||
messages: list[AnthropicMessage]
|
||||
output_config: AnthropicOutputConfig | None = None
|
||||
system: str | list[AnthropicContentBlock] | None = None
|
||||
thinking: AnthropicThinkingConfig | None = None
|
||||
tool_choice: AnthropicToolChoice | None = None
|
||||
tools: list[AnthropicTool] | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def validate_model(cls, v):
|
||||
if not v:
|
||||
raise ValueError("Model is required")
|
||||
return v
|
||||
|
||||
@field_validator("system")
|
||||
@classmethod
|
||||
def validate_system(cls, v):
|
||||
_reject_tool_changes(v)
|
||||
return v
|
||||
|
||||
|
||||
class AnthropicCountTokensResponse(BaseModel):
|
||||
"""Anthropic messages.count_tokens response."""
|
||||
|
||||
input_tokens: int
|
||||
context_management: AnthropicContextManagement | None = None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
|
||||
import uvloop
|
||||
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.render.entry import run_launch_fastapi
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DESCRIPTION = "Launch individual vLLM components."
|
||||
|
||||
|
||||
class LaunchSubcommandBase(CLISubcommand):
|
||||
"""The base class of subcommands for `vllm launch`."""
|
||||
|
||||
help: str
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
"""Add the CLI arguments to the parser.
|
||||
|
||||
By default, uses the subcommand's docstring as the description and adds
|
||||
the standard vLLM serving arguments.
|
||||
Subclasses can override to add component-specific arguments.
|
||||
"""
|
||||
if cls.__doc__:
|
||||
parser.description = inspect.cleandoc(cls.__doc__)
|
||||
make_arg_parser(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RenderSubcommand(LaunchSubcommandBase):
|
||||
"""`vllm launch render` starts a GPU-less rendering server for preprocessing
|
||||
and postprocessing only.
|
||||
|
||||
```bash
|
||||
vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100
|
||||
```
|
||||
|
||||
This command reuses the standard serving parser, so model, frontend,
|
||||
networking, and related CLI options follow the same conventions as
|
||||
[`vllm serve`](https://docs.vllm.ai/en/latest/cli/serve/).
|
||||
"""
|
||||
|
||||
name = "render"
|
||||
help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
uvloop.run(run_launch_fastapi(args))
|
||||
|
||||
|
||||
class LaunchSubcommand(CLISubcommand):
|
||||
"""The `launch` subcommand for the vLLM CLI.
|
||||
|
||||
Uses nested sub-subcommands so each component can define its own
|
||||
arguments independently (e.g. ``vllm launch render``).
|
||||
"""
|
||||
|
||||
name = "launch"
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
if hasattr(args, "model_tag") and args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
|
||||
args.launch_command(args)
|
||||
|
||||
def validate(self, args: argparse.Namespace) -> None:
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
launch_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help=DESCRIPTION,
|
||||
description=DESCRIPTION,
|
||||
usage=f"vllm {self.name} <component> [options]",
|
||||
)
|
||||
launch_subparsers = launch_parser.add_subparsers(
|
||||
required=True, dest="launch_component"
|
||||
)
|
||||
|
||||
for cmd_cls in LaunchSubcommandBase.__subclasses__():
|
||||
cmd_subparser = launch_subparsers.add_parser(
|
||||
cmd_cls.name,
|
||||
help=cmd_cls.help,
|
||||
usage=f"vllm {self.name} {cmd_cls.name} [options]",
|
||||
)
|
||||
cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
|
||||
cmd_cls.add_cli_args(cmd_subparser)
|
||||
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
|
||||
subcmd=f"{self.name} {cmd_cls.name}"
|
||||
)
|
||||
|
||||
return launch_parser
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [LaunchSubcommand()]
|
||||
@@ -42,10 +42,8 @@ def main():
|
||||
|
||||
import vllm.entrypoints.cli.collect_env
|
||||
import vllm.entrypoints.cli.download_kernels
|
||||
import vllm.entrypoints.cli.launch
|
||||
import vllm.entrypoints.cli.openai
|
||||
import vllm.entrypoints.cli.preload
|
||||
import vllm.entrypoints.cli.run_batch
|
||||
import vllm.entrypoints.cli.serve
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
VLLM_SUBCMD_PARSER_EPILOG,
|
||||
@@ -56,11 +54,9 @@ def main():
|
||||
CMD_MODULES = [
|
||||
vllm.entrypoints.cli.openai,
|
||||
vllm.entrypoints.cli.serve,
|
||||
vllm.entrypoints.cli.launch,
|
||||
vllm.entrypoints.cli.collect_env,
|
||||
vllm.entrypoints.cli.download_kernels,
|
||||
vllm.entrypoints.cli.preload,
|
||||
vllm.entrypoints.cli.run_batch,
|
||||
]
|
||||
|
||||
cli_env_setup()
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import importlib.metadata
|
||||
import typing
|
||||
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RunBatchSubcommand(CLISubcommand):
|
||||
"""The `run-batch` subcommand for vLLM CLI."""
|
||||
|
||||
name = "run-batch"
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
from vllm.entrypoints.launchers.run_batch import main as run_batch_main
|
||||
|
||||
logger.info(
|
||||
"vLLM batch processing API version %s", importlib.metadata.version("vllm")
|
||||
)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
# Start the Prometheus metrics server.
|
||||
# LLMEngine uses the Prometheus client
|
||||
# to publish metrics at the /metrics endpoint.
|
||||
if args.enable_metrics:
|
||||
from prometheus_client import start_http_server
|
||||
|
||||
logger.info("Prometheus metrics enabled")
|
||||
start_http_server(port=args.port, addr=args.url)
|
||||
else:
|
||||
logger.info("Prometheus metrics disabled")
|
||||
|
||||
asyncio.run(run_batch_main(args))
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
from vllm.entrypoints.launchers.run_batch import make_arg_parser
|
||||
|
||||
run_batch_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help="Run batch prompts and write results to file.",
|
||||
description=(
|
||||
"Run batch prompts using vLLM's OpenAI-compatible API.\n"
|
||||
"Supports local or HTTP input/output files."
|
||||
),
|
||||
usage="vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
|
||||
)
|
||||
run_batch_parser = make_arg_parser(run_batch_parser)
|
||||
run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
|
||||
return run_batch_parser
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [RunBatchSubcommand()]
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,309 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FastAPI router for the Cohere Chat v2 API.
|
||||
|
||||
Exposes two routes:
|
||||
|
||||
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
|
||||
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
|
||||
generation, mirroring ``POST /v1/chat/completions/render``.
|
||||
|
||||
The Cohere v2 protocol models are sourced from the official ``cohere``
|
||||
Python SDK (``pip install cohere``). To keep that an *optional*
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handlers
|
||||
themselves - are gated on a one-shot probe at module load. If the SDK
|
||||
isn't installed, :func:`attach_router` becomes a no-op (with an info
|
||||
log) and vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the chat route. The render route additionally requires scale-out
|
||||
endpoints to be enabled, i.e. ``--enable-scale-out`` or
|
||||
``--tokens-only``. These gates keep unrelated deployments from
|
||||
accidentally exposing the APIs.
|
||||
|
||||
Note: the handlers must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
the ``CohereChatV2Request`` body annotation against the module's
|
||||
globals. Defining them locally inside ``attach_router`` would hide the
|
||||
type from ``get_type_hints``, causing FastAPI to silently degrade the
|
||||
body parameter into a query parameter and reject every request with
|
||||
422.
|
||||
"""
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
_COHERE_PATH_PREFIX = "/cohere/"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
import cohere # noqa: F401 -- dependency probe
|
||||
except ImportError:
|
||||
_SDK_AVAILABLE = False
|
||||
else:
|
||||
_SDK_AVAILABLE = True
|
||||
|
||||
|
||||
if _SDK_AVAILABLE:
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereError,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
|
||||
router = APIRouter()
|
||||
render_router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
|
||||
def _serving_render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
def _request_id(raw_request: Request | None) -> str | None:
|
||||
"""Best-effort lookup of the active request id.
|
||||
|
||||
Prefers the id the underlying chat handler stamped onto
|
||||
``raw_request.state.request_metadata`` (if it got that far before
|
||||
failing), falling back to the ``X-Request-Id`` HTTP header. May
|
||||
return ``None`` if neither is available, in which case the field
|
||||
is omitted from the response.
|
||||
"""
|
||||
if raw_request is None:
|
||||
return None
|
||||
meta = getattr(raw_request.state, "request_metadata", None)
|
||||
if meta is not None and getattr(meta, "request_id", None):
|
||||
return meta.request_id
|
||||
return raw_request.headers.get("X-Request-Id")
|
||||
|
||||
def _error_response(
|
||||
error: ErrorResponse,
|
||||
raw_request: Request | None,
|
||||
*,
|
||||
fallback_status: int = HTTPStatus.BAD_REQUEST,
|
||||
) -> JSONResponse:
|
||||
"""Translate vLLM's internal error envelope into Cohere's shape."""
|
||||
info = error.error
|
||||
status = info.code or fallback_status
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content=CohereError(
|
||||
message=sanitize_message(info.message),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/cohere/v2/chat",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
|
||||
handler = _serving(raw_request)
|
||||
if handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message="The model does not support the Cohere v2 chat API.",
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
result = await handler.create_chat_v2(request, raw_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in /cohere/v2/chat: %s", e)
|
||||
return _error_response(create_error_response(e), raw_request)
|
||||
|
||||
match result:
|
||||
case ErrorResponse():
|
||||
return _error_response(result, raw_request)
|
||||
case CohereChatV2Response():
|
||||
return JSONResponse(content=result.model_dump(exclude_none=True))
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
@render_router.post(
|
||||
"/cohere/v2/chat/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
|
||||
"""Tokenize a Cohere v2 chat request without running generation.
|
||||
|
||||
The Cohere counterpart to ``POST /v1/chat/completions/render``.
|
||||
The v2 body goes through the same conversion
|
||||
``/cohere/v2/chat`` uses and is then handed to the shared
|
||||
:class:`ServingRender`, so the returned ``GenerateRequest``
|
||||
carries the prompt tokens and sampling params the chat endpoint
|
||||
would have sent to the engine.
|
||||
"""
|
||||
handler = _serving(raw_request)
|
||||
render_handler = _serving_render(raw_request)
|
||||
if handler is None or render_handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message=(
|
||||
"The model does not support the Cohere v2 chat render API."
|
||||
),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
chat_request = handler.to_chat_completion_request(request)
|
||||
result = await render_handler.render_chat_request(chat_request)
|
||||
except Exception as e:
|
||||
logger.exception("Error in /cohere/v2/chat/render: %s", e)
|
||||
return _error_response(create_error_response(e), raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return _error_response(result, raw_request)
|
||||
|
||||
if (kwargs := result.reasoning_parser_kwargs) is not None:
|
||||
kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs(
|
||||
kwargs.chat_template_kwargs
|
||||
)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
|
||||
|
||||
The endpoint handler above already returns :class:`CohereError` for
|
||||
errors it owns, but globally-registered exception handlers (e.g.
|
||||
:func:`validation_exception_handler` for pydantic body errors,
|
||||
:func:`http_exception_handler`, engine error handlers) fire *before*
|
||||
the handler runs and produce vLLM's internal
|
||||
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
|
||||
shape doesn't match the ``CohereError`` schema advertised on the
|
||||
route's OpenAPI ``responses``, so clients (and schema-conformance
|
||||
tests like ``test_openai_schema.py``) would see a mismatch on
|
||||
those paths. This middleware normalises any error body on
|
||||
``/cohere/*`` responses to :class:`CohereError`.
|
||||
"""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
|
||||
return response
|
||||
if response.status_code < 400:
|
||||
return response
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if not content_type.startswith("application/json"):
|
||||
return response
|
||||
|
||||
body = b"".join([chunk async for chunk in response.body_iterator])
|
||||
translated = _translate_vllm_error_body(body, request)
|
||||
if translated is not None:
|
||||
return translated
|
||||
passthrough_headers = {
|
||||
k: v
|
||||
for k, v in response.headers.items()
|
||||
if k.lower() != "content-length"
|
||||
}
|
||||
return Response(
|
||||
content=body,
|
||||
status_code=response.status_code,
|
||||
headers=passthrough_headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
|
||||
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
|
||||
|
||||
Returns ``None`` if ``raw`` does not match the vLLM error envelope
|
||||
(which signals the middleware to pass the body through unchanged).
|
||||
"""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
if not (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("error"), dict)
|
||||
and "message" in data["error"]
|
||||
):
|
||||
return None
|
||||
try:
|
||||
err = ErrorResponse.model_validate(data)
|
||||
except Exception: # noqa: BLE001 - malformed envelope; pass through
|
||||
return None
|
||||
return _error_response(err, request)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register the enabled ``/cohere/v2/chat`` routes on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
|
||||
Cohere v2 endpoints are opt-in because they carry Cohere-specific
|
||||
request/response semantics (grounding citations, tool_plan,
|
||||
PLAN/THINKING_CONTENT blocks) that are only meaningful when
|
||||
serving a Cohere Command-family model.
|
||||
* the optional ``cohere`` SDK isn't installed (the v2 protocol
|
||||
models live there)
|
||||
|
||||
The two skip paths log at different levels: an operator who set
|
||||
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
|
||||
a WARNING (they explicitly asked for the endpoints and they're
|
||||
silently absent), whereas the default-off skip logs at debug.
|
||||
"""
|
||||
enabled = envs.VLLM_ENABLE_COHERE_API
|
||||
if not enabled:
|
||||
logger.debug(
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
|
||||
)
|
||||
return
|
||||
if not _SDK_AVAILABLE:
|
||||
logger.warning(
|
||||
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
|
||||
"installed; /cohere/v2/chat will not be exposed. Install "
|
||||
"with `pip install cohere` to enable the endpoints."
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
args = getattr(app.state, "args", None)
|
||||
if getattr(args, "enable_scale_out", False) or getattr(args, "tokens_only", False):
|
||||
app.include_router(render_router)
|
||||
app.add_middleware(CohereErrorEnvelopeMiddleware)
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
|
||||
|
||||
This module keeps the OpenAI chat completion protocol classes
|
||||
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
|
||||
It provides two things:
|
||||
|
||||
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
|
||||
representation produced by the Cohere parser
|
||||
(:mod:`vllm.parser.cohere_command`) and consumed by
|
||||
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
|
||||
SDK shape -- that conversion happens in the serving layer.
|
||||
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
|
||||
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
|
||||
are only instantiated by the citation-aware serving handler
|
||||
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
|
||||
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
|
||||
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
|
||||
|
||||
The module intentionally does *not* import the ``cohere`` Python SDK so the
|
||||
reasoning parser (which is Cohere-model-specific but ships without the SDK
|
||||
dependency) can import from here freely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_serializer
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
|
||||
class CitationSource(OpenAIBaseModel):
|
||||
"""Source attribution for a :class:`Citation`.
|
||||
|
||||
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
|
||||
discriminator (``document`` or ``tool``); ``id`` is the citing
|
||||
document/tool-output identifier; ``document`` and ``tool_output`` carry
|
||||
the original payload that produced the citation.
|
||||
|
||||
Sources are fully resolved by the parser at emit time
|
||||
(see :func:`_melody_sources_to_vllm` in
|
||||
:mod:`vllm.parser.cohere_command`) using the
|
||||
``POSITION_TO_SOURCE_KEY`` map forwarded through
|
||||
``chat_template_kwargs`` by
|
||||
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
|
||||
That means every instance that reaches the serving / wire layer
|
||||
already has ``type`` / ``id`` populated; the serving layer only has
|
||||
to coerce to :class:`cohere.types.Citation` and rewrite
|
||||
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
|
||||
"""
|
||||
|
||||
type: Literal["document", "tool"] | None = None
|
||||
id: str | None = None
|
||||
document: dict[str, Any] | None = None
|
||||
tool_output: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class Citation(OpenAIBaseModel):
|
||||
"""A citation grounding a span of generated text in source material.
|
||||
|
||||
vLLM-internal representation used by the Cohere reasoning parser and
|
||||
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
|
||||
conversion to the SDK's ``cohere.types.Citation`` happens in
|
||||
:mod:`vllm.entrypoints.cohere.serving`.
|
||||
"""
|
||||
|
||||
start: int | None = None
|
||||
"""Start character offset in the surrounding text content."""
|
||||
end: int | None = None
|
||||
"""End character offset (exclusive) in the surrounding text content."""
|
||||
text: str | None = None
|
||||
"""The cited text snippet."""
|
||||
sources: list[CitationSource] = Field(default_factory=list)
|
||||
"""Source documents / tool outputs that ground this citation."""
|
||||
content_index: int | None = None
|
||||
"""Index of the content block this citation refers to (when the message
|
||||
has multiple content blocks)."""
|
||||
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
|
||||
"""Which kind of content block this citation grounds: the user-visible
|
||||
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
|
||||
tool-plan block (``PLAN``). ``None`` means unspecified."""
|
||||
|
||||
|
||||
class CohereChatMessage(ChatMessage):
|
||||
""":class:`ChatMessage` extension carrying grounding citations.
|
||||
|
||||
Only instantiated by :class:`CohereServingChatV2` (and any other
|
||||
:class:`OpenAIServingChat` subclass that opts in by overriding
|
||||
``_create_chat_message``). Regular OpenAI-compatible handlers keep
|
||||
emitting plain :class:`ChatMessage` so their response schema is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
|
||||
so pydantic serializes this subclass with its own schema (including
|
||||
``citations``) when it flows through
|
||||
:class:`ChatCompletionResponseChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# ``mode="wrap"`` fully overrides (rather than chains) the
|
||||
# parent's ``@model_serializer``, so we explicitly delegate via
|
||||
# ``super()._serialize(handler)`` to preserve the parent's
|
||||
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
|
||||
# unset ``citations`` field so the wire matches the OpenAI-style
|
||||
# contract that optional vLLM extensions are omitted rather than
|
||||
# serialized as ``null``.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
|
||||
|
||||
class CohereDeltaMessage(DeltaMessage):
|
||||
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
|
||||
|
||||
Emitted by :class:`vllm.parser.cohere_command.CohereCommandParser`
|
||||
on delta events whose payload includes citations. Non-Cohere parsers
|
||||
return plain :class:`DeltaMessage`, so their streamed shape is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares
|
||||
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
|
||||
subclass with its own schema when it flows through
|
||||
:class:`ChatCompletionResponseStreamChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# See ``CohereChatMessage._serialize`` for why we delegate via
|
||||
# ``super()`` instead of calling ``handler(self)`` directly.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
@@ -1,389 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cohere Chat v2 API protocol.
|
||||
|
||||
The bulk of the wire types come straight from the official ``cohere``
|
||||
Python SDK so we stay in sync with the upstream specification and
|
||||
avoid re-declaring the schema. We only own three things locally:
|
||||
|
||||
1. The top-level request body model (the SDK doesn't ship one — its
|
||||
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
|
||||
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
|
||||
2. The non-streaming response envelope (the SDK exposes the message
|
||||
shape via :class:`AssistantMessageResponse` but no full response
|
||||
wrapper).
|
||||
3. The streaming discriminated union (the SDK exports each event type
|
||||
individually but not as a combined ``Annotated[Union[...],
|
||||
discriminator]``).
|
||||
|
||||
Importing this module pulls in the ``cohere`` package. The router that
|
||||
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
|
||||
boots without the SDK installed.
|
||||
|
||||
See https://docs.cohere.com/reference/chat for the upstream spec.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from cohere import types as _sdk
|
||||
from cohere.types import (
|
||||
AssistantChatMessageV2,
|
||||
AssistantMessageResponse,
|
||||
ChatMessageV2,
|
||||
ChatRequestSafetyMode,
|
||||
Citation,
|
||||
CitationOptions,
|
||||
Document,
|
||||
ResponseFormatV2,
|
||||
SystemChatMessageV2,
|
||||
Thinking,
|
||||
ToolCallV2,
|
||||
ToolChatMessageV2,
|
||||
ToolV2,
|
||||
UserChatMessageV2,
|
||||
)
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
# Re-export the SDK wire-format types alongside our local extensions so
|
||||
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
|
||||
# they need from this module.
|
||||
__all__ = [
|
||||
"AssistantChatMessageV2",
|
||||
"AssistantMessageResponse",
|
||||
"ChatMessageV2",
|
||||
"Citation",
|
||||
"CitationEndEvent",
|
||||
"CitationOptions",
|
||||
"CitationStartEvent",
|
||||
"CohereChatV2Request",
|
||||
"CohereChatV2Response",
|
||||
"CohereError",
|
||||
"CohereFinishReason",
|
||||
"CohereLogprobItem",
|
||||
"CohereUsage",
|
||||
"CohereUsageBilledUnits",
|
||||
"CohereUsageTokens",
|
||||
"ContentDeltaEvent",
|
||||
"ContentEndEvent",
|
||||
"ContentStartEvent",
|
||||
"Document",
|
||||
"MessageEndEvent",
|
||||
"MessageStartEvent",
|
||||
"ResponseFormatV2",
|
||||
"SystemChatMessageV2",
|
||||
"Thinking",
|
||||
"ToolCallDeltaEvent",
|
||||
"ToolCallEndEvent",
|
||||
"ToolCallStartEvent",
|
||||
"ToolCallV2",
|
||||
"ToolChatMessageV2",
|
||||
"ToolPlanDeltaEvent",
|
||||
"ToolV2",
|
||||
"UserChatMessageV2",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereError(BaseModel):
|
||||
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
|
||||
|
||||
Cohere's documented error schemas are uniform: ``{message, id}``.
|
||||
"""
|
||||
|
||||
message: str
|
||||
id: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool choice / finish reasons
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# These literals aren't first-class enums in the SDK but are documented at
|
||||
# https://docs.cohere.com/reference/chat. We declare them here so the
|
||||
# request/response models can validate them.
|
||||
|
||||
CohereToolChoice = Literal["REQUIRED", "NONE"]
|
||||
|
||||
CohereFinishReason = Literal[
|
||||
"COMPLETE",
|
||||
"STOP_SEQUENCE",
|
||||
"MAX_TOKENS",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
"TIMEOUT",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Request(BaseModel):
|
||||
"""Cohere Chat v2 request body.
|
||||
|
||||
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
|
||||
All structured fields delegate to the official SDK types so the body
|
||||
schema stays in sync with the upstream spec.
|
||||
"""
|
||||
|
||||
model: str
|
||||
messages: list[ChatMessageV2]
|
||||
stream: bool | None = False
|
||||
|
||||
# Tooling
|
||||
tools: list[ToolV2] | None = None
|
||||
strict_tools: bool | None = None
|
||||
tool_choice: CohereToolChoice | None = None
|
||||
|
||||
# Grounding
|
||||
documents: list[str | Document] | None = None
|
||||
citation_options: CitationOptions | None = None
|
||||
|
||||
# Output
|
||||
response_format: ResponseFormatV2 | None = None
|
||||
safety_mode: ChatRequestSafetyMode | None = None
|
||||
max_tokens: int | None = None
|
||||
stop_sequences: (
|
||||
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
) = None
|
||||
|
||||
# Sampling
|
||||
temperature: float | None = None
|
||||
seed: int | None = None
|
||||
frequency_penalty: float | None = None
|
||||
presence_penalty: float | None = None
|
||||
k: int | None = None
|
||||
p: float | None = None
|
||||
logprobs: bool | None = None
|
||||
|
||||
# Reasoning
|
||||
thinking: Thinking | None = None
|
||||
|
||||
# Scheduling
|
||||
priority: int | None = Field(default=None, ge=-(2**63), le=2**63 - 1)
|
||||
|
||||
# vLLM-specific extensions (not in Cohere spec). These mirror what the
|
||||
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
|
||||
# the same engine knobs when needed.
|
||||
watermarking: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Whether to apply the engine's configured watermark to this request."
|
||||
),
|
||||
)
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def _validate_model(cls, v: str) -> str:
|
||||
if not v:
|
||||
raise VLLMValidationError("model is required", parameter="model")
|
||||
return v
|
||||
|
||||
@field_validator("max_tokens")
|
||||
@classmethod
|
||||
def _validate_max_tokens(cls, v: int | None) -> int | None:
|
||||
if v is not None and v < 0:
|
||||
raise VLLMValidationError(
|
||||
"max_tokens must be non-negative",
|
||||
parameter="max_tokens",
|
||||
value=v,
|
||||
)
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_tool_choice(self) -> CohereChatV2Request:
|
||||
if self.tool_choice == "REQUIRED" and not self.tools:
|
||||
raise VLLMValidationError(
|
||||
'tool_choice="REQUIRED" requires at least one tool in tools.',
|
||||
parameter="tool_choice",
|
||||
value=self.tool_choice,
|
||||
)
|
||||
return self
|
||||
|
||||
@field_validator("messages", mode="before")
|
||||
@classmethod
|
||||
def _normalize_message_roles(cls, v: Any) -> Any:
|
||||
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
|
||||
|
||||
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
|
||||
the four literal roles ``user`` / ``assistant`` / ``system`` /
|
||||
``tool``. OpenAI's ``developer`` role is documented as
|
||||
high-priority system instructions, so we alias it onto
|
||||
``system`` *before* the SDK discriminator runs (otherwise it
|
||||
rejects the message with a ``literal_error`` against each union
|
||||
member). Mirrors ``_role_to_melody`` in the renderer so the
|
||||
same alias is honoured no matter which surface the message
|
||||
arrives through.
|
||||
|
||||
``mode="before"`` is required so the rewrite happens before the
|
||||
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
|
||||
union; a default-mode validator would never see ``developer``
|
||||
because validation would have already failed. On any
|
||||
structural malformation (non-iterable input, items without a
|
||||
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
|
||||
and let Pydantic's normal coercion surface a precise error.
|
||||
"""
|
||||
try:
|
||||
return [
|
||||
{**msg, "role": "system"}
|
||||
if msg.get("role", "").lower() == "developer"
|
||||
else msg
|
||||
for msg in v
|
||||
]
|
||||
except (AttributeError, TypeError):
|
||||
return v
|
||||
|
||||
@field_validator("messages")
|
||||
@classmethod
|
||||
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
|
||||
if not v:
|
||||
raise VLLMValidationError(
|
||||
"messages must contain at least one message", parameter="messages"
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage / Logprobs
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
|
||||
# envelope is documented separately in the OpenAPI spec and we declare it
|
||||
# here.
|
||||
|
||||
|
||||
class CohereUsageBilledUnits(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
search_units: float | None = None
|
||||
classifications: float | None = None
|
||||
|
||||
|
||||
class CohereUsageTokens(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereUsage(BaseModel):
|
||||
billed_units: CohereUsageBilledUnits | None = None
|
||||
tokens: CohereUsageTokens | None = None
|
||||
cached_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereLogprobItem(BaseModel):
|
||||
text: str | None = None
|
||||
token_ids: list[int]
|
||||
logprobs: list[float] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-streaming response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Response(BaseModel):
|
||||
"""Cohere Chat v2 non-streaming response body.
|
||||
|
||||
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
|
||||
the documented v2 response envelope (``id``, ``finish_reason``,
|
||||
``usage``, ``logprobs``). The single constructor in
|
||||
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
|
||||
supplying a non-empty ``id`` (falling back to a synthesized one if
|
||||
the upstream response is missing it) to this model.
|
||||
"""
|
||||
|
||||
id: str
|
||||
finish_reason: CohereFinishReason
|
||||
message: AssistantMessageResponse
|
||||
usage: CohereUsage | None = None
|
||||
logprobs: list[CohereLogprobItem] | None = None
|
||||
|
||||
# vLLM-specific extension.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming events
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
|
||||
# Sent Events; each event's ``type`` field carries the discriminator. The
|
||||
# SDK exposes a Pydantic model per event but none of them declare ``type``
|
||||
# as a field (the SDK relies on its own deserializer for discrimination),
|
||||
# so a naive ``model_dump_json()`` would silently drop the discriminator
|
||||
# and break clients that demux on ``type``.
|
||||
#
|
||||
# We therefore subclass each SDK event and bake the wire-format ``type``
|
||||
# string in as a ``Literal`` field with a default. ``model_dump()`` now
|
||||
# emits ``type`` for free, and surfaces can simply construct the event
|
||||
# class and serialize it -- no manual ``type`` parameter needed.
|
||||
#
|
||||
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
|
||||
# ``StreamedChatResponseV2`` schema for the wire-format reference.
|
||||
|
||||
|
||||
class MessageStartEvent(_sdk.ChatMessageStartEvent):
|
||||
type: Literal["message-start"] = "message-start"
|
||||
|
||||
|
||||
class ContentStartEvent(_sdk.ChatContentStartEvent):
|
||||
type: Literal["content-start"] = "content-start"
|
||||
|
||||
|
||||
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
|
||||
type: Literal["content-delta"] = "content-delta"
|
||||
|
||||
|
||||
class ContentEndEvent(_sdk.ChatContentEndEvent):
|
||||
type: Literal["content-end"] = "content-end"
|
||||
|
||||
|
||||
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
|
||||
type: Literal["tool-plan-delta"] = "tool-plan-delta"
|
||||
|
||||
|
||||
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
|
||||
type: Literal["tool-call-start"] = "tool-call-start"
|
||||
|
||||
|
||||
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
|
||||
type: Literal["tool-call-delta"] = "tool-call-delta"
|
||||
|
||||
|
||||
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
|
||||
type: Literal["tool-call-end"] = "tool-call-end"
|
||||
|
||||
|
||||
class CitationStartEvent(_sdk.CitationStartEvent):
|
||||
type: Literal["citation-start"] = "citation-start"
|
||||
|
||||
|
||||
class CitationEndEvent(_sdk.CitationEndEvent):
|
||||
type: Literal["citation-end"] = "citation-end"
|
||||
|
||||
|
||||
class MessageEndEvent(_sdk.ChatMessageEndEvent):
|
||||
type: Literal["message-end"] = "message-end"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,7 +31,6 @@ from vllm.renderers.inputs.preprocess import (
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.utils.counter import Counter
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.utils.tqdm_utils import maybe_tqdm
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
@@ -191,10 +190,7 @@ class OfflineInferenceMixin:
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
continue_final_message=continue_final_message,
|
||||
tools=tools,
|
||||
tokenize=(
|
||||
is_mistral_tokenizer(renderer.tokenizer)
|
||||
or self.model_config.enable_prompt_embeds
|
||||
),
|
||||
tokenize=self.model_config.enable_prompt_embeds,
|
||||
),
|
||||
),
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
|
||||
@@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
@@ -37,17 +35,6 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_completion_api_router(app)
|
||||
|
||||
from vllm.entrypoints.anthropic.api_router import (
|
||||
attach_router as register_anthropic_api_router,
|
||||
)
|
||||
|
||||
register_anthropic_api_router(app)
|
||||
|
||||
from vllm.entrypoints.cohere.api_router import (
|
||||
attach_router as register_cohere_api_router,
|
||||
)
|
||||
|
||||
register_cohere_api_router(app)
|
||||
|
||||
from .generative_scoring.api_router import register_generative_scoring_api_router
|
||||
|
||||
@@ -62,24 +49,6 @@ async def init_generate_state(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
|
||||
# The Cohere serving handler depends on the optional `cohere` SDK for
|
||||
# its wire-format protocol models, and is additionally gated on the
|
||||
# `VLLM_ENABLE_COHERE_API` env flag (see
|
||||
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
|
||||
# entirely when the endpoint isn't going to be exposed, both because
|
||||
# the SDK may not be installed and because the serving object holds
|
||||
# nontrivial state (chat handler, warmup) that would otherwise be
|
||||
# unused.
|
||||
if envs.VLLM_ENABLE_COHERE_API:
|
||||
try:
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
except ImportError:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
else:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
OpenAIServingChatBatch,
|
||||
)
|
||||
@@ -87,6 +56,7 @@ async def init_generate_state(
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
|
||||
# Applied before any serving class is constructed so that each one picks
|
||||
# up the chosen mode on its first cache miss.
|
||||
@@ -168,54 +138,6 @@ async def init_generate_state(
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.anthropic_serving_messages = (
|
||||
AnthropicServingMessages(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
disabled_thinking_effort=args.anthropic_disabled_thinking_effort,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.cohere_serving_chat_v2 = (
|
||||
CohereServingChatV2(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
is_reasoning_model=args.cohere_is_reasoning_model,
|
||||
)
|
||||
if CohereServingChatV2 is not None and "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
|
||||
from .generative_scoring.serving import ServingGenerativeScoring
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
EndpointFn,
|
||||
GetHandlerFn,
|
||||
RequestType,
|
||||
)
|
||||
|
||||
|
||||
def get_generate_invocation_types(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.openai.chat_completion.api_router import (
|
||||
chat,
|
||||
create_chat_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.api_router import (
|
||||
completion,
|
||||
create_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
|
||||
invocation_types += [
|
||||
(ChatCompletionRequest, (chat, create_chat_completion)),
|
||||
(CompletionRequest, (completion, create_completion)),
|
||||
]
|
||||
|
||||
return invocation_types
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.decisions.state import init_decisions_state
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .protocol import StructuredDecisionRequest
|
||||
from .serving import ServingStructuredDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
|
||||
return getattr(request.app.state, "serving_structured_decisions", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/systemone",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_structured_decision(
|
||||
request: StructuredDecisionRequest, raw_request: Request
|
||||
):
|
||||
handler = structured_decisions(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support the structured decision API"
|
||||
)
|
||||
result = await handler.create_decision(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_structured_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
def init_structured_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
strategy = init_decisions_state(
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
load_chat_template(args.chat_template),
|
||||
default_chat_template_kwargs,
|
||||
supported_tasks,
|
||||
)
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models, strategy, request_logger=request_logger
|
||||
)
|
||||
if strategy is not None
|
||||
else None
|
||||
)
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Request and response models of the structured decisions API (/v1/systemone)."""
|
||||
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
class QuestionSpec(OpenAIBaseModel):
|
||||
type: str
|
||||
instructions: Any = ""
|
||||
criteria: Any = None
|
||||
|
||||
|
||||
class StructuredDecisionRequest(OpenAIBaseModel):
|
||||
model: str | None = None
|
||||
state: Any = Field(..., description="What the questions are about: text or JSON.")
|
||||
questions: dict[str, QuestionSpec] = Field(
|
||||
..., description="Question id to question, asked in this order."
|
||||
)
|
||||
instructions: str | None = Field(
|
||||
default=None, description="Context placed ahead of the questions."
|
||||
)
|
||||
images: list[str] | None = Field(
|
||||
default=None, description="Image URLs or data URLs, for models that see."
|
||||
)
|
||||
audio: list[str] | None = Field(
|
||||
default=None, description="Audio URLs or data URLs, for models that hear."
|
||||
)
|
||||
videos: list[str] | None = Field(
|
||||
default=None, description="Video URLs or data URLs, for models that see."
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
request_id: str = Field(default_factory=random_uuid)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class DecisionUsage(OpenAIBaseModel):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class QuestionDiagnostics(OpenAIBaseModel):
|
||||
label_mass: float = Field(
|
||||
description="Probability the model put on the labels, over the full "
|
||||
"vocabulary. A low value means most of the probability went to tokens "
|
||||
"that are not labels."
|
||||
)
|
||||
argmax_is_label: bool
|
||||
|
||||
|
||||
class StructuredDecisionResponse(OpenAIBaseModel):
|
||||
id: str
|
||||
object: Literal["structured_decision"] = "structured_decision"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
answers: dict[str, dict[str, Any]]
|
||||
usage: DecisionUsage
|
||||
diagnostics: dict[str, QuestionDiagnostics]
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.serving import BaseServingDecisions
|
||||
from vllm.entrypoints.openai.decisions.strategies import DecisionLimits
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
|
||||
from .protocol import (
|
||||
DecisionUsage,
|
||||
QuestionDiagnostics,
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
|
||||
|
||||
def parse_questions(
|
||||
request: StructuredDecisionRequest,
|
||||
limits: DecisionLimits,
|
||||
) -> list[Question]:
|
||||
if not request.questions:
|
||||
raise StructuredDecisionError("questions: needs at least one question")
|
||||
if len(request.questions) > limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {limits.max_questions} for this model"
|
||||
)
|
||||
questions = []
|
||||
for qid, spec in request.questions.items():
|
||||
if spec.model_extra:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
|
||||
)
|
||||
questions.append(
|
||||
build_question(
|
||||
qid,
|
||||
spec.type,
|
||||
spec.instructions,
|
||||
spec.criteria,
|
||||
limits.max_options,
|
||||
labeled=limits.labeled,
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
class ServingStructuredDecisions(BaseServingDecisions):
|
||||
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
|
||||
urls = {
|
||||
"image": request.images,
|
||||
"audio": request.audio,
|
||||
"video": request.videos,
|
||||
}
|
||||
media = {modality: items for modality, items in urls.items() if items}
|
||||
if unsupported := media.keys() - self.strategy.modalities:
|
||||
names = ", ".join(sorted(unsupported))
|
||||
raise StructuredDecisionError(f"this model does not take {names} inputs")
|
||||
return media
|
||||
|
||||
async def create_decision(
|
||||
self,
|
||||
request: StructuredDecisionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> StructuredDecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine_client = self.strategy.context.engine_client
|
||||
if engine_client.errored:
|
||||
raise engine_client.dead_error
|
||||
|
||||
base_id = self._base_request_id(raw_request, default=request.request_id)
|
||||
request_id = f"decision-{base_id}"
|
||||
try:
|
||||
questions = parse_questions(request, self.limits)
|
||||
media = self._media(request)
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine_client.check_admission(self.strategy.num_requests(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
request.instructions,
|
||||
self.strategy.render_state(request.state),
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
media=media,
|
||||
)
|
||||
except StructuredDecisionError as e:
|
||||
return self.create_error_response(e)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
|
||||
answers: dict[str, dict[str, Any]] = {}
|
||||
diagnostics: dict[str, QuestionDiagnostics] = {}
|
||||
for q, read in zip(questions, reads):
|
||||
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
|
||||
diagnostics[q.id] = QuestionDiagnostics(
|
||||
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
|
||||
)
|
||||
return StructuredDecisionResponse(
|
||||
id=request_id,
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=answers,
|
||||
usage=DecisionUsage(
|
||||
input_tokens=sum(r.input_tokens for r in reads),
|
||||
output_tokens=sum(r.output_tokens for r in reads),
|
||||
),
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
@@ -7,7 +7,6 @@ from typing import cast
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
@@ -16,7 +15,6 @@ from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
@@ -65,7 +63,7 @@ async def init_app_state(
|
||||
state.vllm_config = vllm_config
|
||||
state.args = args
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
state.tool_server = (
|
||||
await init_tool_server(args) if "generate" in supported_tasks else None
|
||||
)
|
||||
@@ -103,22 +101,6 @@ async def init_app_state(
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
@@ -152,19 +134,6 @@ async def init_app_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
init_structured_decisions_state,
|
||||
)
|
||||
|
||||
init_structured_decisions_state(
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
supported_tasks,
|
||||
default_chat_template_kwargs,
|
||||
)
|
||||
|
||||
await init_endpoint_plugins_state(engine_client, state, args)
|
||||
|
||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||
|
||||
@@ -25,33 +25,11 @@ def register_api_routers(
|
||||
|
||||
register_models_api_router(app)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
)
|
||||
|
||||
register_structured_decisions_api_router(app)
|
||||
|
||||
from vllm.entrypoints.openai.decisions.api_router import (
|
||||
register_decisions_api_router,
|
||||
)
|
||||
|
||||
register_decisions_api_router(app)
|
||||
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
attach_router as register_sagemaker_api_router,
|
||||
)
|
||||
|
||||
register_sagemaker_api_router(app, supported_tasks, model_config)
|
||||
|
||||
if envs.VLLM_SERVER_DEV_MODE:
|
||||
from vllm.entrypoints.serve import register_vllm_dev_api_routers
|
||||
|
||||
register_vllm_dev_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.rl.factories import register_rl_api_routers
|
||||
|
||||
register_rl_api_routers(app)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import (
|
||||
register_generate_api_routers,
|
||||
@@ -59,13 +37,6 @@ def register_api_routers(
|
||||
|
||||
register_generate_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.serve.elastic_ep.api_router import (
|
||||
attach_router as elastic_ep_attach_router,
|
||||
)
|
||||
|
||||
elastic_ep_attach_router(app)
|
||||
|
||||
if "generate" in supported_tasks or "render" in supported_tasks:
|
||||
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||
|
||||
register_scale_out_api_routers(app, supported_tasks)
|
||||
|
||||
@@ -8,7 +8,6 @@ from fastapi import FastAPI
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
@@ -35,22 +34,16 @@ def build_app(
|
||||
app = FastAPI(
|
||||
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
|
||||
)
|
||||
elif args.enable_offline_docs:
|
||||
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
else:
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
app.state.args = args
|
||||
app.root_path = args.root_path
|
||||
|
||||
register_api_routers(args, app, supported_tasks, model_config)
|
||||
|
||||
# Endpoint plugin hooks run after core routers so plugins can inspect them.
|
||||
# This runs even for the CPU only render server. A plugin eligible for
|
||||
# the `render` task still gets its routes registered. It receives
|
||||
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
||||
attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
init_exception_handler(app)
|
||||
init_entrypoints_middleware(args, app, supported_tasks)
|
||||
app = sagemaker_standards_bootstrap(app)
|
||||
return app
|
||||
|
||||
@@ -14,9 +14,6 @@ from typing import Any, Literal
|
||||
import vllm.envs as envs
|
||||
from vllm.config import config
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicDisabledThinkingEffortOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.chat_utils import (
|
||||
@@ -172,32 +169,6 @@ class BaseFrontendArgs:
|
||||
"""If set to False, output deltas will not be logged. Relevant only if
|
||||
--enable-log-outputs is set.
|
||||
"""
|
||||
cohere_is_reasoning_model: bool = True
|
||||
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
|
||||
reasoning Command-family model. When True (default), the assistant's
|
||||
chain-of-thought is surfaced as a ``thinking`` content block (or
|
||||
``content-*`` events on the stream). When False, reasoning is
|
||||
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
|
||||
events) whenever the model emits tool calls, matching older non-
|
||||
reasoning Command models. Has no effect on the non-Cohere
|
||||
endpoints."""
|
||||
cohere_format: str = "cmd4"
|
||||
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
|
||||
format to render: ``cmd4`` (current Command A+ models; default) or
|
||||
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
|
||||
wrong format silently produces a prompt the model wasn't trained
|
||||
on, which most commonly manifests as the model emitting text but no
|
||||
citations / tool calls / thinking blocks. Equivalent to passing
|
||||
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
|
||||
explicit request-level ``chat_template_kwargs.cohere_format`` takes
|
||||
priority."""
|
||||
anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto"
|
||||
"""Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for
|
||||
requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking
|
||||
off for models that support it; ``low`` suits models that always think
|
||||
(e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses
|
||||
``low`` when the renderer rejects ``none`` or renders it the same as a
|
||||
thinking effort, and ``none`` otherwise."""
|
||||
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
@@ -207,10 +178,10 @@ class BaseFrontendArgs:
|
||||
"""
|
||||
enable_scale_out: bool = False
|
||||
"""
|
||||
If set to True, register the scale-out endpoints (`/render`, `/derender`,
|
||||
`/inference/v1/generate` and `/inference/v1/abort_requests`) on `vllm serve`.
|
||||
Has no effect on `vllm launch render` or `vllm serve --tokens-only` which
|
||||
always register their required endpoints regardless of this flag.
|
||||
If set to True, register the scale-out endpoints (`/inference/v1/generate`
|
||||
and `/inference/v1/abort_requests`) on `vllm serve`. `vllm serve
|
||||
--tokens-only` always registers the endpoints it needs regardless of this
|
||||
flag.
|
||||
"""
|
||||
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
||||
"""Controls the ``system_fingerprint`` field on responses.
|
||||
@@ -360,11 +331,6 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
|
||||
"""Maximum number of HTTP headers allowed in a request for h11 parser.
|
||||
Helps mitigate header abuse. Default: 256."""
|
||||
enable_offline_docs: bool = False
|
||||
"""
|
||||
Enable offline FastAPI documentation for air-gapped environments.
|
||||
Uses vendored static assets bundled with vLLM.
|
||||
"""
|
||||
enable_flash_late_interaction: bool = True
|
||||
"""If set, run pooling score MaxSim on GPU in the API server process.
|
||||
Can significantly improve late-interaction scoring performance.
|
||||
@@ -404,16 +370,6 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
return frontend_kwargs
|
||||
|
||||
|
||||
def resolve_default_chat_template_kwargs(
|
||||
args: argparse.Namespace,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
|
||||
kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
return kwargs
|
||||
|
||||
|
||||
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Create the CLI argument parser used by the OpenAI API server.
|
||||
|
||||
@@ -453,9 +409,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
|
||||
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. "
|
||||
"Cannot be combined with --grpc-port, which enables the Rust "
|
||||
"frontend's gRPC services.",
|
||||
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc].",
|
||||
)
|
||||
parser = FrontendArgs.add_cli_args(parser)
|
||||
parser = AsyncEngineArgs.add_cli_args(parser)
|
||||
@@ -465,9 +419,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
|
||||
def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"""Quick checks for model serve args that raise prior to loading."""
|
||||
# `vllm launch <component>` builds its parser with make_arg_parser too (see
|
||||
# LaunchSubcommandBase.add_cli_args), so its args are serve args as well.
|
||||
if hasattr(args, "subparser") and args.subparser not in ("serve", "launch"):
|
||||
if hasattr(args, "subparser") and args.subparser != "serve":
|
||||
return
|
||||
|
||||
# Ensure that the chat template is valid; raises if it likely isn't
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
|
||||
async def init_render_app_state(
|
||||
vllm_config: VllmConfig,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
) -> None:
|
||||
"""Initialise FastAPI app state for a CPU-only render server.
|
||||
|
||||
Unlike :func:`init_app_state` this function does not require an
|
||||
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
|
||||
preprocessing pipeline (renderer, input_processor)
|
||||
directly from the :class:`~vllm.config.VllmConfig`.
|
||||
"""
|
||||
served_model_names = args.served_model_name or [args.model]
|
||||
model_registry = OpenAIModelRegistry(
|
||||
model_config=vllm_config.model_config,
|
||||
base_model_paths=[
|
||||
BaseModelPath(name=name, model_path=args.model)
|
||||
for name in served_model_names
|
||||
],
|
||||
)
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
else:
|
||||
request_logger = None
|
||||
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
state.tool_server = await init_tool_server(args)
|
||||
|
||||
# The config-resolved reasoning parser carries both the CLI flag (merged
|
||||
# in the entrypoint via `create_structured_outputs_config`) and any
|
||||
# model-specific default applied by `verify_and_update_config`
|
||||
# (e.g. "openai_gptoss" for gpt_oss), matching the main API server.
|
||||
# Keep `args.reasoning_parser` as the first source so callers that build
|
||||
# a VllmConfig without that merge still honor an explicit flag.
|
||||
reasoning_parser = (
|
||||
args.reasoning_parser or vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=reasoning_parser,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.openai_serving_models = model_registry
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
model_registry,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
init_render_state(state, request_logger)
|
||||
|
||||
state.vllm_config = vllm_config
|
||||
# Disable stats logging — there is no engine to poll.
|
||||
state.log_stats = False
|
||||
state.engine_client = None
|
||||
state.args = args
|
||||
state.enable_server_load_tracking = False
|
||||
state.server_load_metrics = 0
|
||||
|
||||
# No `EngineClient` exists for the render server, so plugins get `None` and
|
||||
# must handle it themselves (see `EndpointPlugin.init_state`).
|
||||
await init_endpoint_plugins_state(None, state, args)
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import asyncio
|
||||
import signal
|
||||
import socket
|
||||
from argparse import Namespace
|
||||
|
||||
from vllm import AsyncEngineArgs, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import configure_logging_from_args, init_logger
|
||||
|
||||
from ..app import build_app
|
||||
from ..launcher import serve_http, setup_server
|
||||
from ..utils.server_utils import get_uvicorn_log_config
|
||||
from .app_state import init_render_app_state
|
||||
|
||||
logger = init_logger("vllm.entrypoints.launchers.render.entry")
|
||||
|
||||
|
||||
async def build_and_serve_renderer(
|
||||
vllm_config: VllmConfig,
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: Namespace,
|
||||
**uvicorn_kwargs,
|
||||
) -> asyncio.Task:
|
||||
"""Build FastAPI app for a CPU-only render server, initialize state, and
|
||||
start serving.
|
||||
|
||||
Returns the shutdown task for the caller to await.
|
||||
"""
|
||||
# Get uvicorn log config (from file or with endpoint filter)
|
||||
log_config = get_uvicorn_log_config(args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
app = build_app(args, ("render",))
|
||||
await init_render_app_state(vllm_config, app.state, args)
|
||||
|
||||
logger.info("Starting vLLM server on %s", listen_address)
|
||||
|
||||
return await serve_http(
|
||||
app,
|
||||
sock=sock,
|
||||
enable_ssl_refresh=args.enable_ssl_refresh,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level=args.uvicorn_log_level,
|
||||
# NOTE: When the 'disable_uvicorn_access_log' value is True,
|
||||
# no access log will be output.
|
||||
access_log=not args.disable_uvicorn_access_log,
|
||||
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
|
||||
ssl_keyfile=args.ssl_keyfile,
|
||||
ssl_certfile=args.ssl_certfile,
|
||||
ssl_ca_certs=args.ssl_ca_certs,
|
||||
ssl_cert_reqs=args.ssl_cert_reqs,
|
||||
ssl_ciphers=args.ssl_ciphers,
|
||||
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
|
||||
h11_max_header_count=args.h11_max_header_count,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def run_launch_fastapi(args: argparse.Namespace) -> None:
|
||||
"""Run the online serving layer with FastAPI (no GPU inference)."""
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs
|
||||
# its own signal handlers. Once uvicorn is running it replaces this.
|
||||
def _interrupt_init(*_) -> None:
|
||||
raise KeyboardInterrupt("terminated")
|
||||
|
||||
signal.signal(signal.SIGTERM, _interrupt_init)
|
||||
|
||||
# 1. Socket binding
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
|
||||
# 2. Build and serve the API server
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
model_config = engine_args.create_model_config()
|
||||
|
||||
# Render servers preprocess data only — no inference, no quantized kernels.
|
||||
# Clear quantization so VllmConfig skips quant dtype/capability validation.
|
||||
model_config.quantization = None
|
||||
|
||||
# Render servers never allocate KV cache; suppress the spurious CPU KV
|
||||
# cache space warning from CpuPlatform.check_and_update_config.
|
||||
envs.VLLM_CPU_KVCACHE_SPACE = 0
|
||||
|
||||
# Merge frontend parser flags (e.g. --reasoning-parser) into the
|
||||
# structured outputs config, mirroring `create_engine_config`, which this
|
||||
# GPU-less entrypoint bypasses. Model-specific defaults are then applied
|
||||
# by `verify_and_update_config` during VllmConfig construction.
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
logging_config=engine_args.create_logging_config(),
|
||||
structured_outputs_config=engine_args.create_structured_outputs_config(),
|
||||
)
|
||||
shutdown_task = await build_and_serve_renderer(
|
||||
vllm_config, listen_address, sock, args
|
||||
)
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvloop
|
||||
|
||||
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from ..cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
|
||||
cli_env_setup()
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Starts a GPU-less rendering server "
|
||||
"for preprocessing and postprocessing only"
|
||||
)
|
||||
parser = make_arg_parser(parser)
|
||||
args = parser.parse_args()
|
||||
configure_logging_from_args(args)
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
uvloop.run(run_launch_fastapi(args))
|
||||
@@ -1,672 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from argparse import Namespace
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from io import StringIO
|
||||
from typing import Any, NoReturn, TypeAlias
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import aiohttp
|
||||
import pybase64 as base64
|
||||
import pydantic
|
||||
import torch
|
||||
from prometheus_client import start_http_server
|
||||
from pydantic import TypeAdapter, field_validator
|
||||
from pydantic_core.core_schema import ValidationInfo
|
||||
from starlette.datastructures import State
|
||||
from starlette.responses import JSONResponse
|
||||
from tqdm import tqdm
|
||||
from urllib3.util import parse_url
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import config
|
||||
from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import configure_logging_from_args, init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
from .api_server.entry import init_app_state
|
||||
from .cli_args import BaseFrontendArgs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_audio_download_limit_bytes() -> int:
|
||||
return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
|
||||
|
||||
|
||||
def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
|
||||
raise VLLMValidationError(
|
||||
"Maximum file size exceeded",
|
||||
parameter="audio_filesize_mb",
|
||||
value=size_bytes / MiB_bytes,
|
||||
)
|
||||
|
||||
|
||||
BatchRequestInputBody: TypeAlias = ChatCompletionRequest
|
||||
|
||||
|
||||
class BatchRequestInput(OpenAIBaseModel):
|
||||
"""The per-line object of the batch input file.
|
||||
|
||||
NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
|
||||
"""
|
||||
|
||||
# A developer-provided per-request id that will be used to match outputs to
|
||||
# inputs. Must be unique for each request in a batch.
|
||||
custom_id: str
|
||||
|
||||
# The HTTP method to be used for the request. Currently only POST is
|
||||
# supported.
|
||||
method: str
|
||||
|
||||
# The OpenAI API relative URL to be used for the request. Currently
|
||||
# /v1/chat/completions is supported.
|
||||
url: str
|
||||
|
||||
# The parameters of the request.
|
||||
body: BatchRequestInputBody
|
||||
|
||||
@field_validator("body", mode="plain")
|
||||
@classmethod
|
||||
def check_type_for_url(cls, value: Any, info: ValidationInfo):
|
||||
# Use url to disambiguate models
|
||||
url: str = info.data["url"]
|
||||
if url == "/v1/chat/completions":
|
||||
return ChatCompletionRequest.model_validate(value)
|
||||
return TypeAdapter(BatchRequestInputBody).validate_python(value)
|
||||
|
||||
|
||||
AllResponse: TypeAlias = (
|
||||
ChatCompletionResponse
|
||||
)
|
||||
|
||||
|
||||
class BatchResponseData(OpenAIBaseModel):
|
||||
# HTTP status code of the response.
|
||||
status_code: int = 200
|
||||
|
||||
# An unique identifier for the API request.
|
||||
request_id: str
|
||||
|
||||
# The body of the response.
|
||||
body: AllResponse | None = None
|
||||
|
||||
|
||||
class BatchRequestOutput(OpenAIBaseModel):
|
||||
"""The per-line object of the batch output and error files"""
|
||||
|
||||
id: str
|
||||
|
||||
# A developer-provided per-request id that will be used to match outputs to
|
||||
# inputs.
|
||||
custom_id: str
|
||||
|
||||
response: BatchResponseData | None
|
||||
|
||||
# For requests that failed with a non-HTTP error, this will contain more
|
||||
# information on the cause of the failure.
|
||||
error: Any | None
|
||||
|
||||
|
||||
@config
|
||||
class BatchFrontendArgs(BaseFrontendArgs):
|
||||
"""Arguments for the batch runner frontend."""
|
||||
|
||||
input_file: str | None = None
|
||||
"""The path or url to a single input file. Currently supports local file
|
||||
paths, or the http protocol (http or https). If a URL is specified,
|
||||
the file should be available via HTTP GET."""
|
||||
output_file: str | None = None
|
||||
"""The path or url to a single output file. Currently supports
|
||||
local file paths, or web (http or https) urls. If a URL is specified,
|
||||
the file should be available via HTTP PUT."""
|
||||
output_tmp_dir: str | None = None
|
||||
"""The directory to store the output file before uploading it
|
||||
to the output URL."""
|
||||
enable_metrics: bool = False
|
||||
"""Enable Prometheus metrics"""
|
||||
host: str | None = None
|
||||
"""Host name for the Prometheus metrics server
|
||||
(only needed if enable-metrics is set)."""
|
||||
port: int = 8000
|
||||
"""Port number for the Prometheus metrics server
|
||||
(only needed if enable-metrics is set)."""
|
||||
url: str = "0.0.0.0"
|
||||
"""[DEPRECATED] Host name for the Prometheus metrics server
|
||||
(only needed if enable-metrics is set). Use --host instead."""
|
||||
|
||||
@classmethod
|
||||
def _customize_cli_kwargs(
|
||||
cls,
|
||||
frontend_kwargs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
|
||||
|
||||
frontend_kwargs["input_file"]["flags"] = ["-i"]
|
||||
frontend_kwargs["input_file"]["required"] = True
|
||||
frontend_kwargs["output_file"]["flags"] = ["-o"]
|
||||
frontend_kwargs["output_file"]["required"] = True
|
||||
|
||||
frontend_kwargs["enable_metrics"]["action"] = "store_true"
|
||||
|
||||
frontend_kwargs["url"]["deprecated"] = True
|
||||
return frontend_kwargs
|
||||
|
||||
|
||||
def make_arg_parser(parser: FlexibleArgumentParser):
|
||||
parser = BatchFrontendArgs.add_cli_args(parser)
|
||||
parser = AsyncEngineArgs.add_cli_args(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
|
||||
args = make_arg_parser(parser).parse_args()
|
||||
configure_logging_from_args(args)
|
||||
|
||||
# Backward compatibility: If --url is set, use it for host
|
||||
url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
|
||||
host_explicit = any(
|
||||
arg == "--host" or arg.startswith("--host=") for arg in sys.argv
|
||||
)
|
||||
if url_explicit and hasattr(args, "url") and not host_explicit:
|
||||
args.host = args.url
|
||||
logger.warning_once(
|
||||
"Using --url for metrics is deprecated. Please use --host instead."
|
||||
)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
# explicitly use pure text format, with a newline at the end
|
||||
# this makes it impossible to see the animation in the progress bar
|
||||
# but will avoid messing up with ray or multiprocessing, which wraps
|
||||
# each line of output with some prefix.
|
||||
_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
|
||||
|
||||
|
||||
class BatchProgressTracker:
|
||||
def __init__(self):
|
||||
self._total = 0
|
||||
self._pbar: tqdm | None = None
|
||||
|
||||
def submitted(self):
|
||||
self._total += 1
|
||||
|
||||
def completed(self):
|
||||
if self._pbar:
|
||||
self._pbar.update()
|
||||
|
||||
def pbar(self) -> tqdm:
|
||||
enable_tqdm = (
|
||||
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
|
||||
)
|
||||
self._pbar = tqdm(
|
||||
total=self._total,
|
||||
unit="req",
|
||||
desc="Running batch",
|
||||
mininterval=5,
|
||||
disable=not enable_tqdm,
|
||||
bar_format=_BAR_FORMAT,
|
||||
)
|
||||
return self._pbar
|
||||
|
||||
|
||||
async def read_file(path_or_url: str) -> str:
|
||||
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
|
||||
async with aiohttp.ClientSession() as session, session.get(path_or_url) as resp:
|
||||
resp.raise_for_status()
|
||||
return await resp.text()
|
||||
else:
|
||||
with open(path_or_url, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
async def write_local_file(
|
||||
output_path: str, batch_outputs: list[BatchRequestOutput]
|
||||
) -> None:
|
||||
"""Write the responses to a local file.
|
||||
output_path: The path to write the responses to.
|
||||
batch_outputs: The list of batch outputs to write.
|
||||
"""
|
||||
# We should make this async, but as long as run_batch runs as a
|
||||
# standalone program, blocking the event loop won't affect performance.
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
for o in batch_outputs:
|
||||
print(o.model_dump_json(), file=f)
|
||||
|
||||
|
||||
async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
|
||||
"""Upload a local file to a URL.
|
||||
output_url: The URL to upload the file to.
|
||||
data_or_file: Either the data to upload or the path to the file to upload.
|
||||
from_file: If True, data_or_file is the path to the file to upload.
|
||||
"""
|
||||
# Timeout is a common issue when uploading large files.
|
||||
# We retry max_retries times before giving up.
|
||||
max_retries = 5
|
||||
# Number of seconds to wait before retrying.
|
||||
delay = 5
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
# We increase the timeout to 1000 seconds to allow
|
||||
# for large files (default is 300).
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=1000)
|
||||
) as session:
|
||||
if from_file:
|
||||
with open(data_or_file, "rb") as file:
|
||||
async with session.put(output_url, data=file) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
error_text = await response.text()
|
||||
raise Exception(
|
||||
f"Failed to upload file.\n"
|
||||
f"Status: {response.status}\n"
|
||||
f"Response: {error_text}"
|
||||
)
|
||||
else:
|
||||
async with session.put(output_url, data=data_or_file) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
error_text = await response.text()
|
||||
raise Exception(
|
||||
f"Failed to upload data.\n"
|
||||
f"Status: {response.status}\n"
|
||||
f"Response: {error_text}"
|
||||
)
|
||||
# Upload succeeded; do not retry.
|
||||
return
|
||||
except Exception as e:
|
||||
if attempt < max_retries:
|
||||
logger.error(
|
||||
"Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...", # noqa: E501
|
||||
attempt,
|
||||
e,
|
||||
delay,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
raise Exception(
|
||||
f"Failed to upload data (attempt {attempt}). Error message: {str(e)}." # noqa: E501
|
||||
) from e
|
||||
|
||||
|
||||
async def write_file(
|
||||
path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
|
||||
) -> None:
|
||||
"""Write batch_outputs to a file or upload to a URL.
|
||||
path_or_url: The path or URL to write batch_outputs to.
|
||||
batch_outputs: The list of batch outputs to write.
|
||||
output_tmp_dir: The directory to store the output file before uploading it
|
||||
to the output URL.
|
||||
"""
|
||||
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
|
||||
if output_tmp_dir is None:
|
||||
logger.info("Writing outputs to memory buffer")
|
||||
output_buffer = StringIO()
|
||||
for o in batch_outputs:
|
||||
print(o.model_dump_json(), file=output_buffer)
|
||||
output_buffer.seek(0)
|
||||
logger.info("Uploading outputs to %s", path_or_url)
|
||||
await upload_data(
|
||||
path_or_url,
|
||||
output_buffer.read().strip().encode("utf-8"),
|
||||
from_file=False,
|
||||
)
|
||||
else:
|
||||
# Write responses to a temporary file and then upload it to the URL.
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
encoding="utf-8",
|
||||
dir=output_tmp_dir,
|
||||
prefix="tmp_batch_output_",
|
||||
suffix=".jsonl",
|
||||
) as f:
|
||||
logger.info("Writing outputs to temporary local file %s", f.name)
|
||||
await write_local_file(f.name, batch_outputs)
|
||||
logger.info("Uploading outputs to %s", path_or_url)
|
||||
await upload_data(path_or_url, f.name, from_file=True)
|
||||
else:
|
||||
logger.info("Writing outputs to local file %s", path_or_url)
|
||||
await write_local_file(path_or_url, batch_outputs)
|
||||
|
||||
|
||||
async def download_bytes_from_url(
|
||||
url: str,
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
) -> bytes:
|
||||
"""Download data from a URL or decode from a data URL.
|
||||
|
||||
Args:
|
||||
url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
|
||||
allowed_media_domains: If set, only HTTP/HTTPS URLs whose hostname
|
||||
is in this list are permitted. data: URLs are not subject to
|
||||
this restriction.
|
||||
|
||||
Returns:
|
||||
Data as bytes
|
||||
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
|
||||
# Handle data URLs (base64 encoded) - not subject to domain restrictions
|
||||
if parsed.scheme == "data":
|
||||
# Format: data:...;base64,<base64_data>
|
||||
if "," in url:
|
||||
header, data = url.split(",", 1)
|
||||
if "base64" in header:
|
||||
decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
|
||||
if decoded_size_bytes > _get_audio_download_limit_bytes():
|
||||
_raise_audio_filesize_limit(decoded_size_bytes)
|
||||
return base64.b64decode(data)
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
f"Unsupported data URL encoding: {header}",
|
||||
parameter="url",
|
||||
)
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
f"Invalid data URL format: {url}",
|
||||
parameter="url",
|
||||
)
|
||||
|
||||
# Handle HTTP/HTTPS URLs
|
||||
elif parsed.scheme in ("http", "https"):
|
||||
if allowed_media_domains is not None:
|
||||
url_spec = parse_url(url)
|
||||
if url_spec.hostname not in allowed_media_domains:
|
||||
raise VLLMValidationError(
|
||||
f"The URL must be from one of the allowed domains: "
|
||||
f"{allowed_media_domains}. Input URL domain: "
|
||||
f"{url_spec.hostname}",
|
||||
parameter="url",
|
||||
value=url_spec.hostname,
|
||||
)
|
||||
# Use the normalized URL to prevent parsing discrepancies
|
||||
# between urllib3 and aiohttp (e.g. backslash-@ attacks).
|
||||
url = url_spec.url
|
||||
|
||||
try:
|
||||
return await global_http_connection.async_get_bytes(
|
||||
url,
|
||||
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
|
||||
max_bytes=_get_audio_download_limit_bytes(),
|
||||
)
|
||||
except HTTPResponseSizeExceededError as exc:
|
||||
_raise_audio_filesize_limit(exc.received_bytes)
|
||||
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
f"Unsupported URL scheme: {parsed.scheme}. "
|
||||
"Supported schemes: http, https, data",
|
||||
parameter="url",
|
||||
value=parsed.scheme,
|
||||
)
|
||||
|
||||
|
||||
def make_error_request_output(
|
||||
request: BatchRequestInput, error_msg: str
|
||||
) -> BatchRequestOutput:
|
||||
batch_output = BatchRequestOutput(
|
||||
id=f"vllm-{random_uuid()}",
|
||||
custom_id=request.custom_id,
|
||||
response=BatchResponseData(
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
request_id=f"vllm-batch-{random_uuid()}",
|
||||
),
|
||||
error=error_msg,
|
||||
)
|
||||
return batch_output
|
||||
|
||||
|
||||
async def make_async_error_request_output(
|
||||
request: BatchRequestInput, error_msg: str
|
||||
) -> BatchRequestOutput:
|
||||
return make_error_request_output(request, error_msg)
|
||||
|
||||
|
||||
async def run_request(
|
||||
serving_engine_func: Callable,
|
||||
request: BatchRequestInput,
|
||||
tracker: BatchProgressTracker,
|
||||
) -> BatchRequestOutput:
|
||||
try:
|
||||
response = await serving_engine_func(request.body)
|
||||
except Exception as e:
|
||||
response = create_error_response(e)
|
||||
|
||||
if isinstance(response, JSONResponse):
|
||||
with contextlib.suppress(pydantic.ValidationError):
|
||||
response = TypeAdapter(AllResponse | ErrorResponse).validate_python(
|
||||
json.loads(response.body)
|
||||
)
|
||||
|
||||
if isinstance(response, AllResponse):
|
||||
batch_output = BatchRequestOutput(
|
||||
id=f"vllm-{random_uuid()}",
|
||||
custom_id=request.custom_id,
|
||||
response=BatchResponseData(
|
||||
body=response, request_id=f"vllm-batch-{random_uuid()}"
|
||||
),
|
||||
error=None,
|
||||
)
|
||||
elif isinstance(response, ErrorResponse):
|
||||
batch_output = BatchRequestOutput(
|
||||
id=f"vllm-{random_uuid()}",
|
||||
custom_id=request.custom_id,
|
||||
response=BatchResponseData(
|
||||
status_code=response.error.code,
|
||||
request_id=f"vllm-batch-{random_uuid()}",
|
||||
),
|
||||
error=response,
|
||||
)
|
||||
else:
|
||||
batch_output = make_error_request_output(
|
||||
request, error_msg="Request must not be sent in stream mode"
|
||||
)
|
||||
|
||||
tracker.completed()
|
||||
return batch_output
|
||||
|
||||
|
||||
WrapperFn: TypeAlias = Callable[[Callable], Callable]
|
||||
|
||||
|
||||
def handle_endpoint_request(
|
||||
request: BatchRequestInput,
|
||||
tracker: BatchProgressTracker,
|
||||
url_matcher: Callable[[str], bool],
|
||||
handler_getter: Callable[[], Callable | None],
|
||||
wrapper_fn: WrapperFn | None = None,
|
||||
) -> Awaitable[BatchRequestOutput] | None:
|
||||
"""Generic handler for endpoint requests.
|
||||
|
||||
Args:
|
||||
request: The batch request input
|
||||
tracker: Progress tracker for the batch
|
||||
url_matcher: Function that takes a URL and returns True if it matches
|
||||
handler_getter: Function that returns the handler function or None
|
||||
wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
|
||||
|
||||
Returns:
|
||||
Awaitable[BatchRequestOutput] if the request was handled,
|
||||
None if URL didn't match
|
||||
|
||||
"""
|
||||
if not url_matcher(request.url):
|
||||
return None
|
||||
|
||||
handler_fn = handler_getter()
|
||||
if handler_fn is None:
|
||||
error_msg = f"Model does not support endpoint: {request.url}"
|
||||
return make_async_error_request_output(request, error_msg=error_msg)
|
||||
|
||||
# Apply wrapper if provided (e.g., for transcriptions/translations)
|
||||
if wrapper_fn is not None:
|
||||
handler_fn = wrapper_fn(handler_fn)
|
||||
|
||||
tracker.submitted()
|
||||
return run_request(handler_fn, request, tracker)
|
||||
|
||||
|
||||
|
||||
async def build_endpoint_registry(
|
||||
engine_client: EngineClient,
|
||||
args: Namespace,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Build the endpoint registry with all serving objects and handler configurations.
|
||||
|
||||
Args:
|
||||
engine_client: The engine client
|
||||
args: Command line arguments
|
||||
|
||||
Returns:
|
||||
Dictionary mapping endpoint keys to their configurations
|
||||
|
||||
"""
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
|
||||
# Create a state object to hold serving objects
|
||||
state = State()
|
||||
|
||||
# Initialize all serving objects using init_app_state
|
||||
# This provides full functionality including chat template processing,
|
||||
# LoRA support, tool servers, etc.
|
||||
await init_app_state(engine_client, state, args, supported_tasks)
|
||||
|
||||
# Get serving objects from state (defaulting to None if not set)
|
||||
openai_serving_chat = getattr(state, "openai_serving_chat", None)
|
||||
|
||||
# Registry of endpoint configurations
|
||||
endpoint_registry: dict[str, dict[str, Any]] = {
|
||||
"completions": {
|
||||
"url_matcher": lambda url: url == "/v1/chat/completions",
|
||||
"handler_getter": lambda: (
|
||||
openai_serving_chat.create_chat_completion
|
||||
if openai_serving_chat is not None
|
||||
else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
}
|
||||
|
||||
return endpoint_registry
|
||||
|
||||
|
||||
def validate_run_batch_args(args):
|
||||
valid_reasoning_parsers = ReasoningParserManager.list_registered()
|
||||
if (
|
||||
reasoning_parser := args.structured_outputs_config.reasoning_parser
|
||||
) and reasoning_parser not in valid_reasoning_parsers:
|
||||
raise KeyError(
|
||||
f"invalid reasoning parser: {reasoning_parser} "
|
||||
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
|
||||
)
|
||||
|
||||
|
||||
async def run_batch(
|
||||
engine_client: EngineClient,
|
||||
args: Namespace,
|
||||
) -> None:
|
||||
endpoint_registry = await build_endpoint_registry(
|
||||
engine_client=engine_client,
|
||||
args=args,
|
||||
)
|
||||
|
||||
tracker = BatchProgressTracker()
|
||||
logger.info("Reading batch from %s...", args.input_file)
|
||||
|
||||
# Submit all requests in the file to the engine "concurrently".
|
||||
response_futures: list[Awaitable[BatchRequestOutput]] = []
|
||||
for request_json in (await read_file(args.input_file)).strip().split("\n"):
|
||||
# Skip empty lines.
|
||||
request_json = request_json.strip()
|
||||
if not request_json:
|
||||
continue
|
||||
|
||||
request = BatchRequestInput.model_validate_json(request_json)
|
||||
|
||||
# Use the last segment of the URL as the endpoint key.
|
||||
# More advanced URL matching is done in url_matcher of endpoint_registry.
|
||||
endpoint_key = request.url.split("/")[-1]
|
||||
|
||||
result = None
|
||||
if endpoint_key in endpoint_registry:
|
||||
endpoint_config = endpoint_registry[endpoint_key]
|
||||
result = handle_endpoint_request(
|
||||
request,
|
||||
tracker,
|
||||
url_matcher=endpoint_config["url_matcher"],
|
||||
handler_getter=endpoint_config["handler_getter"],
|
||||
wrapper_fn=endpoint_config["wrapper_fn"],
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
response_futures.append(result)
|
||||
else:
|
||||
response_futures.append(
|
||||
make_async_error_request_output(
|
||||
request,
|
||||
error_msg=f"URL {request.url} was used. "
|
||||
"Supported endpoints: /v1/chat/completions.",
|
||||
)
|
||||
)
|
||||
|
||||
with tracker.pbar():
|
||||
responses = await asyncio.gather(*response_futures)
|
||||
|
||||
await write_file(args.output_file, responses, args.output_tmp_dir)
|
||||
|
||||
|
||||
async def main(args: Namespace):
|
||||
from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
|
||||
validate_run_batch_args(args)
|
||||
|
||||
async with build_async_engine_client(
|
||||
args,
|
||||
usage_context=UsageContext.OPENAI_BATCH_RUNNER,
|
||||
) as engine_client:
|
||||
await run_batch(engine_client, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
|
||||
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
|
||||
# to publish metrics at the /metrics endpoint.
|
||||
if args.enable_metrics:
|
||||
logger.info("Prometheus metrics enabled")
|
||||
start_http_server(port=args.port, addr=args.host)
|
||||
else:
|
||||
logger.info("Prometheus metrics disabled")
|
||||
|
||||
asyncio.run(main(args))
|
||||
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
|
||||
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
|
||||
from vllm.entrypoints.rl.offline import RLOfflineMixin
|
||||
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.logger import configure_logging_if_needed, init_logger
|
||||
@@ -59,7 +58,7 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
|
||||
class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
|
||||
"""An LLM for generating texts from given prompts and sampling parameters.
|
||||
|
||||
This class includes a tokenizer, a language model (possibly distributed
|
||||
|
||||
@@ -18,8 +18,6 @@ from vllm.entrypoints.launchers.launcher import (
|
||||
setup_server,
|
||||
validate_api_server_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
|
||||
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
|
||||
|
||||
warnings.warn(
|
||||
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
|
||||
@@ -35,13 +33,11 @@ __all__ = [
|
||||
"build_async_engine_client_from_engine_args",
|
||||
"build_app",
|
||||
"init_app_state",
|
||||
"init_render_app_state",
|
||||
"create_server_socket",
|
||||
"create_server_unix_socket",
|
||||
"validate_api_server_args",
|
||||
"setup_server",
|
||||
"build_and_serve",
|
||||
"build_and_serve_renderer",
|
||||
"run_server",
|
||||
"run_server_worker",
|
||||
"register_api_routers",
|
||||
|
||||
@@ -107,7 +107,6 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
|
||||
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
|
||||
# so that added fields like ``citations`` survive JSON serialization
|
||||
# instead of being stripped down to the base schema. Plain
|
||||
# ``ChatMessage`` instances serialize identically to before.
|
||||
@@ -162,7 +161,6 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
|
||||
# so streaming ``citations`` survive JSON serialization. Plain
|
||||
# ``DeltaMessage`` instances serialize identically to before.
|
||||
delta: SerializeAsAny[DeltaMessage]
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Translate Decisions questions and answers to and from label reads."""
|
||||
|
||||
import json
|
||||
|
||||
from .protocol import (
|
||||
ChoiceAnswer,
|
||||
ChoiceQuestion,
|
||||
DecisionAnswer,
|
||||
DecisionInputMessage,
|
||||
DecisionQuestion,
|
||||
PredicateAnswer,
|
||||
PredicateQuestion,
|
||||
ScoreAnswer,
|
||||
)
|
||||
from .question_types import (
|
||||
LABELS,
|
||||
Option,
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
argmax,
|
||||
)
|
||||
from .question_types import (
|
||||
ChoiceQuestion as LabelQuestion,
|
||||
)
|
||||
|
||||
|
||||
def make_read_question(
|
||||
index: int, question: DecisionQuestion, max_options: int = len(LABELS)
|
||||
) -> Question:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
options = [Option("false"), Option("true")]
|
||||
elif isinstance(question, ChoiceQuestion):
|
||||
options = [
|
||||
Option(json.dumps(c.value, ensure_ascii=False), c.description)
|
||||
for c in question.choices
|
||||
]
|
||||
else:
|
||||
options = [Option(level.label, level.description) for level in question.levels]
|
||||
limit = min(max_options, len(LABELS))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"This model supports at most {limit} choices per question"
|
||||
)
|
||||
return Question(
|
||||
id=str(index),
|
||||
type=LabelQuestion(),
|
||||
instructions=question.instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def make_answer(
|
||||
question: DecisionQuestion, probs: list[float], label_mass: float
|
||||
) -> DecisionAnswer:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
return PredicateAnswer(name=question.name, probability=probs[1])
|
||||
top = argmax(probs)
|
||||
confidence = probs[top] * label_mass
|
||||
if isinstance(question, ChoiceQuestion):
|
||||
return ChoiceAnswer(
|
||||
name=question.name,
|
||||
choice=question.choices[top].value,
|
||||
probabilities=[
|
||||
{"value": choice.value, "probability": probability}
|
||||
for choice, probability in zip(question.choices, probs)
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
return ScoreAnswer(
|
||||
name=question.name,
|
||||
score=sum(i * probability for i, probability in enumerate(probs)),
|
||||
probabilities=[
|
||||
{"value": i, "label": level.label, "probability": probability}
|
||||
for i, (level, probability) in enumerate(zip(question.levels, probs))
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
|
||||
def input_text(input: str | list[DecisionInputMessage]) -> str:
|
||||
if isinstance(input, str):
|
||||
return input
|
||||
return "\n\n".join(
|
||||
message.content
|
||||
if isinstance(message.content, str)
|
||||
else "\n".join(part.text for part in message.content)
|
||||
for message in input
|
||||
)
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import DecisionRequest, DecisionResponse
|
||||
from .serving import OpenAIServingDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/decisions",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=DecisionResponse,
|
||||
responses={
|
||||
status.value: {"model": ErrorResponse}
|
||||
for status in (
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
HTTPStatus.NOT_FOUND,
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
HTTPStatus.NOT_IMPLEMENTED,
|
||||
)
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_decision(request: DecisionRequest, raw_request: Request):
|
||||
handler: OpenAIServingDecisions | None = getattr(
|
||||
raw_request.app.state, "openai_serving_decisions", None
|
||||
)
|
||||
if handler is None:
|
||||
error = create_error_response(
|
||||
NotImplementedError("The model does not support the Decisions API")
|
||||
)
|
||||
return JSONResponse(content=error.model_dump(), status_code=error.error.code)
|
||||
result = await handler.create_decisions(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenAI Decisions API request and response models."""
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
|
||||
InputText = Annotated[StrictStr, Field(max_length=10485760)]
|
||||
ChoiceValue = StrictStr | StrictBool
|
||||
|
||||
|
||||
class DecisionModel(OpenAIBaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DecisionInputText(DecisionModel):
|
||||
type: Literal["input_text"]
|
||||
text: InputText
|
||||
|
||||
|
||||
class DecisionInputMessage(DecisionModel):
|
||||
role: Literal["user"]
|
||||
content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)]
|
||||
type: Literal["message"] = "message"
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def reject_images(cls, data: Any) -> Any:
|
||||
if (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("content"), list)
|
||||
and any(
|
||||
isinstance(part, dict) and part.get("type") == "input_image"
|
||||
for part in data["content"]
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Decisions supports text input only; images are not supported"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class QuestionBase(DecisionModel):
|
||||
instructions: ShortText
|
||||
name: ShortText | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_name(self) -> "QuestionBase":
|
||||
if "name" in self.model_fields_set and self.name is None:
|
||||
raise ValueError("name must be a string when provided")
|
||||
return self
|
||||
|
||||
|
||||
class PredicateQuestion(QuestionBase):
|
||||
type: Literal["predicate"]
|
||||
|
||||
|
||||
class ChoiceOption(DecisionModel):
|
||||
value: ChoiceValue
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ChoiceQuestion(QuestionBase):
|
||||
type: Literal["choice"]
|
||||
choices: list[ChoiceOption] = Field(min_length=2, max_length=255)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_choices(self) -> "ChoiceQuestion":
|
||||
values = [(type(c.value), c.value) for c in self.choices]
|
||||
if len(set(values)) != len(values):
|
||||
raise ValueError("choice values must be distinct")
|
||||
return self
|
||||
|
||||
|
||||
class ScoreLevel(DecisionModel):
|
||||
label: ShortText
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ScoreQuestion(QuestionBase):
|
||||
type: Literal["score"]
|
||||
levels: list[ScoreLevel] = Field(min_length=2, max_length=10)
|
||||
|
||||
|
||||
DecisionQuestion = Annotated[
|
||||
PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type")
|
||||
]
|
||||
|
||||
|
||||
class DecisionRequest(DecisionModel):
|
||||
model: ShortText
|
||||
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
|
||||
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
|
||||
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class PredicateAnswer(DecisionModel):
|
||||
type: Literal["predicate"] = "predicate"
|
||||
name: str | None
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceProbability(DecisionModel):
|
||||
value: ChoiceValue
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceAnswer(DecisionModel):
|
||||
type: Literal["choice"] = "choice"
|
||||
name: str | None
|
||||
choice: ChoiceValue
|
||||
probabilities: list[ChoiceProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
class ScoreProbability(DecisionModel):
|
||||
value: int
|
||||
label: str
|
||||
probability: float
|
||||
|
||||
|
||||
class ScoreAnswer(DecisionModel):
|
||||
type: Literal["score"] = "score"
|
||||
name: str | None
|
||||
score: float
|
||||
probabilities: list[ScoreProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
DecisionAnswer = Annotated[
|
||||
PredicateAnswer | ChoiceAnswer | ScoreAnswer,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class InputTokensDetails(DecisionModel):
|
||||
cached_tokens: int = 0
|
||||
cache_write_tokens: int = 0
|
||||
|
||||
|
||||
class OutputTokensDetails(DecisionModel):
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
|
||||
class OpenAIDecisionUsage(DecisionModel):
|
||||
input_tokens: int
|
||||
input_tokens_details: InputTokensDetails
|
||||
output_tokens: int
|
||||
output_tokens_details: OutputTokensDetails
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class DecisionResponse(DecisionModel):
|
||||
model: str
|
||||
answers: list[DecisionAnswer]
|
||||
usage: OpenAIDecisionUsage
|
||||
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Question types for structured decisions.
|
||||
|
||||
A question type turns a request's criteria into options, writes the question
|
||||
for the model with a label per option, and builds the answer from the label
|
||||
probabilities.
|
||||
"""
|
||||
|
||||
import math
|
||||
import string
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
|
||||
class StructuredDecisionError(ValueError):
|
||||
"""An invalid request. Returned as a 400."""
|
||||
|
||||
|
||||
#: A choice labels its options A to Z in order. A single capital letter is one
|
||||
#: token at the start of a reply for the tokenizers tested. Every read checks it.
|
||||
LABELS = tuple(string.ascii_uppercase)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Option:
|
||||
name: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Question:
|
||||
id: str
|
||||
type: "QuestionType"
|
||||
instructions: str
|
||||
options: tuple[Option, ...]
|
||||
labels: tuple[str, ...]
|
||||
|
||||
|
||||
class QuestionType(ABC):
|
||||
name: ClassVar[str]
|
||||
#: The labels of a question's options, in option order.
|
||||
label_set: ClassVar[tuple[str, ...]] = LABELS
|
||||
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
|
||||
|
||||
@abstractmethod
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
|
||||
|
||||
@abstractmethod
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
"""The answer for ``question``. ``probs[i]`` is the probability of
|
||||
``question.labels[i]`` among the labels, and the list sums to 1.
|
||||
``label_mass`` is the labels' total probability over the vocabulary."""
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str:
|
||||
return (
|
||||
f"{label}: {option.name} - {option.description}"
|
||||
if option.description
|
||||
else f"{label}: {option.name}"
|
||||
)
|
||||
|
||||
def prompt(self, question: Question) -> str:
|
||||
"""The question as the model reads it, after the state."""
|
||||
lines = [f"Question: {question.instructions}"] if question.instructions else []
|
||||
for label, o in zip(question.labels, question.options):
|
||||
lines.append(self.reply_line(label, o))
|
||||
lines.append(self.reply_instruction)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
QUESTION_TYPES: dict[str, QuestionType] = {}
|
||||
|
||||
|
||||
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
|
||||
if cls.name in QUESTION_TYPES:
|
||||
raise ValueError(f"question type {cls.name!r} is already registered")
|
||||
QUESTION_TYPES[cls.name] = cls()
|
||||
return cls
|
||||
|
||||
|
||||
def get_question_type(name: str) -> QuestionType:
|
||||
try:
|
||||
return QUESTION_TYPES[name]
|
||||
except KeyError:
|
||||
raise StructuredDecisionError(
|
||||
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
|
||||
) from None
|
||||
|
||||
|
||||
def build_question(
|
||||
qid: str,
|
||||
type_name: str,
|
||||
instructions: Any,
|
||||
criteria: Any,
|
||||
max_options: int,
|
||||
labeled: bool = True,
|
||||
) -> Question:
|
||||
if not qid:
|
||||
raise StructuredDecisionError("question ids must be non-empty")
|
||||
qtype = get_question_type(type_name)
|
||||
options = qtype.parse_options(qid, criteria)
|
||||
if not options:
|
||||
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
|
||||
names = [o.name for o in options]
|
||||
if len(set(names)) != len(names):
|
||||
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
|
||||
limit = min(max_options, len(qtype.label_set)) if labeled else max_options
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: at most {limit} options for this model"
|
||||
)
|
||||
if not isinstance(instructions, str):
|
||||
instructions = "" if instructions is None else str(instructions)
|
||||
return Question(
|
||||
id=qid,
|
||||
type=qtype,
|
||||
instructions=instructions,
|
||||
options=tuple(options),
|
||||
labels=qtype.label_set[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def argmax(values: list[float]) -> int:
|
||||
return max(range(len(values)), key=values.__getitem__)
|
||||
|
||||
|
||||
def label_softmax(logprobs: list[float]) -> list[float]:
|
||||
"""Softmax of the label logprobs."""
|
||||
top = max(logprobs)
|
||||
weights = [math.exp(lp - top) for lp in logprobs]
|
||||
total = sum(weights)
|
||||
return [w / total for w in weights]
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ChoiceQuestion(QuestionType):
|
||||
"""Pick one option. ``criteria`` maps each option name to a description,
|
||||
or to null."""
|
||||
|
||||
name = "choice"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, dict) or not criteria:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: choice criteria must map option names to "
|
||||
"a description or null"
|
||||
)
|
||||
return [
|
||||
Option(str(name), None if desc is None else str(desc))
|
||||
for name, desc in criteria.items()
|
||||
]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
"choice": question.options[top].name,
|
||||
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
|
||||
# The chosen label's probability over the whole vocabulary, so a
|
||||
# read that mostly wanted a non-label token reports low confidence.
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class NoulQuestion(QuestionType):
|
||||
"""Yes or no. ``criteria`` may describe what true and false mean."""
|
||||
|
||||
name = "noul"
|
||||
label_set = ("yes", "no")
|
||||
reply_instruction = "Answer with yes or no only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if criteria is not None and (
|
||||
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
|
||||
):
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: noul criteria must be an object with true and false"
|
||||
)
|
||||
criteria = criteria or {}
|
||||
true = criteria.get("true")
|
||||
false = criteria.get("false")
|
||||
return [
|
||||
Option("yes", None if true is None else str(true)),
|
||||
Option("no", None if false is None else str(false)),
|
||||
]
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str:
|
||||
# A side without a description still shows its label. Without the
|
||||
# label in the prompt, models answer "Yes" or "No" instead.
|
||||
return f"{label}: {option.description}" if option.description else label
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.name,
|
||||
"noul": probs[0],
|
||||
"probabilities": {"yes": probs[0], "no": probs[1]},
|
||||
# The chosen label's probability over the whole vocabulary, like a
|
||||
# choice's confidence.
|
||||
"confidence": max(probs) * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ScoreQuestion(QuestionType):
|
||||
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
|
||||
level names, from the first to the last level of the scale."""
|
||||
|
||||
name = "score"
|
||||
# 0-indexed like the answer, so a level's label is its score.
|
||||
label_set = tuple(string.digits)
|
||||
reply_instruction = "Answer with the number of one level only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, list) or len(criteria) < 2:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: score criteria must be an ordered list of levels"
|
||||
)
|
||||
return [Option(str(level)) for level in criteria]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
# The expected level, 0-indexed, so a level's index in the request's
|
||||
# criteria is its value.
|
||||
"score": sum(i * p for i, p in enumerate(probs)),
|
||||
"legend": {str(i): o.name for i, o in enumerate(question.options)},
|
||||
"probabilities": {str(i): p for i, p in enumerate(probs)},
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
from .adapters import input_text, make_answer, make_read_question
|
||||
from .protocol import (
|
||||
DecisionRequest,
|
||||
DecisionResponse,
|
||||
InputTokensDetails,
|
||||
OpenAIDecisionUsage,
|
||||
OutputTokensDetails,
|
||||
)
|
||||
from .question_types import StructuredDecisionError
|
||||
from .strategies import ReadStrategy
|
||||
|
||||
|
||||
class BaseServingDecisions(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels,
|
||||
strategy: ReadStrategy,
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=strategy.context.engine_client.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
|
||||
class OpenAIServingDecisions(BaseServingDecisions):
|
||||
async def create_decisions(
|
||||
self, request: DecisionRequest, raw_request: Request | None = None
|
||||
) -> DecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine = self.strategy.context.engine_client
|
||||
if engine.errored:
|
||||
raise engine.dead_error
|
||||
request_id = f"decision-{self._base_request_id(raw_request)}"
|
||||
try:
|
||||
if len(request.questions) > self.limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {self.limits.max_questions} for this model"
|
||||
)
|
||||
questions = [
|
||||
make_read_question(i, question, self.limits.max_options)
|
||||
for i, question in enumerate(request.questions)
|
||||
]
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine.check_admission(len(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
text = input_text(request.input)
|
||||
self._log_inputs(request_id, text, None, lora_request)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
None,
|
||||
text,
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=None,
|
||||
lora_request=lora_request,
|
||||
priority=0,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
except StructuredDecisionError as exc:
|
||||
return self.create_error_response(exc)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
input_tokens = sum(read.input_tokens for read in reads)
|
||||
output_tokens = sum(read.output_tokens for read in reads)
|
||||
return DecisionResponse(
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=[
|
||||
make_answer(question, read.probs, read.label_mass)
|
||||
for question, read in zip(request.questions, reads)
|
||||
],
|
||||
usage=OpenAIDecisionUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=input_tokens + output_tokens,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cached_tokens=sum(read.cached_tokens for read in reads),
|
||||
cache_write_tokens=sum(read.cache_write_tokens for read in reads),
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(),
|
||||
),
|
||||
)
|
||||
@@ -1,46 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .serving import OpenAIServingDecisions
|
||||
from .strategies import ReadContext, ReadStrategy, select_read_strategy
|
||||
|
||||
|
||||
def init_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
) -> ReadStrategy | None:
|
||||
state.openai_serving_decisions = None
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if strategy.task == "generate":
|
||||
state.openai_serving_decisions = OpenAIServingDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
return strategy
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import warnings
|
||||
|
||||
from prometheus_client import start_http_server
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
warnings.warn(
|
||||
"The `python -m vllm.entrypoints.openai.run_batch` command is deprecated "
|
||||
"and may be removed in a future release. Please use `vllm run_batch` instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from vllm.entrypoints.launchers.run_batch import main, parse_args
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
args = parse_args()
|
||||
|
||||
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
|
||||
# to publish metrics at the /metrics endpoint.
|
||||
if args.enable_metrics:
|
||||
logger.info("Prometheus metrics enabled")
|
||||
start_http_server(port=args.port, addr=args.host)
|
||||
else:
|
||||
logger.info("Prometheus metrics disabled")
|
||||
|
||||
asyncio.run(main(args))
|
||||
@@ -1,11 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
|
||||
def register_rl_api_routers(app: FastAPI):
|
||||
from .online.api_router import router as rl_router
|
||||
|
||||
app.include_router(rl_router)
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
from vllm.entrypoints.common.offline import OfflineInferenceMixin
|
||||
|
||||
|
||||
class RLOfflineMixin(OfflineInferenceMixin):
|
||||
def init_weight_transfer_engine(
|
||||
self, request: WeightTransferInitRequest | dict
|
||||
) -> None:
|
||||
"""Initialize weight transfer for RL training.
|
||||
|
||||
Args:
|
||||
request: Weight transfer initialization request with backend-specific info
|
||||
|
||||
"""
|
||||
init_info_dict = (
|
||||
request["init_info"] if isinstance(request, dict) else request.init_info
|
||||
)
|
||||
|
||||
self.llm_engine.collective_rpc(
|
||||
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
|
||||
)
|
||||
|
||||
def shutdown_weight_transfer_engine(self) -> None:
|
||||
"""Release the trainer connection without shutting down inference."""
|
||||
self.llm_engine.collective_rpc("shutdown_weight_transfer_engine")
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Start a new weight update."""
|
||||
self.llm_engine.collective_rpc("start_weight_update")
|
||||
|
||||
def start_draft_weight_update(self) -> None:
|
||||
"""Start a new weight update targeting the speculative draft model."""
|
||||
self.llm_engine.collective_rpc("start_draft_weight_update")
|
||||
|
||||
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
|
||||
"""Update the weights of the model.
|
||||
|
||||
Args:
|
||||
request: Weight update request with backend-specific update info
|
||||
|
||||
"""
|
||||
update_info_dict = (
|
||||
request["update_info"] if isinstance(request, dict) else request.update_info
|
||||
)
|
||||
|
||||
self.llm_engine.collective_rpc(
|
||||
"update_weights", kwargs={"update_info": update_info_dict}
|
||||
)
|
||||
|
||||
def finish_weight_update(self, weight_version: str | None = None) -> None:
|
||||
"""Finish the weight update and set its version if provided."""
|
||||
self.llm_engine.collective_rpc("finish_weight_update")
|
||||
if weight_version is not None:
|
||||
self.llm_engine.set_weight_version(weight_version)
|
||||
|
||||
def update_weight_version(self, new_version: str) -> None:
|
||||
"""Set the weight version without updating weights."""
|
||||
self.llm_engine.set_weight_version(new_version)
|
||||
|
||||
def get_weight_version(self) -> str:
|
||||
"""Return the latest committed weight version."""
|
||||
return self.llm_engine.get_weight_version()
|
||||
@@ -1,286 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Body, HTTPException, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdatePayload,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.rl.online.metrics import weight_operation_metrics
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine import PauseMode
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/pause")
|
||||
async def pause_generation(
|
||||
raw_request: Request,
|
||||
mode: Annotated[PauseMode, Query()] = "abort",
|
||||
clear_cache: Annotated[bool, Query()] = True,
|
||||
) -> JSONResponse:
|
||||
"""Pause generation requests to allow weight updates.
|
||||
|
||||
Args:
|
||||
raw_request: The incoming FastAPI request, used to reach the engine
|
||||
client on the app state.
|
||||
mode: How to handle in-flight requests:
|
||||
- ``"abort"``: Abort all in-flight requests immediately (default).
|
||||
- ``"wait"``: Wait for in-flight requests to complete.
|
||||
- ``"keep"``: Freeze requests in queue; they resume on /resume.
|
||||
clear_cache: DEPRECATED. Whether to clear KV/prefix caches after
|
||||
draining. Ignored when mode="keep".
|
||||
|
||||
"""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
await engine.pause_generation(
|
||||
mode=mode,
|
||||
clear_cache=clear_cache,
|
||||
)
|
||||
return JSONResponse(
|
||||
content={"status": "paused"},
|
||||
status_code=HTTPStatus.OK.value,
|
||||
)
|
||||
|
||||
except ValueError as err:
|
||||
return JSONResponse(
|
||||
content={"error": str(err)},
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
)
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to pause generation")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to pause generation: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/resume")
|
||||
async def resume_generation(raw_request: Request) -> JSONResponse:
|
||||
"""Resume generation after a pause."""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
await engine.resume_generation()
|
||||
return JSONResponse(
|
||||
content={"status": "resumed"},
|
||||
status_code=HTTPStatus.OK.value,
|
||||
)
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to resume generation")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to resume generation: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/abort_requests")
|
||||
async def abort_requests(raw_request: Request) -> JSONResponse:
|
||||
"""Abort in-flight requests without pausing the scheduler.
|
||||
|
||||
Empty/missing ``request_ids`` aborts all in-flight requests.
|
||||
"""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
|
||||
if not isinstance(body, dict):
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Request body must be a JSON object"
|
||||
)
|
||||
|
||||
request_ids = body.get("request_ids")
|
||||
|
||||
try:
|
||||
if request_ids:
|
||||
# Body ids are external (user-supplied) request ids.
|
||||
await engine.abort(request_ids)
|
||||
else:
|
||||
# The dev RL server runs AsyncLLM; abort everything it is tracking.
|
||||
# request_states is keyed by internal ids; parent_requests holds
|
||||
# parallel-sampling parents. Abort both as internal ids.
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
assert isinstance(engine, AsyncLLM)
|
||||
op = engine.output_processor
|
||||
request_ids = [
|
||||
*op.request_states.keys(),
|
||||
*op.parent_requests.keys(),
|
||||
]
|
||||
await engine.abort(request_ids, internal=True)
|
||||
return JSONResponse(
|
||||
content={"status": "aborted", "aborted": len(request_ids)},
|
||||
status_code=HTTPStatus.OK.value,
|
||||
)
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to abort requests")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to abort requests: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/is_paused")
|
||||
async def is_paused(raw_request: Request) -> JSONResponse:
|
||||
"""Return the current pause status."""
|
||||
engine = engine_client(raw_request)
|
||||
|
||||
try:
|
||||
paused = await engine.is_paused()
|
||||
except Exception as err: # pragma: no cover - defensive
|
||||
logger.exception("Failed to fetch pause status")
|
||||
return JSONResponse(
|
||||
content={"error": f"Failed to fetch pause status: {err}"},
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
)
|
||||
|
||||
return JSONResponse(content={"is_paused": paused})
|
||||
|
||||
|
||||
@router.post("/init_weight_transfer_engine")
|
||||
async def init_weight_transfer_engine(
|
||||
raw_request: Request,
|
||||
init_info: Annotated[dict[str, Any], Body(embed=True)],
|
||||
):
|
||||
with weight_operation_metrics().record("init"):
|
||||
await engine_client(raw_request).init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info=init_info)
|
||||
)
|
||||
return JSONResponse(content={"message": "Weight transfer initialized"})
|
||||
|
||||
|
||||
@router.post("/shutdown_weight_transfer_engine")
|
||||
async def shutdown_weight_transfer_engine(raw_request: Request):
|
||||
with weight_operation_metrics().record("shutdown"):
|
||||
await engine_client(raw_request).shutdown_weight_transfer_engine()
|
||||
return JSONResponse(content={"message": "Weight transfer shut down"})
|
||||
|
||||
|
||||
@router.post("/start_weight_update")
|
||||
async def start_weight_update(raw_request: Request):
|
||||
with weight_operation_metrics().record("start"):
|
||||
await engine_client(raw_request).start_weight_update()
|
||||
return JSONResponse(content={"message": "Weight update started"})
|
||||
|
||||
|
||||
@router.post("/start_draft_weight_update")
|
||||
async def start_draft_weight_update(raw_request: Request):
|
||||
with weight_operation_metrics().record("start_draft"):
|
||||
await engine_client(raw_request).start_draft_weight_update()
|
||||
return JSONResponse(content={"message": "Draft weight update started"})
|
||||
|
||||
|
||||
@router.post("/update_weights")
|
||||
async def update_weights(
|
||||
raw_request: Request,
|
||||
update_info: Annotated[WeightTransferUpdatePayload, Body(embed=True)],
|
||||
):
|
||||
with weight_operation_metrics().record("update"):
|
||||
await engine_client(raw_request).update_weights(
|
||||
request=WeightTransferUpdateRequest(update_info=update_info)
|
||||
)
|
||||
return JSONResponse(content={"message": "Weights updated"})
|
||||
|
||||
|
||||
@router.post("/finish_weight_update")
|
||||
async def finish_weight_update(
|
||||
raw_request: Request,
|
||||
weight_version: Annotated[str | None, Body(embed=True)] = None,
|
||||
):
|
||||
with weight_operation_metrics().record("finish"):
|
||||
await engine_client(raw_request).finish_weight_update(weight_version)
|
||||
return JSONResponse(content={"message": "Weight update finished"})
|
||||
|
||||
|
||||
@router.post("/update_weight_version")
|
||||
async def update_weight_version(
|
||||
raw_request: Request,
|
||||
new_version: Annotated[str, Body(embed=True)],
|
||||
):
|
||||
await engine_client(raw_request).update_weight_version(new_version)
|
||||
return JSONResponse(content={"success": True, "new_version": new_version})
|
||||
|
||||
|
||||
@router.get("/weight_info")
|
||||
async def weight_info(raw_request: Request):
|
||||
weight_version = await engine_client(raw_request).get_weight_version()
|
||||
return JSONResponse(content={"weight_version": weight_version})
|
||||
|
||||
|
||||
@router.post("/weight_checker")
|
||||
async def weight_checker(
|
||||
raw_request: Request,
|
||||
action: Annotated[Literal["checksum", "reset", "compare"], Body(embed=True)],
|
||||
baseline: Annotated[dict[str, str] | None, Body(embed=True)] = None,
|
||||
) -> JSONResponse:
|
||||
"""Checksum, reset, or compare model weights against a baseline."""
|
||||
client = engine_client(raw_request)
|
||||
if action == "reset":
|
||||
await client.collective_rpc("reset_weights")
|
||||
return JSONResponse(content={"status": "reset"})
|
||||
if action == "compare" and baseline is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="action='compare' requires a 'baseline' object",
|
||||
)
|
||||
|
||||
checksums: dict[str, str] = {}
|
||||
for worker_checksums in await client.compute_weight_checksums():
|
||||
if duplicates := checksums.keys() & worker_checksums.keys():
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
detail=f"Duplicate weight checksum keys: {sorted(duplicates)}",
|
||||
)
|
||||
checksums.update(worker_checksums)
|
||||
if action == "checksum":
|
||||
return JSONResponse(content={"checksums": checksums})
|
||||
|
||||
assert baseline is not None
|
||||
mismatches = sorted(
|
||||
key
|
||||
for key in checksums.keys() | baseline.keys()
|
||||
if checksums.get(key) != baseline.get(key)
|
||||
)
|
||||
return JSONResponse(content={"match": not mismatches, "mismatches": mismatches})
|
||||
|
||||
|
||||
@router.get("/get_world_size")
|
||||
async def get_world_size(
|
||||
raw_request: Request,
|
||||
include_dp: bool = Query(True),
|
||||
):
|
||||
"""Get the world size from the parallel config.
|
||||
|
||||
Args:
|
||||
raw_request: The incoming FastAPI request, used to reach the engine
|
||||
client on the app state.
|
||||
include_dp: If True (default), returns the world size including
|
||||
data parallelism (TP * PP * DP). If False, returns the world
|
||||
size without data parallelism (TP * PP).
|
||||
|
||||
"""
|
||||
parallel_config = engine_client(raw_request).vllm_config.parallel_config
|
||||
if include_dp:
|
||||
world_size = parallel_config.world_size_across_dp
|
||||
else:
|
||||
world_size = parallel_config.world_size
|
||||
return JSONResponse(content={"world_size": world_size})
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Duration and in-flight metrics for RL weight-transfer operations."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from threading import Lock
|
||||
from time import perf_counter
|
||||
from typing import Literal
|
||||
|
||||
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
|
||||
|
||||
Operation = Literal["init", "shutdown", "start", "start_draft", "update", "finish"]
|
||||
|
||||
|
||||
class WeightOperationMetrics:
|
||||
def __init__(self, registry: CollectorRegistry = REGISTRY):
|
||||
self.duration = Histogram(
|
||||
"vllm:rl_weight_update_operation_duration_seconds",
|
||||
"Duration of one weight-transfer operation.",
|
||||
["operation"],
|
||||
buckets=(0.01, 0.1, 1, 10, 30, 60, 120, 300, 600),
|
||||
registry=registry,
|
||||
)
|
||||
self.in_flight = Gauge(
|
||||
"vllm:rl_weight_update_operations_in_flight",
|
||||
"Weight-transfer operations currently awaited.",
|
||||
["operation"],
|
||||
multiprocess_mode="livesum",
|
||||
registry=registry,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def record(self, operation: Operation) -> Iterator[None]:
|
||||
in_flight = self.in_flight.labels(operation)
|
||||
in_flight.inc()
|
||||
started = perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
in_flight.dec()
|
||||
self.duration.labels(operation).observe(perf_counter() - started)
|
||||
|
||||
|
||||
_metrics: WeightOperationMetrics | None = None
|
||||
_metrics_lock = Lock()
|
||||
|
||||
|
||||
def weight_operation_metrics() -> WeightOperationMetrics:
|
||||
# Lazy, so the collectors exist only after multiprocess Prometheus setup.
|
||||
global _metrics
|
||||
if _metrics is None:
|
||||
with _metrics_lock:
|
||||
if _metrics is None:
|
||||
_metrics = WeightOperationMetrics()
|
||||
return _metrics
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequestUnion,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderChatStreamResponse,
|
||||
DerenderCompletionRequestUnion,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderCompletionStreamResponse,
|
||||
)
|
||||
from .serving import ServingDerender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def derender(request: Request) -> ServingDerender | None:
|
||||
return getattr(request.app.state, "serving_derender", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_chat_completion(
|
||||
request: DerenderChatRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a ChatCompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path; FastAPI validates and
|
||||
routes on the ``stream`` discriminator.
|
||||
|
||||
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
|
||||
the complete token list). Returns a ``ChatCompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
|
||||
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
|
||||
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
|
||||
successive calls, one per SSE chunk from ``/inference/v1/generate``.
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Derender API"
|
||||
)
|
||||
|
||||
if isinstance(request, DerenderChatStreamRequest):
|
||||
stream_result = await handler.derender_chat_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_chat_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/derender",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def derender_completion(
|
||||
request: DerenderCompletionRequestUnion,
|
||||
raw_request: Request,
|
||||
):
|
||||
"""Derender a generate response into a CompletionResponse.
|
||||
|
||||
Accepts both non-streaming (``stream=false``, default) and streaming
|
||||
(``stream=true``) request bodies on the same path.
|
||||
|
||||
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
|
||||
``CompletionResponse``.
|
||||
|
||||
Streaming: body is ``DerenderCompletionStreamRequest`` (one
|
||||
``generate_chunk`` + optional ``stream_state``). Returns a
|
||||
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
|
||||
"""
|
||||
handler = derender(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Derender API")
|
||||
|
||||
if isinstance(request, DerenderCompletionStreamRequest):
|
||||
stream_result = await handler.derender_completion_stream_response(request)
|
||||
if isinstance(stream_result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=stream_result.model_dump(),
|
||||
status_code=stream_result.error.code,
|
||||
)
|
||||
chunk, stream_state = stream_result
|
||||
response = DerenderCompletionStreamResponse(
|
||||
chunk=chunk, stream_state=stream_state
|
||||
)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
result = await handler.derender_completion_response(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
@@ -1,434 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
CompletionStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
|
||||
from ..token_in_token_out.mm_features import extract_mm_features
|
||||
from ..token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateTokensResponse,
|
||||
MultiModalFeatures,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingDerender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
online_derenderer: "OnlineDerenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=models.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.online_derenderer = online_derenderer
|
||||
|
||||
def _validate_derender_bounds(
|
||||
self,
|
||||
generate_responses: list[GenerateTokensResponse],
|
||||
) -> ErrorResponse | None:
|
||||
"""Reject derender payloads that exceed resource bounds.
|
||||
|
||||
Runs before any tokenizer.decode() or parser invocation to prevent
|
||||
CPU/memory exhaustion from oversized caller-supplied token structures.
|
||||
"""
|
||||
max_n = envs.VLLM_MAX_N_SEQUENCES
|
||||
max_model_len = self.model_config.max_model_len
|
||||
# See ModelConfig.max_logprobs for semantics and default value.
|
||||
max_logprobs = self.model_config.max_logprobs
|
||||
|
||||
if len(generate_responses) > max_n:
|
||||
return self.create_error_response(
|
||||
f"generate_responses count ({len(generate_responses)}) "
|
||||
f"exceeds server maximum ({max_n}). "
|
||||
f"Set VLLM_MAX_N_SEQUENCES to increase this limit."
|
||||
)
|
||||
|
||||
for gen in generate_responses:
|
||||
if len(gen.choices) > max_n:
|
||||
return self.create_error_response(
|
||||
f"choices count ({len(gen.choices)}) in response "
|
||||
f"'{gen.request_id}' exceeds server maximum ({max_n})."
|
||||
)
|
||||
|
||||
for choice in gen.choices:
|
||||
if choice.token_ids and len(choice.token_ids) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"token_ids length ({len(choice.token_ids)}) in "
|
||||
f"choice {choice.index} exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
if choice.logprobs and choice.logprobs.content:
|
||||
if len(choice.logprobs.content) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"logprobs.content length "
|
||||
f"({len(choice.logprobs.content)}) in "
|
||||
f"choice {choice.index} exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
for entry in choice.logprobs.content:
|
||||
if (
|
||||
max_logprobs >= 0
|
||||
and entry.top_logprobs
|
||||
and len(entry.top_logprobs) > max_logprobs
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"top_logprobs count "
|
||||
f"({len(entry.top_logprobs)}) in "
|
||||
f"choice {choice.index} exceeds "
|
||||
f"max_logprobs ({max_logprobs})."
|
||||
)
|
||||
|
||||
if gen.prompt_logprobs and len(gen.prompt_logprobs) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_logprobs length ({len(gen.prompt_logprobs)}) "
|
||||
f"in response '{gen.request_id}' exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _validate_prompt_token_ids(
|
||||
self,
|
||||
prompt_token_ids: list[list[int] | None],
|
||||
) -> ErrorResponse | None:
|
||||
"""Reject caller-supplied prompt_token_ids longer than max_model_len."""
|
||||
max_model_len = self.model_config.max_model_len
|
||||
for ids in prompt_token_ids:
|
||||
if ids is not None and len(ids) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_token_ids length ({len(ids)}) exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
return None
|
||||
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: DerenderChatRequest,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||
|
||||
Non-streaming only: expects the complete GenerateResponse with all
|
||||
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
|
||||
|
||||
When ``request.chat_request`` is provided, the parser splits the
|
||||
output into (reasoning, content, tool_calls). Otherwise falls
|
||||
back to plain detokenization.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
bounds_error = self._validate_derender_bounds([request.generate_response])
|
||||
if bounds_error is not None:
|
||||
return bounds_error
|
||||
|
||||
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
if self.online_derenderer.parser is not None and request.chat_request is None:
|
||||
return self.create_error_response(
|
||||
"chat_request is required when a tool or reasoning parser is "
|
||||
"configured because plain detokenization would leak raw parser "
|
||||
"markup into content."
|
||||
)
|
||||
|
||||
try:
|
||||
choices = await self.online_derenderer.derender_chat(
|
||||
request.generate_response,
|
||||
request.chat_request,
|
||||
request.prompt_token_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
prompt_tokens = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||
)
|
||||
gen = request.generate_response
|
||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
logger.debug(
|
||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||
gen.request_id,
|
||||
model_name,
|
||||
len(choices),
|
||||
completion_tokens,
|
||||
)
|
||||
return ChatCompletionResponse(
|
||||
id=gen.request_id,
|
||||
model=model_name,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
metrics=gen.metrics,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
self,
|
||||
request: DerenderCompletionRequest,
|
||||
) -> CompletionResponse | ErrorResponse:
|
||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||
|
||||
Non-streaming only. Mirrors the multi-prompt completions case: one
|
||||
GenerateResponse per prompt, parallel to the list[GenerateRequest]
|
||||
from /v1/completions/render.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if not request.generate_responses:
|
||||
return self.create_error_response("generate_responses must not be empty")
|
||||
|
||||
bounds_error = self._validate_derender_bounds(request.generate_responses)
|
||||
if bounds_error is not None:
|
||||
return bounds_error
|
||||
|
||||
if request.prompt_token_ids is not None:
|
||||
prompt_ids_error = self._validate_prompt_token_ids(request.prompt_token_ids)
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
(
|
||||
choices,
|
||||
total_prompt_tokens,
|
||||
total_completion_tokens,
|
||||
) = await self.online_derenderer.derender_completion(
|
||||
request.generate_responses,
|
||||
request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
|
||||
first = request.generate_responses[0]
|
||||
kv_params = first.kv_transfer_params
|
||||
if any(
|
||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||
):
|
||||
logger.warning(
|
||||
"derender_completion: kv_transfer_params differ across responses; "
|
||||
"setting to None on the aggregated response"
|
||||
)
|
||||
kv_params = None
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
logger.debug(
|
||||
"derender_completion request_id=%s model=%s choices=%d"
|
||||
" completion_tokens=%d",
|
||||
first.request_id,
|
||||
model_name,
|
||||
len(choices),
|
||||
total_completion_tokens,
|
||||
)
|
||||
return CompletionResponse(
|
||||
id=first.request_id,
|
||||
model=model_name,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
# Metrics describe one prompt. Parallel samples are already
|
||||
# suppressed by /generate; multi-prompt responses cannot be merged.
|
||||
metrics=first.metrics if len(request.generate_responses) == 1 else None,
|
||||
)
|
||||
|
||||
async def derender_chat_stream_response(
|
||||
self,
|
||||
request: DerenderChatStreamRequest,
|
||||
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_chat_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk and returns the
|
||||
derendered chunk together with the updated client carried state.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if self.online_derenderer.parser is not None and request.chat_request is None:
|
||||
return self.create_error_response(
|
||||
"chat_request is required when a tool or reasoning parser is "
|
||||
"configured because plain detokenization would leak raw parser "
|
||||
"markup into content."
|
||||
)
|
||||
|
||||
if (
|
||||
self.online_derenderer.parser is not None
|
||||
and request.prompt_token_ids is None
|
||||
):
|
||||
return self.create_error_response(
|
||||
"prompt_token_ids is required when a tool or reasoning "
|
||||
"parser is configured. Without it parse_delta cannot tell "
|
||||
"whether the prompt left reasoning open (e.g. a chat "
|
||||
"template that pre-opens <think>) and would misclassify "
|
||||
"reasoning content as plain content."
|
||||
)
|
||||
|
||||
# Each streamed chunk contains at most one choice per SSE event.
|
||||
# A single DerenderStreamState is threaded
|
||||
# through every choice in the chunk, so >1 would corrupt detok/parser
|
||||
# state across choices. See the matching check in
|
||||
# OnlineDerenderer.derender_chat_stream which this fails ahead of to
|
||||
# avoid touching the tokenizer at all on a malformed chunk.
|
||||
num_choices = len(request.generate_chunk.choices)
|
||||
if num_choices > 1:
|
||||
return self.create_error_response(
|
||||
f"derender_chat_stream expects at most one choice per chunk "
|
||||
f"(got {num_choices})."
|
||||
)
|
||||
|
||||
# prompt_token_ids is folded in here too. It is caller supplied and
|
||||
# otherwise unbounded. A parser configured deployment rescans it
|
||||
# in full on every chunk (is_reasoning_end /
|
||||
# adjust_initial_state_from_prompt, see abstract_parser.parse_delta)
|
||||
# since parser state cannot be carried across calls.
|
||||
prompt_len = (
|
||||
len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0
|
||||
)
|
||||
prior_len = (
|
||||
len(request.stream_state.output_token_ids)
|
||||
if request.stream_state is not None
|
||||
else 0
|
||||
)
|
||||
delta_len = sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
)
|
||||
max_model_len = self.model_config.max_model_len
|
||||
if prompt_len + prior_len + delta_len > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_token_ids length ({prompt_len}) plus "
|
||||
f"output_token_ids length ({prior_len}) plus delta "
|
||||
f"({delta_len}) exceeds max_model_len ({max_model_len})."
|
||||
)
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
try:
|
||||
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
|
||||
model=model_name,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
chat_request=request.chat_request,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
except Exception as exc:
|
||||
# The two ValueErrors derender_chat_stream can raise directly
|
||||
# (missing chat_request, >1 choice per chunk) are already
|
||||
# pre-checked above, so anything reaching here comes from the
|
||||
# parser replaying entirely client controlled state. Hermes
|
||||
# swallows its own streaming exceptions but finalize_generation
|
||||
# and engine based parsers don't, so an unexpected parser
|
||||
# failure on malformed (but well typed) state must surface as
|
||||
# a 400 here rather than an unhandled 500.
|
||||
logger.debug(
|
||||
"derender stream replay failed: %s: %s", type(exc).__name__, exc
|
||||
)
|
||||
return self.create_error_response("invalid stream_state: derender failed")
|
||||
|
||||
logger.debug(
|
||||
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
model_name,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
async def derender_completion_stream_response(
|
||||
self,
|
||||
request: DerenderCompletionStreamRequest,
|
||||
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
|
||||
"""Streaming counterpart to ``derender_completion_response``.
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
|
||||
delta) and returns the derendered chunk and updated state.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
try:
|
||||
(
|
||||
chunk,
|
||||
updated_state,
|
||||
) = await self.online_derenderer.derender_completion_stream(
|
||||
model=model_name,
|
||||
generate_chunk=request.generate_chunk,
|
||||
state=request.stream_state,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc})"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
|
||||
request.generate_chunk.request_id,
|
||||
model_name,
|
||||
sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
),
|
||||
)
|
||||
return chunk, updated_state
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
engine_input: EngineInput,
|
||||
) -> MultiModalFeatures | None:
|
||||
return extract_mm_features(engine_input)
|
||||
@@ -25,35 +25,12 @@ else:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def init_render_state(
|
||||
state: "State",
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
from .derender.serving import ServingDerender
|
||||
from .render.serving import ServingRender
|
||||
|
||||
state.serving_render = ServingRender(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
tool_server=state.tool_server,
|
||||
)
|
||||
|
||||
state.serving_derender = ServingDerender(
|
||||
state.openai_serving_models,
|
||||
state.online_derenderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
|
||||
def init_scale_out_state(
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
engine_client: "EngineClient",
|
||||
request_logger: RequestLogger | None,
|
||||
):
|
||||
init_render_state(state, request_logger)
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
|
||||
state.serving_tokens = ServingTokens(
|
||||
@@ -72,14 +49,10 @@ def register_scale_out_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
from .derender.api_router import router as derender_render
|
||||
from .render.api_router import router as render_render
|
||||
|
||||
args = getattr(app.state, "args", None)
|
||||
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
|
||||
# serving these endpoints, so they always register them regardless of
|
||||
# `--enable-scale-out`.
|
||||
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
|
||||
# `vllm serve --tokens-only` is dedicated to serving these endpoints, so it
|
||||
# always registers them regardless of `--enable-scale-out`.
|
||||
dedicated_mode = getattr(args, "tokens_only", False)
|
||||
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
|
||||
if not enabled:
|
||||
message = (
|
||||
@@ -96,11 +69,7 @@ def register_scale_out_api_routers(
|
||||
|
||||
from .token_in_token_out.api_router import router as token_router
|
||||
|
||||
for route in (
|
||||
*render_render.routes,
|
||||
*derender_render.routes,
|
||||
*token_router.routes,
|
||||
):
|
||||
for route in token_router.routes:
|
||||
if isinstance(route, APIRoute):
|
||||
app.add_api_route(
|
||||
route.path,
|
||||
@@ -110,9 +79,6 @@ def register_scale_out_api_routers(
|
||||
)
|
||||
return
|
||||
|
||||
app.include_router(render_render)
|
||||
app.include_router(derender_render)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from .token_in_token_out.api_router import (
|
||||
attach_router as attach_disagg_router,
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..token_in_token_out.protocol import GenerateRequest
|
||||
from .serving import ServingRender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support Chat Completions Render API"
|
||||
)
|
||||
|
||||
result = await handler.render_chat_request(request, raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_messages(request: AnthropicMessagesRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Messages Render API")
|
||||
|
||||
result = await handler.render_messages_request(request, raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=list[GenerateRequest],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_completion(request: CompletionRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Completions Render API")
|
||||
|
||||
result = await handler.render_completion_request(request, raw_request)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=[item.model_dump() for item in result])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_responses(request: ResponsesRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Responses Render API")
|
||||
|
||||
result = await handler.render_responses_request(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
@@ -1,385 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.generate.base.protocol import MMPayload
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
extract_mm_features,
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
ReasoningParserKwargs,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.parse import MultiModalDataParser
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
)
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
|
||||
|
||||
def _hoist_transfer_params(
|
||||
params: SamplingParams,
|
||||
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
||||
"""Move KV/EC transfer params out of `extra_args`.
|
||||
|
||||
Render emits them only as top-level `GenerateRequest` fields. Generate
|
||||
copies them back into `extra_args` for the engine.
|
||||
"""
|
||||
if not params.extra_args:
|
||||
return None, None
|
||||
# to_sampling_params aliases request.vllm_xargs, so copy before popping.
|
||||
extra = dict(params.extra_args)
|
||||
kv = extra.pop("kv_transfer_params", None)
|
||||
ec = extra.pop("ec_transfer_params", None)
|
||||
params.extra_args = extra or None
|
||||
return kv, ec
|
||||
|
||||
|
||||
class ServingRender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
online_renderer: "OnlineRenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
tool_server: "ToolServer | None" = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=online_renderer.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.online_renderer = online_renderer
|
||||
self.tool_server = tool_server
|
||||
|
||||
self._merge_inline_system = (
|
||||
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||
)
|
||||
|
||||
self._placeholder_metadata_parser: MultiModalDataParser | None = None
|
||||
self._placeholder_metadata_parser_failed = False
|
||||
|
||||
self.default_sampling_params = (
|
||||
online_renderer.model_config.get_diff_sampling_param()
|
||||
)
|
||||
mc = online_renderer.model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
async def render_chat_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
"""Validate the model and preprocess a chat completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
GPU-less render server and delegated to by OpenAIServingChat.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
if request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
_, engine_inputs = result
|
||||
|
||||
if len(engine_inputs) != 1:
|
||||
return self.create_error_response(
|
||||
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
|
||||
)
|
||||
|
||||
engine_input = engine_inputs[0]
|
||||
|
||||
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_completion_tokens
|
||||
if request.max_completion_tokens is not None
|
||||
else request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
# Resolve here what OpenAIServingChat passes to the engine:
|
||||
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
|
||||
reasoning_parser_kwargs = self._reasoning_parser_kwargs(request)
|
||||
parser = None
|
||||
if reasoning_parser_kwargs is not None:
|
||||
assert self.online_renderer.parser is not None
|
||||
parser = self.online_renderer.parser(
|
||||
self.online_renderer.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
|
||||
|
||||
request_id = (
|
||||
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
)
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(
|
||||
engine_input, mm_payload=request.mm_payload
|
||||
),
|
||||
sampling_params=params,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
model=request.model,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
async def render_messages_request(
|
||||
self,
|
||||
request: AnthropicMessagesRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
"""Validate the model and preprocess an Anthropic Messages request.
|
||||
|
||||
Converts the request to the OpenAI chat format using the same
|
||||
conversion as the /v1/messages server path, then delegates to
|
||||
render_chat_request so the rendered tokens match the server exactly.
|
||||
"""
|
||||
chat_req = AnthropicServingMessages.to_chat_completion_request(
|
||||
request, merge_inline_system=self._merge_inline_system
|
||||
)
|
||||
chat_req.mm_payload = request.mm_payload
|
||||
return await self.render_chat_request(chat_req, raw_request)
|
||||
|
||||
async def render_completion_request(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> list[GenerateRequest] | ErrorResponse:
|
||||
"""Validate the model and preprocess a completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
GPU-less render server and delegated to by OpenAIServingCompletion.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_completion(
|
||||
request, skip_mm_cache=True
|
||||
)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
base_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
generate_requests: list[GenerateRequest] = []
|
||||
for i, engine_input in enumerate(result):
|
||||
prompt_components = extract_prompt_components(
|
||||
self.model_config, engine_input
|
||||
)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
request_id = base_id if len(result) == 1 else f"{base_id}-{i}"
|
||||
|
||||
generate_requests.append(
|
||||
GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(
|
||||
engine_input, mm_payload=request.mm_payload
|
||||
),
|
||||
sampling_params=params,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
)
|
||||
|
||||
return generate_requests
|
||||
|
||||
async def render_responses_request(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
if request.previous_response_id is not None:
|
||||
return self.create_error_response(
|
||||
message=(
|
||||
"previous_response_id is not supported by the stateless "
|
||||
"render endpoint."
|
||||
),
|
||||
err_type="invalid_request_error",
|
||||
param="previous_response_id",
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_responses(
|
||||
request,
|
||||
previous_messages=None,
|
||||
previous_response_outputs=None,
|
||||
tool_server=self.tool_server,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
engine_input = result.engine_input
|
||||
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_output_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request.request_id,
|
||||
token_ids=list(token_ids),
|
||||
features=self._extract_mm_features(
|
||||
engine_input, mm_payload=request.mm_payload
|
||||
),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
|
||||
stream=bool(request.stream),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
def _reasoning_parser_kwargs(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ReasoningParserKwargs | None:
|
||||
parser_cls = self.online_renderer.parser
|
||||
if parser_cls is None or parser_cls.reasoning_parser_cls is None:
|
||||
return None
|
||||
return ReasoningParserKwargs(
|
||||
chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs(
|
||||
request
|
||||
)
|
||||
)
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Fields the EC consumer still requires after embeddings transfer."""
|
||||
if self._placeholder_metadata_parser_failed:
|
||||
return set()
|
||||
if self._placeholder_metadata_parser is None:
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
self._placeholder_metadata_parser = (
|
||||
MULTIMODAL_REGISTRY.create_processor(
|
||||
self.model_config
|
||||
).info.data_parser
|
||||
)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not load placeholder metadata fields; "
|
||||
"mm_metadata will use keep_on_cpu fields only."
|
||||
)
|
||||
self._placeholder_metadata_parser_failed = True
|
||||
return set()
|
||||
return set(
|
||||
self._placeholder_metadata_parser.placeholder_metadata_fields(modality)
|
||||
)
|
||||
|
||||
def _extract_mm_features(
|
||||
self,
|
||||
engine_input: EngineInput,
|
||||
*,
|
||||
mm_payload: MMPayload = "tensors",
|
||||
) -> MultiModalFeatures | None:
|
||||
return extract_mm_features(
|
||||
engine_input,
|
||||
metadata_fields_for=self._placeholder_metadata_fields,
|
||||
mm_payload=mm_payload,
|
||||
)
|
||||
@@ -108,8 +108,8 @@ def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
if not getattr(app.state.args, "tokens_only", False):
|
||||
return
|
||||
# The RLHF dev router registers its own /abort_requests first. Registering
|
||||
# this one too would only add a shadowed route with a duplicate operation ID.
|
||||
# An endpoint plugin may already serve /abort_requests. Registering this
|
||||
# one too would only add a shadowed route with a duplicate operation ID.
|
||||
has_abort_route = any(
|
||||
getattr(route, "path", None) == "/abort_requests" for route in app.routes
|
||||
)
|
||||
|
||||
@@ -685,7 +685,6 @@ class DerenderStreamState(BaseModel):
|
||||
entire accumulated text on each invocation, `parse_delta()` itself is
|
||||
O(n) yielding a true worst case compute cost of O(n³) per generation.
|
||||
No caching is performed. Work is bounded by `max_model_len`.
|
||||
See `OnlineDerenderer._derender_chat_stream_parsed`.
|
||||
|
||||
The detokenization strategy carries the incremental decode offsets
|
||||
directly rather than re-sending the whole token history each chunk.
|
||||
@@ -823,7 +822,7 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
`<think>`). `prompt_tokens` is a usage count and cannot serve this
|
||||
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
|
||||
|
||||
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
|
||||
Rejected with a 400 when a tool or reasoning
|
||||
parser is configured and this is omitted. Without it, `parse_delta`
|
||||
cannot tell whether the prompt left reasoning open and would silently
|
||||
misclassify reasoning content as plain content. On all paths it also
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/scale_elastic_ep",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"model": dict},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def scale_elastic_ep(raw_request: Request):
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail="Invalid JSON format") from e
|
||||
|
||||
new_data_parallel_size = body.get("new_data_parallel_size")
|
||||
drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
|
||||
|
||||
if new_data_parallel_size is None:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="new_data_parallel_size is required"
|
||||
)
|
||||
|
||||
if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="new_data_parallel_size must be a positive integer",
|
||||
)
|
||||
|
||||
if not isinstance(drain_timeout, int) or drain_timeout <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="drain_timeout must be a positive integer"
|
||||
)
|
||||
|
||||
client = engine_client(raw_request)
|
||||
try:
|
||||
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
|
||||
return JSONResponse(
|
||||
{
|
||||
"message": f"Scaled to {new_data_parallel_size} data parallel engines",
|
||||
}
|
||||
)
|
||||
except TimeoutError as e:
|
||||
raise HTTPException(
|
||||
status_code=408,
|
||||
detail="Scale failed due to request drain timeout "
|
||||
f"after {drain_timeout} seconds",
|
||||
) from e
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
logger.error("Scale failed: %s", e)
|
||||
raise HTTPException(status_code=500, detail="Scale failed") from e
|
||||
|
||||
|
||||
@router.post("/is_scaling_elastic_ep")
|
||||
async def is_scaling_elastic_ep(raw_request: Request):
|
||||
# External operation status comes from the shared store. Middleware gating
|
||||
# remains process-local, so scaling requests must reach every old-rank API.
|
||||
try:
|
||||
status = await engine_client(raw_request).get_external_elastic_ep_status()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to query external Elastic EP status: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="External Elastic EP status is temporarily unavailable",
|
||||
) from e
|
||||
if status is None:
|
||||
# Non-external EEP modes retain the process-local middleware state.
|
||||
is_scaling = get_scaling_elastic_ep()
|
||||
status = {
|
||||
"phase": (
|
||||
ExternalElasticEPScalePhase.COMMITTING.value
|
||||
if is_scaling
|
||||
else ExternalElasticEPScalePhase.IDLE.value
|
||||
),
|
||||
"epoch": None,
|
||||
"error": None,
|
||||
"requested_data_parallel_size": None,
|
||||
}
|
||||
else:
|
||||
is_scaling = status["phase"] in (
|
||||
ExternalElasticEPScalePhase.PREPARING.value,
|
||||
ExternalElasticEPScalePhase.COMMITTING.value,
|
||||
)
|
||||
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -16,7 +16,3 @@ def register_instrumentator_api_routers(app: FastAPI):
|
||||
from .metrics import attach_router as metrics_attach_router
|
||||
|
||||
metrics_attach_router(app)
|
||||
|
||||
from .offline_docs import attach_router as offline_docs_attach_router
|
||||
|
||||
offline_docs_attach_router(app)
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline FastAPI documentation support for air-gapped environments."""
|
||||
|
||||
import pathlib
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.openapi.docs import (
|
||||
get_swagger_ui_html,
|
||||
get_swagger_ui_oauth2_redirect_html,
|
||||
)
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Attach offline docs router if enabled via args."""
|
||||
args = getattr(app.state, "args", None)
|
||||
if args is None or not getattr(args, "enable_offline_docs", False):
|
||||
return
|
||||
|
||||
static_dir = pathlib.Path(__file__).parent / "static"
|
||||
|
||||
if not static_dir.exists():
|
||||
logger.warning(
|
||||
"Static directory not found at %s. Offline docs will not be available.",
|
||||
static_dir,
|
||||
)
|
||||
return
|
||||
|
||||
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
|
||||
|
||||
@app.get("/docs", include_in_schema=False)
|
||||
async def custom_swagger_ui_html():
|
||||
return get_swagger_ui_html(
|
||||
openapi_url=app.openapi_url,
|
||||
title=app.title + " - Swagger UI",
|
||||
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
|
||||
swagger_js_url="/static/swagger-ui-bundle.js",
|
||||
swagger_css_url="/static/swagger-ui.css",
|
||||
)
|
||||
|
||||
@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
|
||||
async def swagger_ui_redirect():
|
||||
return get_swagger_ui_oauth2_redirect_html()
|
||||
|
||||
logger.info("Offline documentation enabled with vendored static assets")
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -8,7 +8,7 @@ from starlette.datastructures import Headers
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""SageMaker-specific integration for vLLM."""
|
||||
@@ -1,156 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
import pydantic
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
|
||||
# (requires typing_extensions >= 4.13)
|
||||
RequestType = Any
|
||||
GetHandlerFn = Callable[[Request], BaseServing | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
|
||||
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
|
||||
"""Snapshot handler levels for loggers that may be affected by third-party
|
||||
logging configuration side effects (e.g. model_hosting_container_standards
|
||||
calling configure_root_logger() at import time)."""
|
||||
loggers = [logging.getLogger(), logging.getLogger("vllm")]
|
||||
seen: set[int] = set()
|
||||
snapshot: list[tuple[logging.Handler, int]] = []
|
||||
for logger in loggers:
|
||||
for handler in logger.handlers:
|
||||
if id(handler) not in seen:
|
||||
seen.add(id(handler))
|
||||
snapshot.append((handler, handler.level))
|
||||
return snapshot
|
||||
|
||||
|
||||
def _restore_handler_levels(
|
||||
snapshot: list[tuple[logging.Handler, int]],
|
||||
) -> None:
|
||||
"""Restore handler levels from a snapshot."""
|
||||
for handler, level in snapshot:
|
||||
handler.setLevel(level)
|
||||
|
||||
|
||||
def attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Attach the SageMaker hosting endpoints to the API server.
|
||||
|
||||
Handler levels are snapshotted and restored because importing
|
||||
model_hosting_container_standards may reconfigure root logging.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
_attach_router(app, supported_tasks, model_config)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
|
||||
|
||||
def _attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Register the SageMaker hosting routes (/ping, /invocations) on the
|
||||
app."""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
|
||||
|
||||
INVOCATION_VALIDATORS = [
|
||||
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
|
||||
for request_type, (get_handler, endpoint) in INVOCATION_TYPES
|
||||
]
|
||||
|
||||
@router.post("/ping", response_class=Response)
|
||||
@router.get("/ping", response_class=Response)
|
||||
@sagemaker_standards.register_ping_handler
|
||||
async def ping(raw_request: Request) -> Response:
|
||||
"""Ping check. Endpoint required for SageMaker."""
|
||||
return await health(raw_request)
|
||||
|
||||
@router.post(
|
||||
"/invocations",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@sagemaker_standards.register_invocation_handler
|
||||
@sagemaker_standards.stateful_session_manager()
|
||||
@sagemaker_standards.inject_adapter_id(adapter_path="model")
|
||||
async def invocations(raw_request: Request):
|
||||
"""For SageMaker, routes requests based on the request type."""
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
|
||||
valid_endpoints = [
|
||||
(validator, endpoint)
|
||||
for validator, (get_handler, endpoint) in INVOCATION_VALIDATORS
|
||||
if get_handler(raw_request) is not None
|
||||
]
|
||||
|
||||
for request_validator, endpoint in valid_endpoints:
|
||||
try:
|
||||
request = request_validator.validate_python(body)
|
||||
except pydantic.ValidationError:
|
||||
continue
|
||||
|
||||
return await endpoint(request, raw_request)
|
||||
|
||||
type_names = [
|
||||
t.__name__ if isinstance(t := validator._type, type) else str(t)
|
||||
for validator, _ in valid_endpoints
|
||||
]
|
||||
msg = f"Cannot find suitable handler for request. Expected one of: {type_names}"
|
||||
res = base(raw_request).create_error_response(message=msg)
|
||||
return JSONResponse(content=res.model_dump(), status_code=res.error.code)
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
|
||||
"""Bootstrap the app with the SageMaker hosting standards.
|
||||
|
||||
Handler levels are restored right after the import because importing
|
||||
model_hosting_container_standards may reconfigure root logging, and
|
||||
bootstrap must run with the original levels.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
app = sagemaker_standards.bootstrap(app)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
return app
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ASGI middleware for WebSocket Prometheus metrics.
|
||||
|
||||
Modeled after prometheus-fastapi-instrumentator, this middleware
|
||||
transparently instruments WebSocket endpoints with standard metrics
|
||||
without requiring changes to handler code.
|
||||
|
||||
NOTE: This module intentionally has zero vllm imports so that it can
|
||||
be extracted into a standalone package (similar to
|
||||
prometheus-fastapi-instrumentator) in the future. Please keep it that way.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Awaitable
|
||||
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
# Standard WebSocket metric names (not vllm-specific, following
|
||||
# the same convention as prometheus-fastapi-instrumentator).
|
||||
_active_sessions = Gauge(
|
||||
name="vllm:websocket_connections_active",
|
||||
documentation="Number of currently active WebSocket connections.",
|
||||
multiprocess_mode="livesum",
|
||||
)
|
||||
|
||||
_total_sessions = Counter(
|
||||
name="vllm:websocket_connections_total",
|
||||
documentation="Total number of WebSocket connections.",
|
||||
)
|
||||
|
||||
_session_duration = Histogram(
|
||||
name="vllm:websocket_connection_duration_seconds",
|
||||
documentation="Duration of WebSocket connections in seconds.",
|
||||
buckets=[0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1800],
|
||||
)
|
||||
|
||||
|
||||
class WebSocketMetricsMiddleware:
|
||||
"""Pure ASGI middleware that instruments WebSocket connections.
|
||||
|
||||
Tracks active connections (gauge), total connections (counter),
|
||||
and connection duration (histogram) for all WebSocket endpoints.
|
||||
|
||||
Usage::
|
||||
|
||||
app.add_middleware(WebSocketMetricsMiddleware)
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
|
||||
if scope["type"] != "websocket":
|
||||
return self.app(scope, receive, send)
|
||||
|
||||
return self._handle_websocket(scope, receive, send)
|
||||
|
||||
async def _handle_websocket(
|
||||
self, scope: Scope, receive: Receive, send: Send
|
||||
) -> None:
|
||||
start_time: float | None = None
|
||||
|
||||
async def send_wrapper(message: Message) -> None:
|
||||
nonlocal start_time
|
||||
if message["type"] == "websocket.accept":
|
||||
start_time = time.monotonic()
|
||||
_active_sessions.inc()
|
||||
_total_sessions.inc()
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, send_wrapper)
|
||||
finally:
|
||||
if start_time is not None:
|
||||
_active_sessions.dec()
|
||||
_session_duration.observe(time.monotonic() - start_time)
|
||||
Reference in New Issue
Block a user