[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack

- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions,
  rl, render-launcher and batch entrypoints; keep scale-out token routes
- Drop the Swagger static bundle and the /docs, /redoc routes
- Delete device-keyed tuning configs for fused_moe, quant utils, mamba
  select-state and helion; no sm86 entry ever existed for any of them
- Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant
  quantizers and their config, metrics and loader plumbing
- Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu
  requirements and the setup.py cpu target path
- Delete 29 environment variables nothing reads and 14 chat-template
  fallbacks for models this fork no longer registers
- Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark,
  dsv3 gemm) with their kernels, bindings, wrappers and CMake entries
- Delete the connector test suites that targeted removed connectors
- Cut the Mistral tokenizer and parser stack, its tests, and the
  mistral_common and ijson dependency pins
- Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier
  cut deleted while keeping the model tree

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 17:10:56 +07:00
co-authored by Hinata
parent bd70a61c20
commit 2066d3143f
1086 changed files with 122 additions and 291919 deletions
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-123
View File
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.entrypoints.anthropic.protocol import (
AnthropicCountTokensRequest,
AnthropicCountTokensResponse,
AnthropicError,
AnthropicErrorResponse,
AnthropicMessagesRequest,
AnthropicMessagesResponse,
)
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
router = APIRouter()
def messages(request: Request) -> AnthropicServingMessages:
return request.app.state.anthropic_serving_messages
def translate_error_response(response: ErrorResponse) -> JSONResponse:
anthropic_error = AnthropicErrorResponse(
error=AnthropicError(
type=response.error.type,
message=response.error.message,
)
)
return JSONResponse(
status_code=response.error.code, content=anthropic_error.model_dump()
)
@router.post(
"/v1/messages",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
handler = messages(raw_request)
if handler is None:
base_server = raw_request.app.state.serving_tokenization
error = base_server.create_error_response(
NotImplementedError("The model does not support Messages API")
)
return translate_error_response(error)
try:
generator = await handler.create_messages(request, raw_request)
except Exception as e:
logger.exception("Error in create_messages: %s", e)
return translate_error_response(create_error_response(e))
if isinstance(generator, ErrorResponse):
return translate_error_response(generator)
elif isinstance(generator, AnthropicMessagesResponse):
resp = generator.model_dump(exclude_none=True)
logger.debug("Anthropic Messages Response: %s", resp)
return JSONResponse(content=resp)
return StreamingResponse(content=generator, media_type="text/event-stream")
@router.post(
"/v1/messages/count_tokens",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"model": AnthropicCountTokensResponse},
HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def count_tokens(request: AnthropicCountTokensRequest, raw_request: Request):
handler = messages(raw_request)
if handler is None:
base_server = raw_request.app.state.serving_tokenization
error = base_server.create_error_response(
NotImplementedError("The model does not support Messages API")
)
return translate_error_response(error)
try:
response = await handler.count_tokens(request, raw_request)
except Exception as e:
logger.exception("Error in count_tokens: %s", e)
return translate_error_response(create_error_response(e))
if isinstance(response, ErrorResponse):
return translate_error_response(response)
return JSONResponse(content=response.model_dump(exclude_none=True))
def attach_router(app: FastAPI):
app.include_router(router)
-451
View File
@@ -1,451 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pydantic models for Anthropic API protocol."""
import time
from typing import Annotated, Any, Literal
from pydantic import BaseModel, Field, field_validator, model_validator
import vllm.envs as envs
from vllm.entrypoints.generate.base.protocol import (
MMPayload,
resolve_return_mm_kwargs_alias,
)
class AnthropicError(BaseModel):
"""Error structure for Anthropic API."""
type: str
message: str
class AnthropicErrorResponse(BaseModel):
"""Error response structure for Anthropic API."""
type: Literal["error"] = "error"
error: AnthropicError
class AnthropicUsage(BaseModel):
"""Token usage information."""
input_tokens: int
output_tokens: int
cache_creation_input_tokens: int | None = None
cache_read_input_tokens: int | None = None
class AnthropicContentBlock(BaseModel):
"""Content block in message."""
type: Literal[
"text",
"image",
"tool_use",
"tool_result",
"tool_reference",
"thinking",
"redacted_thinking",
"tool_addition",
"tool_removal",
]
text: str | None = None
# For image content
source: dict[str, Any] | None = None
# For tool use/result
id: str | None = None
tool_use_id: str | None = None
name: str | None = None
input: dict[str, Any] | None = None
content: str | list[dict[str, Any]] | None = None
is_error: bool | None = None
# For tool_reference content
tool_name: str | None = None
# For thinking content
thinking: str | None = None
signature: str | None = None
# For redacted thinking content (safety-filtered by the API)
data: str | None = None
# For tool_addition/tool_removal content
tool: "AnthropicToolChange | None" = None
@model_validator(mode="after")
def validate_tool_change(self) -> "AnthropicContentBlock":
if self.type not in ("tool_addition", "tool_removal"):
return self
if self.tool is None:
raise ValueError(f"tool is required for {self.type} blocks")
if self.type == "tool_removal" and self.tool.type != "tool_reference":
raise ValueError("tool_removal only accepts a tool_reference")
return self
def _reject_tool_changes(content: str | list[AnthropicContentBlock] | None) -> None:
if content is None or isinstance(content, str):
return
for block in content:
if block.type in ("tool_addition", "tool_removal"):
raise ValueError(
f'{block.type} blocks are only allowed in messages with role "system"'
)
class AnthropicMessage(BaseModel):
"""Message structure."""
role: Literal["user", "assistant", "system"]
content: str | list[AnthropicContentBlock]
@model_validator(mode="after")
def validate_tool_change_role(self) -> "AnthropicMessage":
if self.role != "system":
_reject_tool_changes(self.content)
return self
class AnthropicTool(BaseModel):
"""Tool definition."""
name: str
description: str | None = None
input_schema: dict[str, Any]
strict: bool | None = None
defer_loading: bool | None = None
@field_validator("input_schema")
@classmethod
def validate_input_schema(cls, v):
if not isinstance(v, dict):
raise ValueError("input_schema must be a dictionary")
if "type" not in v:
v["type"] = "object" # Default to object type
return v
class AnthropicToolChangeReference(BaseModel):
"""A tool named by a tool_addition or tool_removal block."""
type: Literal["tool_reference"]
name: str
class AnthropicToolChangeDefinition(BaseModel):
"""A tool defined by value in a tool_addition block."""
type: Literal["tool_definition"]
definition: AnthropicTool
AnthropicToolChange = Annotated[
AnthropicToolChangeReference | AnthropicToolChangeDefinition,
Field(discriminator="type"),
]
class AnthropicToolChoice(BaseModel):
"""Tool Choice definition."""
type: Literal["auto", "any", "tool", "none"]
name: str | None = None
disable_parallel_tool_use: bool | None = None
@model_validator(mode="after")
def validate_name_required_for_tool(self) -> "AnthropicToolChoice":
if self.type == "tool" and not self.name:
raise ValueError("tool_choice.name is required when type is 'tool'")
return self
class AnthropicJsonOutputFormat(BaseModel):
"""JSON output format configuration."""
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
type: Literal["json_schema"] = "json_schema"
AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"]
# reasoning_effort sent to the chat template for thinking.type=disabled.
AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort]
AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort]
AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"]
class AnthropicOutputConfig(BaseModel):
"""Configuration options for the model's output, such as the output format."""
effort: AnthropicEffort | None = None
format: AnthropicJsonOutputFormat | None = None
class AnthropicThinkingConfigEnabled(BaseModel):
"""Extended thinking with a fixed token budget.
``display`` is accepted but ignored: reasoning is always returned.
"""
type: Literal["enabled"]
budget_tokens: int = Field(ge=1024)
display: AnthropicThinkingDisplay | None = None
class AnthropicThinkingConfigDisabled(BaseModel):
type: Literal["disabled"]
class AnthropicThinkingConfigAdaptive(BaseModel):
"""Extended thinking whose depth the model chooses.
``display`` is accepted but ignored: reasoning is always returned.
"""
type: Literal["adaptive"]
display: AnthropicThinkingDisplay | None = None
AnthropicThinkingConfig = Annotated[
AnthropicThinkingConfigEnabled
| AnthropicThinkingConfigDisabled
| AnthropicThinkingConfigAdaptive,
Field(discriminator="type"),
]
class AnthropicMessagesRequest(BaseModel):
"""Anthropic Messages API request."""
model: str
messages: list[AnthropicMessage]
max_tokens: int
metadata: dict[str, Any] | None = None
output_config: AnthropicOutputConfig | None = None
thinking: AnthropicThinkingConfig | None = None
stop_sequences: (
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
) = None
stream: bool | None = False
system: str | list[AnthropicContentBlock] | None = None
temperature: float | None = None
tool_choice: AnthropicToolChoice | None = None
tools: list[AnthropicTool] | None = None
top_k: int | None = None
top_p: float | None = None
# vLLM-specific fields that are not in Anthropic spec
mm_payload: MMPayload = Field(
default="tensors",
description=(
"What the render response's `features` carry for multimodal "
"items. 'tensors' serializes the processed tensors into "
"`kwargs_data` and `mm_metadata`. 'none' sets both to null for "
"callers that need only the token layout and item hashes, such as "
"cache-aware routers. Do not send a 'none' response to "
"`/inference/v1/generate` which reads a null `kwargs_data` as "
"every item being cached. Only supported on the render endpoints. "
"Ignored on regular generation endpoints."
),
)
return_mm_kwargs: bool | None = Field(
default=None,
deprecated="return_mm_kwargs is deprecated in favor of the mm_payload field",
description=(
"Deprecated alias: false is the same as `mm_payload='none'`. "
"Ignored when `mm_payload` is set."
),
)
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
watermarking: bool | None = Field(
default=None,
description=(
"Whether to apply the engine's configured watermark to this request."
),
)
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None,
description=(
"ECTransfer parameters used for encoder-cache disaggregated serving."
),
)
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
default=None,
description=(
"Additional request parameters with (list of) string or "
"numeric values, used by custom extensions."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def validate_model(cls, v):
if not v:
raise ValueError("Model is required")
return v
@field_validator("max_tokens")
@classmethod
def validate_max_tokens(cls, v):
if v <= 0:
raise ValueError("max_tokens must be positive")
return v
@field_validator("system")
@classmethod
def validate_system(cls, v):
_reject_tool_changes(v)
return v
@model_validator(mode="before")
@classmethod
def apply_return_mm_kwargs_alias(cls, data: Any) -> Any:
return resolve_return_mm_kwargs_alias(data)
@model_validator(mode="after")
def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
# P/D prefill legs are sent with max_tokens=1 and never decode; the
# decode leg carries the client's max_tokens and is still checked.
if self.kv_transfer_params and self.kv_transfer_params.get("do_remote_decode"):
return self
if (
isinstance(self.thinking, AnthropicThinkingConfigEnabled)
and self.thinking.budget_tokens >= self.max_tokens
):
raise ValueError("thinking.budget_tokens must be less than max_tokens")
return self
class AnthropicDelta(BaseModel):
"""Delta for streaming responses."""
type: (
Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
| None
) = None
text: str | None = None
thinking: str | None = None
partial_json: str | None = None
signature: str | None = None
# Message delta
stop_reason: (
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
) = None
stop_sequence: str | None = None
class AnthropicStreamEvent(BaseModel):
"""Streaming event."""
type: Literal[
"message_start",
"message_delta",
"message_stop",
"content_block_start",
"content_block_delta",
"content_block_stop",
"ping",
"error",
]
message: "AnthropicMessagesResponse | None" = None
delta: AnthropicDelta | None = None
content_block: AnthropicContentBlock | None = None
index: int | None = None
error: AnthropicError | None = None
usage: AnthropicUsage | None = None
class AnthropicMessagesResponse(BaseModel):
"""Anthropic Messages API response."""
id: str
type: Literal["message"] = "message"
role: Literal["assistant"] = "assistant"
content: list[AnthropicContentBlock]
model: str
stop_reason: (
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
) = None
stop_sequence: str | None = None
usage: AnthropicUsage | None = None
# vLLM-specific fields that are not in Anthropic spec
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
ec_transfer_params: dict[str, Any] | None = Field(
default=None, description="ECTransfer parameters."
)
def model_post_init(self, __context):
if not self.id:
self.id = f"msg_{int(time.time() * 1000)}"
class AnthropicContextManagement(BaseModel):
"""Context management information for token counting."""
original_input_tokens: int
class AnthropicCountTokensRequest(BaseModel):
"""Anthropic messages.count_tokens request."""
model: str
messages: list[AnthropicMessage]
output_config: AnthropicOutputConfig | None = None
system: str | list[AnthropicContentBlock] | None = None
thinking: AnthropicThinkingConfig | None = None
tool_choice: AnthropicToolChoice | None = None
tools: list[AnthropicTool] | None = None
# vLLM-specific fields that are not in Anthropic spec
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def validate_model(cls, v):
if not v:
raise ValueError("Model is required")
return v
@field_validator("system")
@classmethod
def validate_system(cls, v):
_reject_tool_changes(v)
return v
class AnthropicCountTokensResponse(BaseModel):
"""Anthropic messages.count_tokens response."""
input_tokens: int
context_management: AnthropicContextManagement | None = None
File diff suppressed because it is too large Load Diff
-115
View File
@@ -1,115 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import inspect
import uvloop
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.launchers.cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
from vllm.entrypoints.launchers.render.entry import run_launch_fastapi
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.utils.argparse_utils import FlexibleArgumentParser
logger = init_logger(__name__)
DESCRIPTION = "Launch individual vLLM components."
class LaunchSubcommandBase(CLISubcommand):
"""The base class of subcommands for `vllm launch`."""
help: str
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
"""Add the CLI arguments to the parser.
By default, uses the subcommand's docstring as the description and adds
the standard vLLM serving arguments.
Subclasses can override to add component-specific arguments.
"""
if cls.__doc__:
parser.description = inspect.cleandoc(cls.__doc__)
make_arg_parser(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
raise NotImplementedError
class RenderSubcommand(LaunchSubcommandBase):
"""`vllm launch render` starts a GPU-less rendering server for preprocessing
and postprocessing only.
```bash
vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100
```
This command reuses the standard serving parser, so model, frontend,
networking, and related CLI options follow the same conventions as
[`vllm serve`](https://docs.vllm.ai/en/latest/cli/serve/).
"""
name = "render"
help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
@staticmethod
def cmd(args: argparse.Namespace) -> None:
uvloop.run(run_launch_fastapi(args))
class LaunchSubcommand(CLISubcommand):
"""The `launch` subcommand for the vLLM CLI.
Uses nested sub-subcommands so each component can define its own
arguments independently (e.g. ``vllm launch render``).
"""
name = "launch"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
if hasattr(args, "model_tag") and args.model_tag is not None:
args.model = args.model_tag
args.launch_command(args)
def validate(self, args: argparse.Namespace) -> None:
validate_parsed_serve_args(args)
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
launch_parser = subparsers.add_parser(
self.name,
help=DESCRIPTION,
description=DESCRIPTION,
usage=f"vllm {self.name} <component> [options]",
)
launch_subparsers = launch_parser.add_subparsers(
required=True, dest="launch_component"
)
for cmd_cls in LaunchSubcommandBase.__subclasses__():
cmd_subparser = launch_subparsers.add_parser(
cmd_cls.name,
help=cmd_cls.help,
usage=f"vllm {self.name} {cmd_cls.name} [options]",
)
cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
cmd_cls.add_cli_args(cmd_subparser)
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=f"{self.name} {cmd_cls.name}"
)
return launch_parser
def cmd_init() -> list[CLISubcommand]:
return [LaunchSubcommand()]
-4
View File
@@ -42,10 +42,8 @@ def main():
import vllm.entrypoints.cli.collect_env
import vllm.entrypoints.cli.download_kernels
import vllm.entrypoints.cli.launch
import vllm.entrypoints.cli.openai
import vllm.entrypoints.cli.preload
import vllm.entrypoints.cli.run_batch
import vllm.entrypoints.cli.serve
from vllm.entrypoints.serve.utils.api_utils import (
VLLM_SUBCMD_PARSER_EPILOG,
@@ -56,11 +54,9 @@ def main():
CMD_MODULES = [
vllm.entrypoints.cli.openai,
vllm.entrypoints.cli.serve,
vllm.entrypoints.cli.launch,
vllm.entrypoints.cli.collect_env,
vllm.entrypoints.cli.download_kernels,
vllm.entrypoints.cli.preload,
vllm.entrypoints.cli.run_batch,
]
cli_env_setup()
-68
View File
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import asyncio
import importlib.metadata
import typing
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
if typing.TYPE_CHECKING:
from vllm.utils.argparse_utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
logger = init_logger(__name__)
class RunBatchSubcommand(CLISubcommand):
"""The `run-batch` subcommand for vLLM CLI."""
name = "run-batch"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
from vllm.entrypoints.launchers.run_batch import main as run_batch_main
logger.info(
"vLLM batch processing API version %s", importlib.metadata.version("vllm")
)
logger.info("args: %s", args)
# Start the Prometheus metrics server.
# LLMEngine uses the Prometheus client
# to publish metrics at the /metrics endpoint.
if args.enable_metrics:
from prometheus_client import start_http_server
logger.info("Prometheus metrics enabled")
start_http_server(port=args.port, addr=args.url)
else:
logger.info("Prometheus metrics disabled")
asyncio.run(run_batch_main(args))
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
from vllm.entrypoints.launchers.run_batch import make_arg_parser
run_batch_parser = subparsers.add_parser(
self.name,
help="Run batch prompts and write results to file.",
description=(
"Run batch prompts using vLLM's OpenAI-compatible API.\n"
"Supports local or HTTP input/output files."
),
usage="vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
)
run_batch_parser = make_arg_parser(run_batch_parser)
run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
return run_batch_parser
def cmd_init() -> list[CLISubcommand]:
return [RunBatchSubcommand()]
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-309
View File
@@ -1,309 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FastAPI router for the Cohere Chat v2 API.
Exposes two routes:
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
generation, mirroring ``POST /v1/chat/completions/render``.
The Cohere v2 protocol models are sourced from the official ``cohere``
Python SDK (``pip install cohere``). To keep that an *optional*
dependency for vLLM, the SDK-dependent imports - and the route handlers
themselves - are gated on a one-shot probe at module load. If the SDK
isn't installed, :func:`attach_router` becomes a no-op (with an info
log) and vLLM continues to boot normally.
Even when the SDK is installed, :func:`attach_router` also requires
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
the chat route. The render route additionally requires scale-out
endpoints to be enabled, i.e. ``--enable-scale-out`` or
``--tokens-only``. These gates keep unrelated deployments from
accidentally exposing the APIs.
Note: the handlers must live at module scope (not inside
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
the ``CohereChatV2Request`` body annotation against the module's
globals. Defining them locally inside ``attach_router`` would hide the
type from ``get_type_hints``, causing FastAPI to silently degrade the
body parameter into a query parameter and reject every request with
422.
"""
import json
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.middleware.base import BaseHTTPMiddleware
import vllm.envs as envs
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
_COHERE_PATH_PREFIX = "/cohere/"
logger = init_logger(__name__)
try:
import cohere # noqa: F401 -- dependency probe
except ImportError:
_SDK_AVAILABLE = False
else:
_SDK_AVAILABLE = True
if _SDK_AVAILABLE:
from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
CohereChatV2Response,
CohereError,
)
from vllm.entrypoints.cohere.serving import CohereServingChatV2
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
router = APIRouter()
render_router = APIRouter()
def _serving(request: Request) -> CohereServingChatV2 | None:
return getattr(request.app.state, "cohere_serving_chat_v2", None)
def _serving_render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
def _request_id(raw_request: Request | None) -> str | None:
"""Best-effort lookup of the active request id.
Prefers the id the underlying chat handler stamped onto
``raw_request.state.request_metadata`` (if it got that far before
failing), falling back to the ``X-Request-Id`` HTTP header. May
return ``None`` if neither is available, in which case the field
is omitted from the response.
"""
if raw_request is None:
return None
meta = getattr(raw_request.state, "request_metadata", None)
if meta is not None and getattr(meta, "request_id", None):
return meta.request_id
return raw_request.headers.get("X-Request-Id")
def _error_response(
error: ErrorResponse,
raw_request: Request | None,
*,
fallback_status: int = HTTPStatus.BAD_REQUEST,
) -> JSONResponse:
"""Translate vLLM's internal error envelope into Cohere's shape."""
info = error.error
status = info.code or fallback_status
return JSONResponse(
status_code=status,
content=CohereError(
message=sanitize_message(info.message),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
@router.post(
"/cohere/v2/chat",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
@load_aware_call
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
handler = _serving(raw_request)
if handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message="The model does not support the Cohere v2 chat API.",
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
result = await handler.create_chat_v2(request, raw_request)
except Exception as e:
logger.exception("Error in /cohere/v2/chat: %s", e)
return _error_response(create_error_response(e), raw_request)
match result:
case ErrorResponse():
return _error_response(result, raw_request)
case CohereChatV2Response():
return JSONResponse(content=result.model_dump(exclude_none=True))
case _:
return StreamingResponse(content=result, media_type="text/event-stream")
@render_router.post(
"/cohere/v2/chat/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
"""Tokenize a Cohere v2 chat request without running generation.
The Cohere counterpart to ``POST /v1/chat/completions/render``.
The v2 body goes through the same conversion
``/cohere/v2/chat`` uses and is then handed to the shared
:class:`ServingRender`, so the returned ``GenerateRequest``
carries the prompt tokens and sampling params the chat endpoint
would have sent to the engine.
"""
handler = _serving(raw_request)
render_handler = _serving_render(raw_request)
if handler is None or render_handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message=(
"The model does not support the Cohere v2 chat render API."
),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
chat_request = handler.to_chat_completion_request(request)
result = await render_handler.render_chat_request(chat_request)
except Exception as e:
logger.exception("Error in /cohere/v2/chat/render: %s", e)
return _error_response(create_error_response(e), raw_request)
if isinstance(result, ErrorResponse):
return _error_response(result, raw_request)
if (kwargs := result.reasoning_parser_kwargs) is not None:
kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs(
kwargs.chat_template_kwargs
)
return JSONResponse(content=result.model_dump())
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
The endpoint handler above already returns :class:`CohereError` for
errors it owns, but globally-registered exception handlers (e.g.
:func:`validation_exception_handler` for pydantic body errors,
:func:`http_exception_handler`, engine error handlers) fire *before*
the handler runs and produce vLLM's internal
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
shape doesn't match the ``CohereError`` schema advertised on the
route's OpenAPI ``responses``, so clients (and schema-conformance
tests like ``test_openai_schema.py``) would see a mismatch on
those paths. This middleware normalises any error body on
``/cohere/*`` responses to :class:`CohereError`.
"""
async def dispatch(self, request: Request, call_next):
response = await call_next(request)
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
return response
if response.status_code < 400:
return response
content_type = response.headers.get("content-type", "")
if not content_type.startswith("application/json"):
return response
body = b"".join([chunk async for chunk in response.body_iterator])
translated = _translate_vllm_error_body(body, request)
if translated is not None:
return translated
passthrough_headers = {
k: v
for k, v in response.headers.items()
if k.lower() != "content-length"
}
return Response(
content=body,
status_code=response.status_code,
headers=passthrough_headers,
media_type=content_type,
)
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
Returns ``None`` if ``raw`` does not match the vLLM error envelope
(which signals the middleware to pass the body through unchanged).
"""
try:
data = json.loads(raw)
except (json.JSONDecodeError, TypeError, ValueError):
return None
if not (
isinstance(data, dict)
and isinstance(data.get("error"), dict)
and "message" in data["error"]
):
return None
try:
err = ErrorResponse.model_validate(data)
except Exception: # noqa: BLE001 - malformed envelope; pass through
return None
return _error_response(err, request)
def attach_router(app: FastAPI) -> None:
"""Register the enabled ``/cohere/v2/chat`` routes on ``app``.
No-op when either:
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
Cohere v2 endpoints are opt-in because they carry Cohere-specific
request/response semantics (grounding citations, tool_plan,
PLAN/THINKING_CONTENT blocks) that are only meaningful when
serving a Cohere Command-family model.
* the optional ``cohere`` SDK isn't installed (the v2 protocol
models live there)
The two skip paths log at different levels: an operator who set
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
a WARNING (they explicitly asked for the endpoints and they're
silently absent), whereas the default-off skip logs at debug.
"""
enabled = envs.VLLM_ENABLE_COHERE_API
if not enabled:
logger.debug(
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
)
return
if not _SDK_AVAILABLE:
logger.warning(
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
"installed; /cohere/v2/chat will not be exposed. Install "
"with `pip install cohere` to enable the endpoints."
)
return
app.include_router(router)
args = getattr(app.state, "args", None)
if getattr(args, "enable_scale_out", False) or getattr(args, "tokens_only", False):
app.include_router(render_router)
app.add_middleware(CohereErrorEnvelopeMiddleware)
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
This module keeps the OpenAI chat completion protocol classes
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
It provides two things:
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
representation produced by the Cohere parser
(:mod:`vllm.parser.cohere_command`) and consumed by
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
SDK shape -- that conversion happens in the serving layer.
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
are only instantiated by the citation-aware serving handler
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
The module intentionally does *not* import the ``cohere`` Python SDK so the
reasoning parser (which is Cohere-model-specific but ships without the SDK
dependency) can import from here freely.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import Field, model_serializer
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
class CitationSource(OpenAIBaseModel):
"""Source attribution for a :class:`Citation`.
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
discriminator (``document`` or ``tool``); ``id`` is the citing
document/tool-output identifier; ``document`` and ``tool_output`` carry
the original payload that produced the citation.
Sources are fully resolved by the parser at emit time
(see :func:`_melody_sources_to_vllm` in
:mod:`vllm.parser.cohere_command`) using the
``POSITION_TO_SOURCE_KEY`` map forwarded through
``chat_template_kwargs`` by
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
That means every instance that reaches the serving / wire layer
already has ``type`` / ``id`` populated; the serving layer only has
to coerce to :class:`cohere.types.Citation` and rewrite
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
"""
type: Literal["document", "tool"] | None = None
id: str | None = None
document: dict[str, Any] | None = None
tool_output: dict[str, Any] | None = None
class Citation(OpenAIBaseModel):
"""A citation grounding a span of generated text in source material.
vLLM-internal representation used by the Cohere reasoning parser and
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
conversion to the SDK's ``cohere.types.Citation`` happens in
:mod:`vllm.entrypoints.cohere.serving`.
"""
start: int | None = None
"""Start character offset in the surrounding text content."""
end: int | None = None
"""End character offset (exclusive) in the surrounding text content."""
text: str | None = None
"""The cited text snippet."""
sources: list[CitationSource] = Field(default_factory=list)
"""Source documents / tool outputs that ground this citation."""
content_index: int | None = None
"""Index of the content block this citation refers to (when the message
has multiple content blocks)."""
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
"""Which kind of content block this citation grounds: the user-visible
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
tool-plan block (``PLAN``). ``None`` means unspecified."""
class CohereChatMessage(ChatMessage):
""":class:`ChatMessage` extension carrying grounding citations.
Only instantiated by :class:`CohereServingChatV2` (and any other
:class:`OpenAIServingChat` subclass that opts in by overriding
``_create_chat_message``). Regular OpenAI-compatible handlers keep
emitting plain :class:`ChatMessage` so their response schema is
unchanged.
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
so pydantic serializes this subclass with its own schema (including
``citations``) when it flows through
:class:`ChatCompletionResponseChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# ``mode="wrap"`` fully overrides (rather than chains) the
# parent's ``@model_serializer``, so we explicitly delegate via
# ``super()._serialize(handler)`` to preserve the parent's
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
# unset ``citations`` field so the wire matches the OpenAI-style
# contract that optional vLLM extensions are omitted rather than
# serialized as ``null``.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
class CohereDeltaMessage(DeltaMessage):
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
Emitted by :class:`vllm.parser.cohere_command.CohereCommandParser`
on delta events whose payload includes citations. Non-Cohere parsers
return plain :class:`DeltaMessage`, so their streamed shape is
unchanged.
The response envelope declares
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
subclass with its own schema when it flows through
:class:`ChatCompletionResponseStreamChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# See ``CohereChatMessage._serialize`` for why we delegate via
# ``super()`` instead of calling ``handler(self)`` directly.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
-389
View File
@@ -1,389 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cohere Chat v2 API protocol.
The bulk of the wire types come straight from the official ``cohere``
Python SDK so we stay in sync with the upstream specification and
avoid re-declaring the schema. We only own three things locally:
1. The top-level request body model (the SDK doesn't ship one — its
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
2. The non-streaming response envelope (the SDK exposes the message
shape via :class:`AssistantMessageResponse` but no full response
wrapper).
3. The streaming discriminated union (the SDK exports each event type
individually but not as a combined ``Annotated[Union[...],
discriminator]``).
Importing this module pulls in the ``cohere`` package. The router that
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
boots without the SDK installed.
See https://docs.cohere.com/reference/chat for the upstream spec.
"""
from __future__ import annotations
from typing import Annotated, Any, Literal
from cohere import types as _sdk
from cohere.types import (
AssistantChatMessageV2,
AssistantMessageResponse,
ChatMessageV2,
ChatRequestSafetyMode,
Citation,
CitationOptions,
Document,
ResponseFormatV2,
SystemChatMessageV2,
Thinking,
ToolCallV2,
ToolChatMessageV2,
ToolV2,
UserChatMessageV2,
)
from pydantic import BaseModel, Field, field_validator, model_validator
import vllm.envs as envs
from vllm.exceptions import VLLMValidationError
# Re-export the SDK wire-format types alongside our local extensions so
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
# they need from this module.
__all__ = [
"AssistantChatMessageV2",
"AssistantMessageResponse",
"ChatMessageV2",
"Citation",
"CitationEndEvent",
"CitationOptions",
"CitationStartEvent",
"CohereChatV2Request",
"CohereChatV2Response",
"CohereError",
"CohereFinishReason",
"CohereLogprobItem",
"CohereUsage",
"CohereUsageBilledUnits",
"CohereUsageTokens",
"ContentDeltaEvent",
"ContentEndEvent",
"ContentStartEvent",
"Document",
"MessageEndEvent",
"MessageStartEvent",
"ResponseFormatV2",
"SystemChatMessageV2",
"Thinking",
"ToolCallDeltaEvent",
"ToolCallEndEvent",
"ToolCallStartEvent",
"ToolCallV2",
"ToolChatMessageV2",
"ToolPlanDeltaEvent",
"ToolV2",
"UserChatMessageV2",
]
# ---------------------------------------------------------------------------
# Errors
# ---------------------------------------------------------------------------
class CohereError(BaseModel):
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
Cohere's documented error schemas are uniform: ``{message, id}``.
"""
message: str
id: str | None = None
# ---------------------------------------------------------------------------
# Tool choice / finish reasons
# ---------------------------------------------------------------------------
#
# These literals aren't first-class enums in the SDK but are documented at
# https://docs.cohere.com/reference/chat. We declare them here so the
# request/response models can validate them.
CohereToolChoice = Literal["REQUIRED", "NONE"]
CohereFinishReason = Literal[
"COMPLETE",
"STOP_SEQUENCE",
"MAX_TOKENS",
"TOOL_CALL",
"ERROR",
"TIMEOUT",
]
# ---------------------------------------------------------------------------
# Request
# ---------------------------------------------------------------------------
class CohereChatV2Request(BaseModel):
"""Cohere Chat v2 request body.
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
All structured fields delegate to the official SDK types so the body
schema stays in sync with the upstream spec.
"""
model: str
messages: list[ChatMessageV2]
stream: bool | None = False
# Tooling
tools: list[ToolV2] | None = None
strict_tools: bool | None = None
tool_choice: CohereToolChoice | None = None
# Grounding
documents: list[str | Document] | None = None
citation_options: CitationOptions | None = None
# Output
response_format: ResponseFormatV2 | None = None
safety_mode: ChatRequestSafetyMode | None = None
max_tokens: int | None = None
stop_sequences: (
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
) = None
# Sampling
temperature: float | None = None
seed: int | None = None
frequency_penalty: float | None = None
presence_penalty: float | None = None
k: int | None = None
p: float | None = None
logprobs: bool | None = None
# Reasoning
thinking: Thinking | None = None
# Scheduling
priority: int | None = Field(default=None, ge=-(2**63), le=2**63 - 1)
# vLLM-specific extensions (not in Cohere spec). These mirror what the
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
# the same engine knobs when needed.
watermarking: bool | None = Field(
default=None,
description=(
"Whether to apply the engine's configured watermark to this request."
),
)
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def _validate_model(cls, v: str) -> str:
if not v:
raise VLLMValidationError("model is required", parameter="model")
return v
@field_validator("max_tokens")
@classmethod
def _validate_max_tokens(cls, v: int | None) -> int | None:
if v is not None and v < 0:
raise VLLMValidationError(
"max_tokens must be non-negative",
parameter="max_tokens",
value=v,
)
return v
@model_validator(mode="after")
def _validate_tool_choice(self) -> CohereChatV2Request:
if self.tool_choice == "REQUIRED" and not self.tools:
raise VLLMValidationError(
'tool_choice="REQUIRED" requires at least one tool in tools.',
parameter="tool_choice",
value=self.tool_choice,
)
return self
@field_validator("messages", mode="before")
@classmethod
def _normalize_message_roles(cls, v: Any) -> Any:
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
the four literal roles ``user`` / ``assistant`` / ``system`` /
``tool``. OpenAI's ``developer`` role is documented as
high-priority system instructions, so we alias it onto
``system`` *before* the SDK discriminator runs (otherwise it
rejects the message with a ``literal_error`` against each union
member). Mirrors ``_role_to_melody`` in the renderer so the
same alias is honoured no matter which surface the message
arrives through.
``mode="before"`` is required so the rewrite happens before the
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
union; a default-mode validator would never see ``developer``
because validation would have already failed. On any
structural malformation (non-iterable input, items without a
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
and let Pydantic's normal coercion surface a precise error.
"""
try:
return [
{**msg, "role": "system"}
if msg.get("role", "").lower() == "developer"
else msg
for msg in v
]
except (AttributeError, TypeError):
return v
@field_validator("messages")
@classmethod
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
if not v:
raise VLLMValidationError(
"messages must contain at least one message", parameter="messages"
)
return v
# ---------------------------------------------------------------------------
# Usage / Logprobs
# ---------------------------------------------------------------------------
#
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
# envelope is documented separately in the OpenAPI spec and we declare it
# here.
class CohereUsageBilledUnits(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
search_units: float | None = None
classifications: float | None = None
class CohereUsageTokens(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
class CohereUsage(BaseModel):
billed_units: CohereUsageBilledUnits | None = None
tokens: CohereUsageTokens | None = None
cached_tokens: float | None = None
class CohereLogprobItem(BaseModel):
text: str | None = None
token_ids: list[int]
logprobs: list[float] | None = None
# ---------------------------------------------------------------------------
# Non-streaming response
# ---------------------------------------------------------------------------
class CohereChatV2Response(BaseModel):
"""Cohere Chat v2 non-streaming response body.
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
the documented v2 response envelope (``id``, ``finish_reason``,
``usage``, ``logprobs``). The single constructor in
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
supplying a non-empty ``id`` (falling back to a synthesized one if
the upstream response is missing it) to this model.
"""
id: str
finish_reason: CohereFinishReason
message: AssistantMessageResponse
usage: CohereUsage | None = None
logprobs: list[CohereLogprobItem] | None = None
# vLLM-specific extension.
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
# ---------------------------------------------------------------------------
# Streaming events
# ---------------------------------------------------------------------------
#
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
# Sent Events; each event's ``type`` field carries the discriminator. The
# SDK exposes a Pydantic model per event but none of them declare ``type``
# as a field (the SDK relies on its own deserializer for discrimination),
# so a naive ``model_dump_json()`` would silently drop the discriminator
# and break clients that demux on ``type``.
#
# We therefore subclass each SDK event and bake the wire-format ``type``
# string in as a ``Literal`` field with a default. ``model_dump()`` now
# emits ``type`` for free, and surfaces can simply construct the event
# class and serialize it -- no manual ``type`` parameter needed.
#
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
# ``StreamedChatResponseV2`` schema for the wire-format reference.
class MessageStartEvent(_sdk.ChatMessageStartEvent):
type: Literal["message-start"] = "message-start"
class ContentStartEvent(_sdk.ChatContentStartEvent):
type: Literal["content-start"] = "content-start"
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
type: Literal["content-delta"] = "content-delta"
class ContentEndEvent(_sdk.ChatContentEndEvent):
type: Literal["content-end"] = "content-end"
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
type: Literal["tool-plan-delta"] = "tool-plan-delta"
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
type: Literal["tool-call-start"] = "tool-call-start"
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
type: Literal["tool-call-delta"] = "tool-call-delta"
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
type: Literal["tool-call-end"] = "tool-call-end"
class CitationStartEvent(_sdk.CitationStartEvent):
type: Literal["citation-start"] = "citation-start"
class CitationEndEvent(_sdk.CitationEndEvent):
type: Literal["citation-end"] = "citation-end"
class MessageEndEvent(_sdk.ChatMessageEndEvent):
type: Literal["message-end"] = "message-end"
File diff suppressed because it is too large Load Diff
+1 -5
View File
@@ -31,7 +31,6 @@ from vllm.renderers.inputs.preprocess import (
)
from vllm.sampling_params import RequestOutputKind
from vllm.utils.counter import Counter
from vllm.utils.mistral import is_mistral_tokenizer
from vllm.utils.tqdm_utils import maybe_tqdm
from vllm.v1.engine.llm_engine import LLMEngine
@@ -191,10 +190,7 @@ class OfflineInferenceMixin:
add_generation_prompt=add_generation_prompt,
continue_final_message=continue_final_message,
tools=tools,
tokenize=(
is_mistral_tokenizer(renderer.tokenizer)
or self.model_config.enable_prompt_embeds
),
tokenize=self.model_config.enable_prompt_embeds,
),
),
mm_processor_kwargs=mm_processor_kwargs,
+1 -79
View File
@@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any
from fastapi import FastAPI
import vllm.envs as envs
if TYPE_CHECKING:
from argparse import Namespace
@@ -37,17 +35,6 @@ def register_generate_api_routers(app: FastAPI):
register_completion_api_router(app)
from vllm.entrypoints.anthropic.api_router import (
attach_router as register_anthropic_api_router,
)
register_anthropic_api_router(app)
from vllm.entrypoints.cohere.api_router import (
attach_router as register_cohere_api_router,
)
register_cohere_api_router(app)
from .generative_scoring.api_router import register_generative_scoring_api_router
@@ -62,24 +49,6 @@ async def init_generate_state(
supported_tasks: tuple["SupportedTask", ...],
default_chat_template_kwargs: dict[str, Any],
):
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.renderers.chat_utils import load_chat_template
# The Cohere serving handler depends on the optional `cohere` SDK for
# its wire-format protocol models, and is additionally gated on the
# `VLLM_ENABLE_COHERE_API` env flag (see
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
# entirely when the endpoint isn't going to be exposed, both because
# the SDK may not be installed and because the serving object holds
# nontrivial state (chat handler, warmup) that would otherwise be
# unused.
if envs.VLLM_ENABLE_COHERE_API:
try:
from vllm.entrypoints.cohere.serving import CohereServingChatV2
except ImportError:
CohereServingChatV2 = None # type: ignore[assignment,misc]
else:
CohereServingChatV2 = None # type: ignore[assignment,misc]
from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch,
)
@@ -87,6 +56,7 @@ async def init_generate_state(
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
from vllm.renderers.chat_utils import load_chat_template
# Applied before any serving class is constructed so that each one picks
# up the chosen mode on its first cache miss.
@@ -168,54 +138,6 @@ async def init_generate_state(
if "generate" in supported_tasks
else None
)
state.anthropic_serving_messages = (
AnthropicServingMessages(
engine_client,
state.openai_serving_models,
args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
default_chat_template_kwargs=default_chat_template_kwargs,
disabled_thinking_effort=args.anthropic_disabled_thinking_effort,
)
if "generate" in supported_tasks
else None
)
state.cohere_serving_chat_v2 = (
CohereServingChatV2(
engine_client,
state.openai_serving_models,
args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
default_chat_template_kwargs=default_chat_template_kwargs,
is_reasoning_model=args.cohere_is_reasoning_model,
)
if CohereServingChatV2 is not None and "generate" in supported_tasks
else None
)
from .generative_scoring.serving import ServingGenerativeScoring
-42
View File
@@ -1,42 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.config import ModelConfig
from vllm.tasks import SupportedTask
if TYPE_CHECKING:
from vllm.entrypoints.serve.sagemaker.api_router import (
EndpointFn,
GetHandlerFn,
RequestType,
)
def get_generate_invocation_types(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
# NOTE: Items defined earlier take higher priority
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
if "generate" in supported_tasks:
from vllm.entrypoints.openai.chat_completion.api_router import (
chat,
create_chat_completion,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.api_router import (
completion,
create_completion,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
invocation_types += [
(ChatCompletionRequest, (chat, create_chat_completion)),
(CompletionRequest, (completion, create_completion)),
]
return invocation_types
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,86 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from http import HTTPStatus
from typing import Any
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.decisions.state import init_decisions_state
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.renderers.chat_utils import load_chat_template
from vllm.tasks import SupportedTask
from .protocol import StructuredDecisionRequest
from .serving import ServingStructuredDecisions
router = APIRouter()
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
return getattr(request.app.state, "serving_structured_decisions", None)
@router.post(
"/v1/systemone",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_structured_decision(
request: StructuredDecisionRequest, raw_request: Request
):
handler = structured_decisions(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support the structured decision API"
)
result = await handler.create_decision(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def register_structured_decisions_api_router(app: FastAPI):
app.include_router(router)
def init_structured_decisions_state(
engine_client: EngineClient,
state: State,
args: Namespace,
request_logger: RequestLogger | None,
supported_tasks: tuple[SupportedTask, ...],
default_chat_template_kwargs: dict[str, Any],
):
strategy = init_decisions_state(
engine_client,
state,
args,
request_logger,
load_chat_template(args.chat_template),
default_chat_template_kwargs,
supported_tasks,
)
state.serving_structured_decisions = (
ServingStructuredDecisions(
state.openai_serving_models, strategy, request_logger=request_logger
)
if strategy is not None
else None
)
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Request and response models of the structured decisions API (/v1/systemone)."""
import time
from typing import Any, Literal
from pydantic import Field, model_validator
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
from vllm.utils import random_uuid
class QuestionSpec(OpenAIBaseModel):
type: str
instructions: Any = ""
criteria: Any = None
class StructuredDecisionRequest(OpenAIBaseModel):
model: str | None = None
state: Any = Field(..., description="What the questions are about: text or JSON.")
questions: dict[str, QuestionSpec] = Field(
..., description="Question id to question, asked in this order."
)
instructions: str | None = Field(
default=None, description="Context placed ahead of the questions."
)
images: list[str] | None = Field(
default=None, description="Image URLs or data URLs, for models that see."
)
audio: list[str] | None = Field(
default=None, description="Audio URLs or data URLs, for models that hear."
)
videos: list[str] | None = Field(
default=None, description="Video URLs or data URLs, for models that see."
)
chat_template_kwargs: dict[str, Any] | None = None
cache_salt: str | None = Field(
default=None,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
request_id: str = Field(default_factory=random_uuid)
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
class DecisionUsage(OpenAIBaseModel):
input_tokens: int
output_tokens: int
class QuestionDiagnostics(OpenAIBaseModel):
label_mass: float = Field(
description="Probability the model put on the labels, over the full "
"vocabulary. A low value means most of the probability went to tokens "
"that are not labels."
)
argmax_is_label: bool
class StructuredDecisionResponse(OpenAIBaseModel):
id: str
object: Literal["structured_decision"] = "structured_decision"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
answers: dict[str, dict[str, Any]]
usage: DecisionUsage
diagnostics: dict[str, QuestionDiagnostics]
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from typing import Any
from fastapi import Request
from vllm.entrypoints.openai.decisions.question_types import (
Question,
StructuredDecisionError,
build_question,
)
from vllm.entrypoints.openai.decisions.serving import BaseServingDecisions
from vllm.entrypoints.openai.decisions.strategies import DecisionLimits
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from .protocol import (
DecisionUsage,
QuestionDiagnostics,
StructuredDecisionRequest,
StructuredDecisionResponse,
)
def parse_questions(
request: StructuredDecisionRequest,
limits: DecisionLimits,
) -> list[Question]:
if not request.questions:
raise StructuredDecisionError("questions: needs at least one question")
if len(request.questions) > limits.max_questions:
raise StructuredDecisionError(
f"questions: at most {limits.max_questions} for this model"
)
questions = []
for qid, spec in request.questions.items():
if spec.model_extra:
raise StructuredDecisionError(
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
)
questions.append(
build_question(
qid,
spec.type,
spec.instructions,
spec.criteria,
limits.max_options,
labeled=limits.labeled,
)
)
return questions
class ServingStructuredDecisions(BaseServingDecisions):
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
urls = {
"image": request.images,
"audio": request.audio,
"video": request.videos,
}
media = {modality: items for modality, items in urls.items() if items}
if unsupported := media.keys() - self.strategy.modalities:
names = ", ".join(sorted(unsupported))
raise StructuredDecisionError(f"this model does not take {names} inputs")
return media
async def create_decision(
self,
request: StructuredDecisionRequest,
raw_request: Request | None = None,
) -> StructuredDecisionResponse | ErrorResponse:
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
return error
engine_client = self.strategy.context.engine_client
if engine_client.errored:
raise engine_client.dead_error
base_id = self._base_request_id(raw_request, default=request.request_id)
request_id = f"decision-{base_id}"
try:
questions = parse_questions(request, self.limits)
media = self._media(request)
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine_client.check_admission(self.strategy.num_requests(questions))
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
reads = await self.strategy.read(
questions,
request.instructions,
self.strategy.render_state(request.state),
request_id=request_id,
chat_template_kwargs=request.chat_template_kwargs,
lora_request=lora_request,
priority=request.priority,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
media=media,
)
except StructuredDecisionError as e:
return self.create_error_response(e)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
answers: dict[str, dict[str, Any]] = {}
diagnostics: dict[str, QuestionDiagnostics] = {}
for q, read in zip(questions, reads):
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
diagnostics[q.id] = QuestionDiagnostics(
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
)
return StructuredDecisionResponse(
id=request_id,
model=self.models.model_name(lora_request),
answers=answers,
usage=DecisionUsage(
input_tokens=sum(r.input_tokens for r in reads),
output_tokens=sum(r.output_tokens for r in reads),
),
diagnostics=diagnostics,
)
@@ -7,7 +7,6 @@ from typing import cast
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -16,7 +15,6 @@ from vllm.entrypoints.serve.utils.api_utils import process_lora_modules
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
@@ -65,7 +63,7 @@ async def init_app_state(
state.vllm_config = vllm_config
state.args = args
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
state.tool_server = (
await init_tool_server(args) if "generate" in supported_tasks else None
)
@@ -103,22 +101,6 @@ async def init_app_state(
)
state.online_renderer.warmup()
state.online_derenderer = OnlineDerenderer(
model_config=engine_client.model_config,
renderer=engine_client.renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.serving_tokenization = ServingTokenization(
state.openai_serving_models,
state.online_renderer,
@@ -152,19 +134,6 @@ async def init_app_state(
engine_client, state, args, request_logger, supported_tasks
)
from vllm.entrypoints.generate.structured_decisions.api_router import (
init_structured_decisions_state,
)
init_structured_decisions_state(
engine_client,
state,
args,
request_logger,
supported_tasks,
default_chat_template_kwargs,
)
await init_endpoint_plugins_state(engine_client, state, args)
state.enable_server_load_tracking = args.enable_server_load_tracking
@@ -25,33 +25,11 @@ def register_api_routers(
register_models_api_router(app)
from vllm.entrypoints.generate.structured_decisions.api_router import (
register_structured_decisions_api_router,
)
register_structured_decisions_api_router(app)
from vllm.entrypoints.openai.decisions.api_router import (
register_decisions_api_router,
)
register_decisions_api_router(app)
from vllm.entrypoints.serve.sagemaker.api_router import (
attach_router as register_sagemaker_api_router,
)
register_sagemaker_api_router(app, supported_tasks, model_config)
if envs.VLLM_SERVER_DEV_MODE:
from vllm.entrypoints.serve import register_vllm_dev_api_routers
register_vllm_dev_api_routers(app)
from vllm.entrypoints.rl.factories import register_rl_api_routers
register_rl_api_routers(app)
if "generate" in supported_tasks:
from vllm.entrypoints.generate.api_router import (
register_generate_api_routers,
@@ -59,13 +37,6 @@ def register_api_routers(
register_generate_api_routers(app)
from vllm.entrypoints.serve.elastic_ep.api_router import (
attach_router as elastic_ep_attach_router,
)
elastic_ep_attach_router(app)
if "generate" in supported_tasks or "render" in supported_tasks:
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
register_scale_out_api_routers(app, supported_tasks)
+1 -8
View File
@@ -8,7 +8,6 @@ from fastapi import FastAPI
from vllm.config import ModelConfig
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.plugins.endpoint_plugins.interface import attach_endpoint_plugins
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
@@ -35,22 +34,16 @@ def build_app(
app = FastAPI(
openapi_url=None, docs_url=None, redoc_url=None, lifespan=lifespan
)
elif args.enable_offline_docs:
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
else:
app = FastAPI(lifespan=lifespan)
app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
app.state.args = args
app.root_path = args.root_path
register_api_routers(args, app, supported_tasks, model_config)
# Endpoint plugin hooks run after core routers so plugins can inspect them.
# This runs even for the CPU only render server. A plugin eligible for
# the `render` task still gets its routes registered. It receives
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
attach_endpoint_plugins(app, supported_tasks)
init_exception_handler(app)
init_entrypoints_middleware(args, app, supported_tasks)
app = sagemaker_standards_bootstrap(app)
return app
+6 -54
View File
@@ -14,9 +14,6 @@ from typing import Any, Literal
import vllm.envs as envs
from vllm.config import config
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
from vllm.entrypoints.anthropic.protocol import (
AnthropicDisabledThinkingEffortOption,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.logger import init_logger
from vllm.renderers.chat_utils import (
@@ -172,32 +169,6 @@ class BaseFrontendArgs:
"""If set to False, output deltas will not be logged. Relevant only if
--enable-log-outputs is set.
"""
cohere_is_reasoning_model: bool = True
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
reasoning Command-family model. When True (default), the assistant's
chain-of-thought is surfaced as a ``thinking`` content block (or
``content-*`` events on the stream). When False, reasoning is
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
events) whenever the model emits tool calls, matching older non-
reasoning Command models. Has no effect on the non-Cohere
endpoints."""
cohere_format: str = "cmd4"
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
format to render: ``cmd4`` (current Command A+ models; default) or
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
wrong format silently produces a prompt the model wasn't trained
on, which most commonly manifests as the model emitting text but no
citations / tool calls / thinking blocks. Equivalent to passing
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
explicit request-level ``chat_template_kwargs.cohere_format`` takes
priority."""
anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto"
"""Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for
requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking
off for models that support it; ``low`` suits models that always think
(e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses
``low`` when the renderer rejects ``none`` or renders it the same as a
thinking effort, and ``none`` otherwise."""
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
"""If set to True, log the stack trace of error responses"""
tokens_only: bool = False
@@ -207,10 +178,10 @@ class BaseFrontendArgs:
"""
enable_scale_out: bool = False
"""
If set to True, register the scale-out endpoints (`/render`, `/derender`,
`/inference/v1/generate` and `/inference/v1/abort_requests`) on `vllm serve`.
Has no effect on `vllm launch render` or `vllm serve --tokens-only` which
always register their required endpoints regardless of this flag.
If set to True, register the scale-out endpoints (`/inference/v1/generate`
and `/inference/v1/abort_requests`) on `vllm serve`. `vllm serve
--tokens-only` always registers the endpoints it needs regardless of this
flag.
"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
@@ -360,11 +331,6 @@ class FrontendArgs(BaseFrontendArgs):
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
"""Maximum number of HTTP headers allowed in a request for h11 parser.
Helps mitigate header abuse. Default: 256."""
enable_offline_docs: bool = False
"""
Enable offline FastAPI documentation for air-gapped environments.
Uses vendored static assets bundled with vLLM.
"""
enable_flash_late_interaction: bool = True
"""If set, run pooling score MaxSim on GPU in the API server process.
Can significantly improve late-interaction scoring performance.
@@ -404,16 +370,6 @@ class FrontendArgs(BaseFrontendArgs):
return frontend_kwargs
def resolve_default_chat_template_kwargs(
args: argparse.Namespace,
) -> dict[str, Any]:
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
kwargs.setdefault("cohere_format", args.cohere_format)
return kwargs
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Create the CLI argument parser used by the OpenAI API server.
@@ -453,9 +409,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
action="store_true",
default=False,
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. "
"Cannot be combined with --grpc-port, which enables the Rust "
"frontend's gRPC services.",
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc].",
)
parser = FrontendArgs.add_cli_args(parser)
parser = AsyncEngineArgs.add_cli_args(parser)
@@ -465,9 +419,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
def validate_parsed_serve_args(args: argparse.Namespace):
"""Quick checks for model serve args that raise prior to loading."""
# `vllm launch <component>` builds its parser with make_arg_parser too (see
# LaunchSubcommandBase.add_cli_args), so its args are serve args as well.
if hasattr(args, "subparser") and args.subparser not in ("serve", "launch"):
if hasattr(args, "subparser") and args.subparser != "serve":
return
# Ensure that the chat template is valid; raises if it likely isn't
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from starlette.datastructures import State
from vllm.config import VllmConfig
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.scale_out.factories import init_render_state
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers import renderer_from_config
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
async def init_render_app_state(
vllm_config: VllmConfig,
state: State,
args: Namespace,
) -> None:
"""Initialise FastAPI app state for a CPU-only render server.
Unlike :func:`init_app_state` this function does not require an
:class:`~vllm.engine.protocol.EngineClient`; it bootstraps the
preprocessing pipeline (renderer, input_processor)
directly from the :class:`~vllm.config.VllmConfig`.
"""
served_model_names = args.served_model_name or [args.model]
model_registry = OpenAIModelRegistry(
model_config=vllm_config.model_config,
base_model_paths=[
BaseModelPath(name=name, model_path=args.model)
for name in served_model_names
],
)
if args.enable_log_requests:
request_logger = RequestLogger(max_log_len=args.max_log_len)
else:
request_logger = None
renderer = renderer_from_config(vllm_config)
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = await init_tool_server(args)
# The config-resolved reasoning parser carries both the CLI flag (merged
# in the entrypoint via `create_structured_outputs_config`) and any
# model-specific default applied by `verify_and_update_config`
# (e.g. "openai_gptoss" for gpt_oss), matching the main API server.
# Keep `args.reasoning_parser` as the first source so callers that build
# a VllmConfig without that merge still honor an explicit flag.
reasoning_parser = (
args.reasoning_parser or vllm_config.structured_outputs_config.reasoning_parser
)
state.online_renderer = OnlineRenderer(
model_config=vllm_config.model_config,
renderer=renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_renderer.warmup()
state.online_derenderer = OnlineDerenderer(
model_config=vllm_config.model_config,
renderer=renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
tool_strict_level=args.tool_strict_level,
reasoning_parser=reasoning_parser,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.openai_serving_models = model_registry
state.serving_tokenization = ServingTokenization(
model_registry,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
init_render_state(state, request_logger)
state.vllm_config = vllm_config
# Disable stats logging — there is no engine to poll.
state.log_stats = False
state.engine_client = None
state.args = args
state.enable_server_load_tracking = False
state.server_load_metrics = 0
# No `EngineClient` exists for the render server, so plugins get `None` and
# must handle it themselves (see `EndpointPlugin.init_state`).
await init_endpoint_plugins_state(None, state, args)
-129
View File
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import asyncio
import signal
import socket
from argparse import Namespace
from vllm import AsyncEngineArgs, envs
from vllm.config import VllmConfig
from vllm.logger import configure_logging_from_args, init_logger
from ..app import build_app
from ..launcher import serve_http, setup_server
from ..utils.server_utils import get_uvicorn_log_config
from .app_state import init_render_app_state
logger = init_logger("vllm.entrypoints.launchers.render.entry")
async def build_and_serve_renderer(
vllm_config: VllmConfig,
listen_address: str,
sock: socket.socket,
args: Namespace,
**uvicorn_kwargs,
) -> asyncio.Task:
"""Build FastAPI app for a CPU-only render server, initialize state, and
start serving.
Returns the shutdown task for the caller to await.
"""
# Get uvicorn log config (from file or with endpoint filter)
log_config = get_uvicorn_log_config(args)
if log_config is not None:
uvicorn_kwargs["log_config"] = log_config
app = build_app(args, ("render",))
await init_render_app_state(vllm_config, app.state, args)
logger.info("Starting vLLM server on %s", listen_address)
return await serve_http(
app,
sock=sock,
enable_ssl_refresh=args.enable_ssl_refresh,
host=args.host,
port=args.port,
log_level=args.uvicorn_log_level,
# NOTE: When the 'disable_uvicorn_access_log' value is True,
# no access log will be output.
access_log=not args.disable_uvicorn_access_log,
timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
ssl_keyfile=args.ssl_keyfile,
ssl_certfile=args.ssl_certfile,
ssl_ca_certs=args.ssl_ca_certs,
ssl_cert_reqs=args.ssl_cert_reqs,
ssl_ciphers=args.ssl_ciphers,
h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
h11_max_header_count=args.h11_max_header_count,
**uvicorn_kwargs,
)
async def run_launch_fastapi(args: argparse.Namespace) -> None:
"""Run the online serving layer with FastAPI (no GPU inference)."""
# Interrupt initialization if SIGTERM arrives before uvicorn installs
# its own signal handlers. Once uvicorn is running it replaces this.
def _interrupt_init(*_) -> None:
raise KeyboardInterrupt("terminated")
signal.signal(signal.SIGTERM, _interrupt_init)
# 1. Socket binding
listen_address, sock = setup_server(args, reuse_port=False)
# 2. Build and serve the API server
engine_args = AsyncEngineArgs.from_cli_args(args)
model_config = engine_args.create_model_config()
# Render servers preprocess data only — no inference, no quantized kernels.
# Clear quantization so VllmConfig skips quant dtype/capability validation.
model_config.quantization = None
# Render servers never allocate KV cache; suppress the spurious CPU KV
# cache space warning from CpuPlatform.check_and_update_config.
envs.VLLM_CPU_KVCACHE_SPACE = 0
# Merge frontend parser flags (e.g. --reasoning-parser) into the
# structured outputs config, mirroring `create_engine_config`, which this
# GPU-less entrypoint bypasses. Model-specific defaults are then applied
# by `verify_and_update_config` during VllmConfig construction.
vllm_config = VllmConfig(
model_config=model_config,
logging_config=engine_args.create_logging_config(),
structured_outputs_config=engine_args.create_structured_outputs_config(),
)
shutdown_task = await build_and_serve_renderer(
vllm_config, listen_address, sock, args
)
try:
await shutdown_task
finally:
sock.close()
if __name__ == "__main__":
import uvloop
from vllm.entrypoints.serve.utils.api_utils import cli_env_setup
from vllm.utils.argparse_utils import FlexibleArgumentParser
from ..cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
cli_env_setup()
parser = FlexibleArgumentParser(
description="Starts a GPU-less rendering server "
"for preprocessing and postprocessing only"
)
parser = make_arg_parser(parser)
args = parser.parse_args()
configure_logging_from_args(args)
validate_parsed_serve_args(args)
uvloop.run(run_launch_fastapi(args))
-672
View File
@@ -1,672 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import contextlib
import json
import sys
import tempfile
from argparse import Namespace
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from io import StringIO
from typing import Any, NoReturn, TypeAlias
from urllib.parse import urlparse
import aiohttp
import pybase64 as base64
import pydantic
import torch
from prometheus_client import start_http_server
from pydantic import TypeAdapter, field_validator
from pydantic_core.core_schema import ValidationInfo
from starlette.datastructures import State
from starlette.responses import JSONResponse
from tqdm import tqdm
from urllib3.util import parse_url
import vllm.envs as envs
from vllm.config import config
from vllm.connections import HTTPResponseSizeExceededError, global_http_connection
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import (
ErrorResponse,
OpenAIBaseModel,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import configure_logging_from_args, init_logger
from vllm.reasoning import ReasoningParserManager
from vllm.utils import random_uuid
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.mem_constants import MiB_bytes
from vllm.version import __version__ as VLLM_VERSION
from .api_server.entry import init_app_state
from .cli_args import BaseFrontendArgs
logger = init_logger(__name__)
def _get_audio_download_limit_bytes() -> int:
return int(envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB * MiB_bytes)
def _raise_audio_filesize_limit(size_bytes: int) -> NoReturn:
raise VLLMValidationError(
"Maximum file size exceeded",
parameter="audio_filesize_mb",
value=size_bytes / MiB_bytes,
)
BatchRequestInputBody: TypeAlias = ChatCompletionRequest
class BatchRequestInput(OpenAIBaseModel):
"""The per-line object of the batch input file.
NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
"""
# A developer-provided per-request id that will be used to match outputs to
# inputs. Must be unique for each request in a batch.
custom_id: str
# The HTTP method to be used for the request. Currently only POST is
# supported.
method: str
# The OpenAI API relative URL to be used for the request. Currently
# /v1/chat/completions is supported.
url: str
# The parameters of the request.
body: BatchRequestInputBody
@field_validator("body", mode="plain")
@classmethod
def check_type_for_url(cls, value: Any, info: ValidationInfo):
# Use url to disambiguate models
url: str = info.data["url"]
if url == "/v1/chat/completions":
return ChatCompletionRequest.model_validate(value)
return TypeAdapter(BatchRequestInputBody).validate_python(value)
AllResponse: TypeAlias = (
ChatCompletionResponse
)
class BatchResponseData(OpenAIBaseModel):
# HTTP status code of the response.
status_code: int = 200
# An unique identifier for the API request.
request_id: str
# The body of the response.
body: AllResponse | None = None
class BatchRequestOutput(OpenAIBaseModel):
"""The per-line object of the batch output and error files"""
id: str
# A developer-provided per-request id that will be used to match outputs to
# inputs.
custom_id: str
response: BatchResponseData | None
# For requests that failed with a non-HTTP error, this will contain more
# information on the cause of the failure.
error: Any | None
@config
class BatchFrontendArgs(BaseFrontendArgs):
"""Arguments for the batch runner frontend."""
input_file: str | None = None
"""The path or url to a single input file. Currently supports local file
paths, or the http protocol (http or https). If a URL is specified,
the file should be available via HTTP GET."""
output_file: str | None = None
"""The path or url to a single output file. Currently supports
local file paths, or web (http or https) urls. If a URL is specified,
the file should be available via HTTP PUT."""
output_tmp_dir: str | None = None
"""The directory to store the output file before uploading it
to the output URL."""
enable_metrics: bool = False
"""Enable Prometheus metrics"""
host: str | None = None
"""Host name for the Prometheus metrics server
(only needed if enable-metrics is set)."""
port: int = 8000
"""Port number for the Prometheus metrics server
(only needed if enable-metrics is set)."""
url: str = "0.0.0.0"
"""[DEPRECATED] Host name for the Prometheus metrics server
(only needed if enable-metrics is set). Use --host instead."""
@classmethod
def _customize_cli_kwargs(
cls,
frontend_kwargs: dict[str, Any],
) -> dict[str, Any]:
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
frontend_kwargs["input_file"]["flags"] = ["-i"]
frontend_kwargs["input_file"]["required"] = True
frontend_kwargs["output_file"]["flags"] = ["-o"]
frontend_kwargs["output_file"]["required"] = True
frontend_kwargs["enable_metrics"]["action"] = "store_true"
frontend_kwargs["url"]["deprecated"] = True
return frontend_kwargs
def make_arg_parser(parser: FlexibleArgumentParser):
parser = BatchFrontendArgs.add_cli_args(parser)
parser = AsyncEngineArgs.add_cli_args(parser)
return parser
def parse_args():
parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
args = make_arg_parser(parser).parse_args()
configure_logging_from_args(args)
# Backward compatibility: If --url is set, use it for host
url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
host_explicit = any(
arg == "--host" or arg.startswith("--host=") for arg in sys.argv
)
if url_explicit and hasattr(args, "url") and not host_explicit:
args.host = args.url
logger.warning_once(
"Using --url for metrics is deprecated. Please use --host instead."
)
return args
# explicitly use pure text format, with a newline at the end
# this makes it impossible to see the animation in the progress bar
# but will avoid messing up with ray or multiprocessing, which wraps
# each line of output with some prefix.
_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
class BatchProgressTracker:
def __init__(self):
self._total = 0
self._pbar: tqdm | None = None
def submitted(self):
self._total += 1
def completed(self):
if self._pbar:
self._pbar.update()
def pbar(self) -> tqdm:
enable_tqdm = (
not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
)
self._pbar = tqdm(
total=self._total,
unit="req",
desc="Running batch",
mininterval=5,
disable=not enable_tqdm,
bar_format=_BAR_FORMAT,
)
return self._pbar
async def read_file(path_or_url: str) -> str:
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
async with aiohttp.ClientSession() as session, session.get(path_or_url) as resp:
resp.raise_for_status()
return await resp.text()
else:
with open(path_or_url, encoding="utf-8") as f:
return f.read()
async def write_local_file(
output_path: str, batch_outputs: list[BatchRequestOutput]
) -> None:
"""Write the responses to a local file.
output_path: The path to write the responses to.
batch_outputs: The list of batch outputs to write.
"""
# We should make this async, but as long as run_batch runs as a
# standalone program, blocking the event loop won't affect performance.
with open(output_path, "w", encoding="utf-8") as f:
for o in batch_outputs:
print(o.model_dump_json(), file=f)
async def upload_data(output_url: str, data_or_file: str, from_file: bool) -> None:
"""Upload a local file to a URL.
output_url: The URL to upload the file to.
data_or_file: Either the data to upload or the path to the file to upload.
from_file: If True, data_or_file is the path to the file to upload.
"""
# Timeout is a common issue when uploading large files.
# We retry max_retries times before giving up.
max_retries = 5
# Number of seconds to wait before retrying.
delay = 5
for attempt in range(1, max_retries + 1):
try:
# We increase the timeout to 1000 seconds to allow
# for large files (default is 300).
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=1000)
) as session:
if from_file:
with open(data_or_file, "rb") as file:
async with session.put(output_url, data=file) as response:
if not 200 <= response.status < 300:
error_text = await response.text()
raise Exception(
f"Failed to upload file.\n"
f"Status: {response.status}\n"
f"Response: {error_text}"
)
else:
async with session.put(output_url, data=data_or_file) as response:
if not 200 <= response.status < 300:
error_text = await response.text()
raise Exception(
f"Failed to upload data.\n"
f"Status: {response.status}\n"
f"Response: {error_text}"
)
# Upload succeeded; do not retry.
return
except Exception as e:
if attempt < max_retries:
logger.error(
"Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...", # noqa: E501
attempt,
e,
delay,
)
await asyncio.sleep(delay)
else:
raise Exception(
f"Failed to upload data (attempt {attempt}). Error message: {str(e)}." # noqa: E501
) from e
async def write_file(
path_or_url: str, batch_outputs: list[BatchRequestOutput], output_tmp_dir: str
) -> None:
"""Write batch_outputs to a file or upload to a URL.
path_or_url: The path or URL to write batch_outputs to.
batch_outputs: The list of batch outputs to write.
output_tmp_dir: The directory to store the output file before uploading it
to the output URL.
"""
if path_or_url.startswith("http://") or path_or_url.startswith("https://"):
if output_tmp_dir is None:
logger.info("Writing outputs to memory buffer")
output_buffer = StringIO()
for o in batch_outputs:
print(o.model_dump_json(), file=output_buffer)
output_buffer.seek(0)
logger.info("Uploading outputs to %s", path_or_url)
await upload_data(
path_or_url,
output_buffer.read().strip().encode("utf-8"),
from_file=False,
)
else:
# Write responses to a temporary file and then upload it to the URL.
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
dir=output_tmp_dir,
prefix="tmp_batch_output_",
suffix=".jsonl",
) as f:
logger.info("Writing outputs to temporary local file %s", f.name)
await write_local_file(f.name, batch_outputs)
logger.info("Uploading outputs to %s", path_or_url)
await upload_data(path_or_url, f.name, from_file=True)
else:
logger.info("Writing outputs to local file %s", path_or_url)
await write_local_file(path_or_url, batch_outputs)
async def download_bytes_from_url(
url: str,
allowed_media_domains: list[str] | None = None,
) -> bytes:
"""Download data from a URL or decode from a data URL.
Args:
url: Either an HTTP/HTTPS URL or a data URL (data:...;base64,...)
allowed_media_domains: If set, only HTTP/HTTPS URLs whose hostname
is in this list are permitted. data: URLs are not subject to
this restriction.
Returns:
Data as bytes
"""
parsed = urlparse(url)
# Handle data URLs (base64 encoded) - not subject to domain restrictions
if parsed.scheme == "data":
# Format: data:...;base64,<base64_data>
if "," in url:
header, data = url.split(",", 1)
if "base64" in header:
decoded_size_bytes = len(data.rstrip("=")) * 3 // 4
if decoded_size_bytes > _get_audio_download_limit_bytes():
_raise_audio_filesize_limit(decoded_size_bytes)
return base64.b64decode(data)
else:
raise VLLMValidationError(
f"Unsupported data URL encoding: {header}",
parameter="url",
)
else:
raise VLLMValidationError(
f"Invalid data URL format: {url}",
parameter="url",
)
# Handle HTTP/HTTPS URLs
elif parsed.scheme in ("http", "https"):
if allowed_media_domains is not None:
url_spec = parse_url(url)
if url_spec.hostname not in allowed_media_domains:
raise VLLMValidationError(
f"The URL must be from one of the allowed domains: "
f"{allowed_media_domains}. Input URL domain: "
f"{url_spec.hostname}",
parameter="url",
value=url_spec.hostname,
)
# Use the normalized URL to prevent parsing discrepancies
# between urllib3 and aiohttp (e.g. backslash-@ attacks).
url = url_spec.url
try:
return await global_http_connection.async_get_bytes(
url,
allow_redirects=envs.VLLM_MEDIA_URL_ALLOW_REDIRECTS,
max_bytes=_get_audio_download_limit_bytes(),
)
except HTTPResponseSizeExceededError as exc:
_raise_audio_filesize_limit(exc.received_bytes)
else:
raise VLLMValidationError(
f"Unsupported URL scheme: {parsed.scheme}. "
"Supported schemes: http, https, data",
parameter="url",
value=parsed.scheme,
)
def make_error_request_output(
request: BatchRequestInput, error_msg: str
) -> BatchRequestOutput:
batch_output = BatchRequestOutput(
id=f"vllm-{random_uuid()}",
custom_id=request.custom_id,
response=BatchResponseData(
status_code=HTTPStatus.BAD_REQUEST,
request_id=f"vllm-batch-{random_uuid()}",
),
error=error_msg,
)
return batch_output
async def make_async_error_request_output(
request: BatchRequestInput, error_msg: str
) -> BatchRequestOutput:
return make_error_request_output(request, error_msg)
async def run_request(
serving_engine_func: Callable,
request: BatchRequestInput,
tracker: BatchProgressTracker,
) -> BatchRequestOutput:
try:
response = await serving_engine_func(request.body)
except Exception as e:
response = create_error_response(e)
if isinstance(response, JSONResponse):
with contextlib.suppress(pydantic.ValidationError):
response = TypeAdapter(AllResponse | ErrorResponse).validate_python(
json.loads(response.body)
)
if isinstance(response, AllResponse):
batch_output = BatchRequestOutput(
id=f"vllm-{random_uuid()}",
custom_id=request.custom_id,
response=BatchResponseData(
body=response, request_id=f"vllm-batch-{random_uuid()}"
),
error=None,
)
elif isinstance(response, ErrorResponse):
batch_output = BatchRequestOutput(
id=f"vllm-{random_uuid()}",
custom_id=request.custom_id,
response=BatchResponseData(
status_code=response.error.code,
request_id=f"vllm-batch-{random_uuid()}",
),
error=response,
)
else:
batch_output = make_error_request_output(
request, error_msg="Request must not be sent in stream mode"
)
tracker.completed()
return batch_output
WrapperFn: TypeAlias = Callable[[Callable], Callable]
def handle_endpoint_request(
request: BatchRequestInput,
tracker: BatchProgressTracker,
url_matcher: Callable[[str], bool],
handler_getter: Callable[[], Callable | None],
wrapper_fn: WrapperFn | None = None,
) -> Awaitable[BatchRequestOutput] | None:
"""Generic handler for endpoint requests.
Args:
request: The batch request input
tracker: Progress tracker for the batch
url_matcher: Function that takes a URL and returns True if it matches
handler_getter: Function that returns the handler function or None
wrapper_fn: Optional function to wrap the handler (e.g., for transcriptions)
Returns:
Awaitable[BatchRequestOutput] if the request was handled,
None if URL didn't match
"""
if not url_matcher(request.url):
return None
handler_fn = handler_getter()
if handler_fn is None:
error_msg = f"Model does not support endpoint: {request.url}"
return make_async_error_request_output(request, error_msg=error_msg)
# Apply wrapper if provided (e.g., for transcriptions/translations)
if wrapper_fn is not None:
handler_fn = wrapper_fn(handler_fn)
tracker.submitted()
return run_request(handler_fn, request, tracker)
async def build_endpoint_registry(
engine_client: EngineClient,
args: Namespace,
) -> dict[str, dict[str, Any]]:
"""Build the endpoint registry with all serving objects and handler configurations.
Args:
engine_client: The engine client
args: Command line arguments
Returns:
Dictionary mapping endpoint keys to their configurations
"""
supported_tasks = await engine_client.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
# Create a state object to hold serving objects
state = State()
# Initialize all serving objects using init_app_state
# This provides full functionality including chat template processing,
# LoRA support, tool servers, etc.
await init_app_state(engine_client, state, args, supported_tasks)
# Get serving objects from state (defaulting to None if not set)
openai_serving_chat = getattr(state, "openai_serving_chat", None)
# Registry of endpoint configurations
endpoint_registry: dict[str, dict[str, Any]] = {
"completions": {
"url_matcher": lambda url: url == "/v1/chat/completions",
"handler_getter": lambda: (
openai_serving_chat.create_chat_completion
if openai_serving_chat is not None
else None
),
"wrapper_fn": None,
},
}
return endpoint_registry
def validate_run_batch_args(args):
valid_reasoning_parsers = ReasoningParserManager.list_registered()
if (
reasoning_parser := args.structured_outputs_config.reasoning_parser
) and reasoning_parser not in valid_reasoning_parsers:
raise KeyError(
f"invalid reasoning parser: {reasoning_parser} "
f"(chose from {{ {','.join(valid_reasoning_parsers)} }})"
)
async def run_batch(
engine_client: EngineClient,
args: Namespace,
) -> None:
endpoint_registry = await build_endpoint_registry(
engine_client=engine_client,
args=args,
)
tracker = BatchProgressTracker()
logger.info("Reading batch from %s...", args.input_file)
# Submit all requests in the file to the engine "concurrently".
response_futures: list[Awaitable[BatchRequestOutput]] = []
for request_json in (await read_file(args.input_file)).strip().split("\n"):
# Skip empty lines.
request_json = request_json.strip()
if not request_json:
continue
request = BatchRequestInput.model_validate_json(request_json)
# Use the last segment of the URL as the endpoint key.
# More advanced URL matching is done in url_matcher of endpoint_registry.
endpoint_key = request.url.split("/")[-1]
result = None
if endpoint_key in endpoint_registry:
endpoint_config = endpoint_registry[endpoint_key]
result = handle_endpoint_request(
request,
tracker,
url_matcher=endpoint_config["url_matcher"],
handler_getter=endpoint_config["handler_getter"],
wrapper_fn=endpoint_config["wrapper_fn"],
)
if result is not None:
response_futures.append(result)
else:
response_futures.append(
make_async_error_request_output(
request,
error_msg=f"URL {request.url} was used. "
"Supported endpoints: /v1/chat/completions.",
)
)
with tracker.pbar():
responses = await asyncio.gather(*response_futures)
await write_file(args.output_file, responses, args.output_tmp_dir)
async def main(args: Namespace):
from vllm.entrypoints.launchers.api_server.entry import build_async_engine_client
from vllm.usage.usage_lib import UsageContext
validate_run_batch_args(args)
async with build_async_engine_client(
args,
usage_context=UsageContext.OPENAI_BATCH_RUNNER,
) as engine_client:
await run_batch(engine_client, args)
if __name__ == "__main__":
args = parse_args()
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
logger.info("args: %s", args)
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
# to publish metrics at the /metrics endpoint.
if args.enable_metrics:
logger.info("Prometheus metrics enabled")
start_http_server(port=args.port, addr=args.host)
else:
logger.info("Prometheus metrics disabled")
asyncio.run(main(args))
+1 -2
View File
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
from vllm.engine.arg_utils import EngineArgs
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
from vllm.entrypoints.rl.offline import RLOfflineMixin
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
from vllm.inputs import PromptType
from vllm.logger import configure_logging_if_needed, init_logger
@@ -59,7 +58,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
class LLM(BeamSearchOfflineMixin, OfflineInferenceMixin):
"""An LLM for generating texts from given prompts and sampling parameters.
This class includes a tokenizer, a language model (possibly distributed
-4
View File
@@ -18,8 +18,6 @@ from vllm.entrypoints.launchers.launcher import (
setup_server,
validate_api_server_args,
)
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
warnings.warn(
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
@@ -35,13 +33,11 @@ __all__ = [
"build_async_engine_client_from_engine_args",
"build_app",
"init_app_state",
"init_render_app_state",
"create_server_socket",
"create_server_unix_socket",
"validate_api_server_args",
"setup_server",
"build_and_serve",
"build_and_serve_renderer",
"run_server",
"run_server_worker",
"register_api_routers",
@@ -107,7 +107,6 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
class ChatCompletionResponseChoice(OpenAIBaseModel):
index: int
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
# so that added fields like ``citations`` survive JSON serialization
# instead of being stripped down to the base schema. Plain
# ``ChatMessage`` instances serialize identically to before.
@@ -162,7 +161,6 @@ class ChatCompletionResponse(OpenAIBaseModel):
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
index: int
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
# so streaming ``citations`` survive JSON serialization. Plain
# ``DeltaMessage`` instances serialize identically to before.
delta: SerializeAsAny[DeltaMessage]
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Translate Decisions questions and answers to and from label reads."""
import json
from .protocol import (
ChoiceAnswer,
ChoiceQuestion,
DecisionAnswer,
DecisionInputMessage,
DecisionQuestion,
PredicateAnswer,
PredicateQuestion,
ScoreAnswer,
)
from .question_types import (
LABELS,
Option,
Question,
StructuredDecisionError,
argmax,
)
from .question_types import (
ChoiceQuestion as LabelQuestion,
)
def make_read_question(
index: int, question: DecisionQuestion, max_options: int = len(LABELS)
) -> Question:
if isinstance(question, PredicateQuestion):
options = [Option("false"), Option("true")]
elif isinstance(question, ChoiceQuestion):
options = [
Option(json.dumps(c.value, ensure_ascii=False), c.description)
for c in question.choices
]
else:
options = [Option(level.label, level.description) for level in question.levels]
limit = min(max_options, len(LABELS))
if len(options) > limit:
raise StructuredDecisionError(
f"This model supports at most {limit} choices per question"
)
return Question(
id=str(index),
type=LabelQuestion(),
instructions=question.instructions,
options=tuple(options),
labels=LABELS[: len(options)],
)
def make_answer(
question: DecisionQuestion, probs: list[float], label_mass: float
) -> DecisionAnswer:
if isinstance(question, PredicateQuestion):
return PredicateAnswer(name=question.name, probability=probs[1])
top = argmax(probs)
confidence = probs[top] * label_mass
if isinstance(question, ChoiceQuestion):
return ChoiceAnswer(
name=question.name,
choice=question.choices[top].value,
probabilities=[
{"value": choice.value, "probability": probability}
for choice, probability in zip(question.choices, probs)
],
confidence=confidence,
)
return ScoreAnswer(
name=question.name,
score=sum(i * probability for i, probability in enumerate(probs)),
probabilities=[
{"value": i, "label": level.label, "probability": probability}
for i, (level, probability) in enumerate(zip(question.levels, probs))
],
confidence=confidence,
)
def input_text(input: str | list[DecisionInputMessage]) -> str:
if isinstance(input, str):
return input
return "\n\n".join(
message.content
if isinstance(message.content, str)
else "\n".join(part.text for part in message.content)
for message in input
)
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import DecisionRequest, DecisionResponse
from .serving import OpenAIServingDecisions
router = APIRouter()
@router.post(
"/v1/decisions",
dependencies=[Depends(validate_json_request)],
response_model=DecisionResponse,
responses={
status.value: {"model": ErrorResponse}
for status in (
HTTPStatus.BAD_REQUEST,
HTTPStatus.NOT_FOUND,
HTTPStatus.INTERNAL_SERVER_ERROR,
HTTPStatus.NOT_IMPLEMENTED,
)
},
)
@with_cancellation
@load_aware_call
async def create_decision(request: DecisionRequest, raw_request: Request):
handler: OpenAIServingDecisions | None = getattr(
raw_request.app.state, "openai_serving_decisions", None
)
if handler is None:
error = create_error_response(
NotImplementedError("The model does not support the Decisions API")
)
return JSONResponse(content=error.model_dump(), status_code=error.error.code)
result = await handler.create_decisions(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def register_decisions_api_router(app: FastAPI):
app.include_router(router)
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""OpenAI Decisions API request and response models."""
from typing import Annotated, Any, Literal
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
InputText = Annotated[StrictStr, Field(max_length=10485760)]
ChoiceValue = StrictStr | StrictBool
class DecisionModel(OpenAIBaseModel):
model_config = ConfigDict(extra="forbid")
class DecisionInputText(DecisionModel):
type: Literal["input_text"]
text: InputText
class DecisionInputMessage(DecisionModel):
role: Literal["user"]
content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)]
type: Literal["message"] = "message"
@model_validator(mode="before")
@classmethod
def reject_images(cls, data: Any) -> Any:
if (
isinstance(data, dict)
and isinstance(data.get("content"), list)
and any(
isinstance(part, dict) and part.get("type") == "input_image"
for part in data["content"]
)
):
raise ValueError(
"Decisions supports text input only; images are not supported"
)
return data
class QuestionBase(DecisionModel):
instructions: ShortText
name: ShortText | None = None
@model_validator(mode="after")
def validate_name(self) -> "QuestionBase":
if "name" in self.model_fields_set and self.name is None:
raise ValueError("name must be a string when provided")
return self
class PredicateQuestion(QuestionBase):
type: Literal["predicate"]
class ChoiceOption(DecisionModel):
value: ChoiceValue
description: ShortText = ""
class ChoiceQuestion(QuestionBase):
type: Literal["choice"]
choices: list[ChoiceOption] = Field(min_length=2, max_length=255)
@model_validator(mode="after")
def validate_choices(self) -> "ChoiceQuestion":
values = [(type(c.value), c.value) for c in self.choices]
if len(set(values)) != len(values):
raise ValueError("choice values must be distinct")
return self
class ScoreLevel(DecisionModel):
label: ShortText
description: ShortText = ""
class ScoreQuestion(QuestionBase):
type: Literal["score"]
levels: list[ScoreLevel] = Field(min_length=2, max_length=10)
DecisionQuestion = Annotated[
PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type")
]
class DecisionRequest(DecisionModel):
model: ShortText
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
class PredicateAnswer(DecisionModel):
type: Literal["predicate"] = "predicate"
name: str | None
probability: float
class ChoiceProbability(DecisionModel):
value: ChoiceValue
probability: float
class ChoiceAnswer(DecisionModel):
type: Literal["choice"] = "choice"
name: str | None
choice: ChoiceValue
probabilities: list[ChoiceProbability]
confidence: float
class ScoreProbability(DecisionModel):
value: int
label: str
probability: float
class ScoreAnswer(DecisionModel):
type: Literal["score"] = "score"
name: str | None
score: float
probabilities: list[ScoreProbability]
confidence: float
DecisionAnswer = Annotated[
PredicateAnswer | ChoiceAnswer | ScoreAnswer,
Field(discriminator="type"),
]
class InputTokensDetails(DecisionModel):
cached_tokens: int = 0
cache_write_tokens: int = 0
class OutputTokensDetails(DecisionModel):
reasoning_tokens: int = 0
class OpenAIDecisionUsage(DecisionModel):
input_tokens: int
input_tokens_details: InputTokensDetails
output_tokens: int
output_tokens_details: OutputTokensDetails
total_tokens: int
class DecisionResponse(DecisionModel):
model: str
answers: list[DecisionAnswer]
usage: OpenAIDecisionUsage
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Question types for structured decisions.
A question type turns a request's criteria into options, writes the question
for the model with a label per option, and builds the answer from the label
probabilities.
"""
import math
import string
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
class StructuredDecisionError(ValueError):
"""An invalid request. Returned as a 400."""
#: A choice labels its options A to Z in order. A single capital letter is one
#: token at the start of a reply for the tokenizers tested. Every read checks it.
LABELS = tuple(string.ascii_uppercase)
@dataclass(frozen=True)
class Option:
name: str
description: str | None = None
@dataclass(frozen=True)
class Question:
id: str
type: "QuestionType"
instructions: str
options: tuple[Option, ...]
labels: tuple[str, ...]
class QuestionType(ABC):
name: ClassVar[str]
#: The labels of a question's options, in option order.
label_set: ClassVar[tuple[str, ...]] = LABELS
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
@abstractmethod
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
@abstractmethod
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
"""The answer for ``question``. ``probs[i]`` is the probability of
``question.labels[i]`` among the labels, and the list sums to 1.
``label_mass`` is the labels' total probability over the vocabulary."""
def reply_line(self, label: str, option: Option) -> str:
return (
f"{label}: {option.name} - {option.description}"
if option.description
else f"{label}: {option.name}"
)
def prompt(self, question: Question) -> str:
"""The question as the model reads it, after the state."""
lines = [f"Question: {question.instructions}"] if question.instructions else []
for label, o in zip(question.labels, question.options):
lines.append(self.reply_line(label, o))
lines.append(self.reply_instruction)
return "\n".join(lines)
QUESTION_TYPES: dict[str, QuestionType] = {}
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
if cls.name in QUESTION_TYPES:
raise ValueError(f"question type {cls.name!r} is already registered")
QUESTION_TYPES[cls.name] = cls()
return cls
def get_question_type(name: str) -> QuestionType:
try:
return QUESTION_TYPES[name]
except KeyError:
raise StructuredDecisionError(
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
) from None
def build_question(
qid: str,
type_name: str,
instructions: Any,
criteria: Any,
max_options: int,
labeled: bool = True,
) -> Question:
if not qid:
raise StructuredDecisionError("question ids must be non-empty")
qtype = get_question_type(type_name)
options = qtype.parse_options(qid, criteria)
if not options:
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
names = [o.name for o in options]
if len(set(names)) != len(names):
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
limit = min(max_options, len(qtype.label_set)) if labeled else max_options
if len(options) > limit:
raise StructuredDecisionError(
f"question {qid!r}: at most {limit} options for this model"
)
if not isinstance(instructions, str):
instructions = "" if instructions is None else str(instructions)
return Question(
id=qid,
type=qtype,
instructions=instructions,
options=tuple(options),
labels=qtype.label_set[: len(options)],
)
def argmax(values: list[float]) -> int:
return max(range(len(values)), key=values.__getitem__)
def label_softmax(logprobs: list[float]) -> list[float]:
"""Softmax of the label logprobs."""
top = max(logprobs)
weights = [math.exp(lp - top) for lp in logprobs]
total = sum(weights)
return [w / total for w in weights]
@register_question_type
class ChoiceQuestion(QuestionType):
"""Pick one option. ``criteria`` maps each option name to a description,
or to null."""
name = "choice"
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, dict) or not criteria:
raise StructuredDecisionError(
f"question {qid!r}: choice criteria must map option names to "
"a description or null"
)
return [
Option(str(name), None if desc is None else str(desc))
for name, desc in criteria.items()
]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
"choice": question.options[top].name,
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
# The chosen label's probability over the whole vocabulary, so a
# read that mostly wanted a non-label token reports low confidence.
"confidence": probs[top] * label_mass,
}
@register_question_type
class NoulQuestion(QuestionType):
"""Yes or no. ``criteria`` may describe what true and false mean."""
name = "noul"
label_set = ("yes", "no")
reply_instruction = "Answer with yes or no only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if criteria is not None and (
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
):
raise StructuredDecisionError(
f"question {qid!r}: noul criteria must be an object with true and false"
)
criteria = criteria or {}
true = criteria.get("true")
false = criteria.get("false")
return [
Option("yes", None if true is None else str(true)),
Option("no", None if false is None else str(false)),
]
def reply_line(self, label: str, option: Option) -> str:
# A side without a description still shows its label. Without the
# label in the prompt, models answer "Yes" or "No" instead.
return f"{label}: {option.description}" if option.description else label
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
return {
"type": self.name,
"noul": probs[0],
"probabilities": {"yes": probs[0], "no": probs[1]},
# The chosen label's probability over the whole vocabulary, like a
# choice's confidence.
"confidence": max(probs) * label_mass,
}
@register_question_type
class ScoreQuestion(QuestionType):
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
level names, from the first to the last level of the scale."""
name = "score"
# 0-indexed like the answer, so a level's label is its score.
label_set = tuple(string.digits)
reply_instruction = "Answer with the number of one level only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, list) or len(criteria) < 2:
raise StructuredDecisionError(
f"question {qid!r}: score criteria must be an ordered list of levels"
)
return [Option(str(level)) for level in criteria]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
# The expected level, 0-indexed, so a level's index in the request's
# criteria is its value.
"score": sum(i * p for i, p in enumerate(probs)),
"legend": {str(i): o.name for i, o in enumerate(question.options)},
"probabilities": {str(i): p for i, p in enumerate(probs)},
"confidence": probs[top] * label_mass,
}
@@ -1,102 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from fastapi import Request
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from .adapters import input_text, make_answer, make_read_question
from .protocol import (
DecisionRequest,
DecisionResponse,
InputTokensDetails,
OpenAIDecisionUsage,
OutputTokensDetails,
)
from .question_types import StructuredDecisionError
from .strategies import ReadStrategy
class BaseServingDecisions(BaseServing):
def __init__(
self,
models: OpenAIServingModels,
strategy: ReadStrategy,
*,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=strategy.context.engine_client.model_config,
request_logger=request_logger,
)
self.strategy = strategy
self.limits = strategy.limits()
class OpenAIServingDecisions(BaseServingDecisions):
async def create_decisions(
self, request: DecisionRequest, raw_request: Request | None = None
) -> DecisionResponse | ErrorResponse:
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
return error
engine = self.strategy.context.engine_client
if engine.errored:
raise engine.dead_error
request_id = f"decision-{self._base_request_id(raw_request)}"
try:
if len(request.questions) > self.limits.max_questions:
raise StructuredDecisionError(
f"questions: at most {self.limits.max_questions} for this model"
)
questions = [
make_read_question(i, question, self.limits.max_options)
for i, question in enumerate(request.questions)
]
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine.check_admission(len(questions))
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
text = input_text(request.input)
self._log_inputs(request_id, text, None, lora_request)
reads = await self.strategy.read(
questions,
None,
text,
request_id=request_id,
chat_template_kwargs=None,
lora_request=lora_request,
priority=0,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
)
except StructuredDecisionError as exc:
return self.create_error_response(exc)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
input_tokens = sum(read.input_tokens for read in reads)
output_tokens = sum(read.output_tokens for read in reads)
return DecisionResponse(
model=self.models.model_name(lora_request),
answers=[
make_answer(question, read.probs, read.label_mass)
for question, read in zip(request.questions, reads)
],
usage=OpenAIDecisionUsage(
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens,
input_tokens_details=InputTokensDetails(
cached_tokens=sum(read.cached_tokens for read in reads),
cache_write_tokens=sum(read.cache_write_tokens for read in reads),
),
output_tokens_details=OutputTokensDetails(),
),
)
@@ -1,46 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from typing import Any
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.tasks import SupportedTask
from .serving import OpenAIServingDecisions
from .strategies import ReadContext, ReadStrategy, select_read_strategy
def init_decisions_state(
engine_client: EngineClient,
state: State,
args: Namespace,
request_logger: RequestLogger | None,
chat_template: str | None,
default_chat_template_kwargs: dict[str, Any],
supported_tasks: tuple[SupportedTask, ...],
) -> ReadStrategy | None:
state.openai_serving_decisions = None
try:
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
strategy = strategy_cls(
ReadContext(
engine_client=engine_client,
online_renderer=state.online_renderer,
chat_template=chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
)
)
except ValueError:
return None
if strategy.task == "generate":
state.openai_serving_decisions = OpenAIServingDecisions(
state.openai_serving_models,
strategy,
request_logger=request_logger,
)
return strategy
File diff suppressed because it is too large Load Diff
-37
View File
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import warnings
from prometheus_client import start_http_server
from vllm.logger import init_logger
from vllm.version import __version__ as VLLM_VERSION
warnings.warn(
"The `python -m vllm.entrypoints.openai.run_batch` command is deprecated "
"and may be removed in a future release. Please use `vllm run_batch` instead.",
DeprecationWarning,
stacklevel=1,
)
if __name__ == "__main__":
from vllm.entrypoints.launchers.run_batch import main, parse_args
logger = init_logger(__name__)
args = parse_args()
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
logger.info("args: %s", args)
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
# to publish metrics at the /metrics endpoint.
if args.enable_metrics:
logger.info("Prometheus metrics enabled")
start_http_server(port=args.port, addr=args.host)
else:
logger.info("Prometheus metrics disabled")
asyncio.run(main(args))
View File
-11
View File
@@ -1,11 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import FastAPI
def register_rl_api_routers(app: FastAPI):
from .online.api_router import router as rl_router
app.include_router(rl_router)
-68
View File
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdateRequest,
)
from vllm.entrypoints.common.offline import OfflineInferenceMixin
class RLOfflineMixin(OfflineInferenceMixin):
def init_weight_transfer_engine(
self, request: WeightTransferInitRequest | dict
) -> None:
"""Initialize weight transfer for RL training.
Args:
request: Weight transfer initialization request with backend-specific info
"""
init_info_dict = (
request["init_info"] if isinstance(request, dict) else request.init_info
)
self.llm_engine.collective_rpc(
"init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
)
def shutdown_weight_transfer_engine(self) -> None:
"""Release the trainer connection without shutting down inference."""
self.llm_engine.collective_rpc("shutdown_weight_transfer_engine")
def start_weight_update(self) -> None:
"""Start a new weight update."""
self.llm_engine.collective_rpc("start_weight_update")
def start_draft_weight_update(self) -> None:
"""Start a new weight update targeting the speculative draft model."""
self.llm_engine.collective_rpc("start_draft_weight_update")
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
"""Update the weights of the model.
Args:
request: Weight update request with backend-specific update info
"""
update_info_dict = (
request["update_info"] if isinstance(request, dict) else request.update_info
)
self.llm_engine.collective_rpc(
"update_weights", kwargs={"update_info": update_info_dict}
)
def finish_weight_update(self, weight_version: str | None = None) -> None:
"""Finish the weight update and set its version if provided."""
self.llm_engine.collective_rpc("finish_weight_update")
if weight_version is not None:
self.llm_engine.set_weight_version(weight_version)
def update_weight_version(self, new_version: str) -> None:
"""Set the weight version without updating weights."""
self.llm_engine.set_weight_version(new_version)
def get_weight_version(self) -> str:
"""Return the latest committed weight version."""
return self.llm_engine.get_weight_version()
-286
View File
@@ -1,286 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from http import HTTPStatus
from typing import Annotated, Any, Literal
from fastapi import APIRouter, Body, HTTPException, Query, Request
from fastapi.responses import JSONResponse
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdatePayload,
WeightTransferUpdateRequest,
)
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.rl.online.metrics import weight_operation_metrics
from vllm.logger import init_logger
from vllm.v1.engine import PauseMode
logger = init_logger(__name__)
def engine_client(request: Request) -> EngineClient:
return request.app.state.engine_client
router = APIRouter()
@router.post("/pause")
async def pause_generation(
raw_request: Request,
mode: Annotated[PauseMode, Query()] = "abort",
clear_cache: Annotated[bool, Query()] = True,
) -> JSONResponse:
"""Pause generation requests to allow weight updates.
Args:
raw_request: The incoming FastAPI request, used to reach the engine
client on the app state.
mode: How to handle in-flight requests:
- ``"abort"``: Abort all in-flight requests immediately (default).
- ``"wait"``: Wait for in-flight requests to complete.
- ``"keep"``: Freeze requests in queue; they resume on /resume.
clear_cache: DEPRECATED. Whether to clear KV/prefix caches after
draining. Ignored when mode="keep".
"""
engine = engine_client(raw_request)
try:
await engine.pause_generation(
mode=mode,
clear_cache=clear_cache,
)
return JSONResponse(
content={"status": "paused"},
status_code=HTTPStatus.OK.value,
)
except ValueError as err:
return JSONResponse(
content={"error": str(err)},
status_code=HTTPStatus.BAD_REQUEST.value,
)
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to pause generation")
return JSONResponse(
content={"error": f"Failed to pause generation: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
@router.post("/resume")
async def resume_generation(raw_request: Request) -> JSONResponse:
"""Resume generation after a pause."""
engine = engine_client(raw_request)
try:
await engine.resume_generation()
return JSONResponse(
content={"status": "resumed"},
status_code=HTTPStatus.OK.value,
)
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to resume generation")
return JSONResponse(
content={"error": f"Failed to resume generation: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
@router.post("/abort_requests")
async def abort_requests(raw_request: Request) -> JSONResponse:
"""Abort in-flight requests without pausing the scheduler.
Empty/missing ``request_ids`` aborts all in-flight requests.
"""
engine = engine_client(raw_request)
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
if not isinstance(body, dict):
raise HTTPException(
status_code=400, detail="Request body must be a JSON object"
)
request_ids = body.get("request_ids")
try:
if request_ids:
# Body ids are external (user-supplied) request ids.
await engine.abort(request_ids)
else:
# The dev RL server runs AsyncLLM; abort everything it is tracking.
# request_states is keyed by internal ids; parent_requests holds
# parallel-sampling parents. Abort both as internal ids.
from vllm.v1.engine.async_llm import AsyncLLM
assert isinstance(engine, AsyncLLM)
op = engine.output_processor
request_ids = [
*op.request_states.keys(),
*op.parent_requests.keys(),
]
await engine.abort(request_ids, internal=True)
return JSONResponse(
content={"status": "aborted", "aborted": len(request_ids)},
status_code=HTTPStatus.OK.value,
)
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to abort requests")
return JSONResponse(
content={"error": f"Failed to abort requests: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
@router.get("/is_paused")
async def is_paused(raw_request: Request) -> JSONResponse:
"""Return the current pause status."""
engine = engine_client(raw_request)
try:
paused = await engine.is_paused()
except Exception as err: # pragma: no cover - defensive
logger.exception("Failed to fetch pause status")
return JSONResponse(
content={"error": f"Failed to fetch pause status: {err}"},
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
)
return JSONResponse(content={"is_paused": paused})
@router.post("/init_weight_transfer_engine")
async def init_weight_transfer_engine(
raw_request: Request,
init_info: Annotated[dict[str, Any], Body(embed=True)],
):
with weight_operation_metrics().record("init"):
await engine_client(raw_request).init_weight_transfer_engine(
WeightTransferInitRequest(init_info=init_info)
)
return JSONResponse(content={"message": "Weight transfer initialized"})
@router.post("/shutdown_weight_transfer_engine")
async def shutdown_weight_transfer_engine(raw_request: Request):
with weight_operation_metrics().record("shutdown"):
await engine_client(raw_request).shutdown_weight_transfer_engine()
return JSONResponse(content={"message": "Weight transfer shut down"})
@router.post("/start_weight_update")
async def start_weight_update(raw_request: Request):
with weight_operation_metrics().record("start"):
await engine_client(raw_request).start_weight_update()
return JSONResponse(content={"message": "Weight update started"})
@router.post("/start_draft_weight_update")
async def start_draft_weight_update(raw_request: Request):
with weight_operation_metrics().record("start_draft"):
await engine_client(raw_request).start_draft_weight_update()
return JSONResponse(content={"message": "Draft weight update started"})
@router.post("/update_weights")
async def update_weights(
raw_request: Request,
update_info: Annotated[WeightTransferUpdatePayload, Body(embed=True)],
):
with weight_operation_metrics().record("update"):
await engine_client(raw_request).update_weights(
request=WeightTransferUpdateRequest(update_info=update_info)
)
return JSONResponse(content={"message": "Weights updated"})
@router.post("/finish_weight_update")
async def finish_weight_update(
raw_request: Request,
weight_version: Annotated[str | None, Body(embed=True)] = None,
):
with weight_operation_metrics().record("finish"):
await engine_client(raw_request).finish_weight_update(weight_version)
return JSONResponse(content={"message": "Weight update finished"})
@router.post("/update_weight_version")
async def update_weight_version(
raw_request: Request,
new_version: Annotated[str, Body(embed=True)],
):
await engine_client(raw_request).update_weight_version(new_version)
return JSONResponse(content={"success": True, "new_version": new_version})
@router.get("/weight_info")
async def weight_info(raw_request: Request):
weight_version = await engine_client(raw_request).get_weight_version()
return JSONResponse(content={"weight_version": weight_version})
@router.post("/weight_checker")
async def weight_checker(
raw_request: Request,
action: Annotated[Literal["checksum", "reset", "compare"], Body(embed=True)],
baseline: Annotated[dict[str, str] | None, Body(embed=True)] = None,
) -> JSONResponse:
"""Checksum, reset, or compare model weights against a baseline."""
client = engine_client(raw_request)
if action == "reset":
await client.collective_rpc("reset_weights")
return JSONResponse(content={"status": "reset"})
if action == "compare" and baseline is None:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="action='compare' requires a 'baseline' object",
)
checksums: dict[str, str] = {}
for worker_checksums in await client.compute_weight_checksums():
if duplicates := checksums.keys() & worker_checksums.keys():
raise HTTPException(
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
detail=f"Duplicate weight checksum keys: {sorted(duplicates)}",
)
checksums.update(worker_checksums)
if action == "checksum":
return JSONResponse(content={"checksums": checksums})
assert baseline is not None
mismatches = sorted(
key
for key in checksums.keys() | baseline.keys()
if checksums.get(key) != baseline.get(key)
)
return JSONResponse(content={"match": not mismatches, "mismatches": mismatches})
@router.get("/get_world_size")
async def get_world_size(
raw_request: Request,
include_dp: bool = Query(True),
):
"""Get the world size from the parallel config.
Args:
raw_request: The incoming FastAPI request, used to reach the engine
client on the app state.
include_dp: If True (default), returns the world size including
data parallelism (TP * PP * DP). If False, returns the world
size without data parallelism (TP * PP).
"""
parallel_config = engine_client(raw_request).vllm_config.parallel_config
if include_dp:
world_size = parallel_config.world_size_across_dp
else:
world_size = parallel_config.world_size
return JSONResponse(content={"world_size": world_size})
-56
View File
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Duration and in-flight metrics for RL weight-transfer operations."""
from collections.abc import Iterator
from contextlib import contextmanager
from threading import Lock
from time import perf_counter
from typing import Literal
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
Operation = Literal["init", "shutdown", "start", "start_draft", "update", "finish"]
class WeightOperationMetrics:
def __init__(self, registry: CollectorRegistry = REGISTRY):
self.duration = Histogram(
"vllm:rl_weight_update_operation_duration_seconds",
"Duration of one weight-transfer operation.",
["operation"],
buckets=(0.01, 0.1, 1, 10, 30, 60, 120, 300, 600),
registry=registry,
)
self.in_flight = Gauge(
"vllm:rl_weight_update_operations_in_flight",
"Weight-transfer operations currently awaited.",
["operation"],
multiprocess_mode="livesum",
registry=registry,
)
@contextmanager
def record(self, operation: Operation) -> Iterator[None]:
in_flight = self.in_flight.labels(operation)
in_flight.inc()
started = perf_counter()
try:
yield
finally:
in_flight.dec()
self.duration.labels(operation).observe(perf_counter() - started)
_metrics: WeightOperationMetrics | None = None
_metrics_lock = Lock()
def weight_operation_metrics() -> WeightOperationMetrics:
# Lazy, so the collectors exist only after multiprocess Prometheus setup.
global _metrics
if _metrics is None:
with _metrics_lock:
if _metrics is None:
_metrics = WeightOperationMetrics()
return _metrics
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import (
DerenderChatRequestUnion,
DerenderChatStreamRequest,
DerenderChatStreamResponse,
DerenderCompletionRequestUnion,
DerenderCompletionStreamRequest,
DerenderCompletionStreamResponse,
)
from .serving import ServingDerender
logger = init_logger(__name__)
router = APIRouter()
def derender(request: Request) -> ServingDerender | None:
return getattr(request.app.state, "serving_derender", None)
@router.post(
"/v1/chat/completions/derender",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_chat_completion(
request: DerenderChatRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a ChatCompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path; FastAPI validates and
routes on the ``stream`` discriminator.
Non-streaming: body is ``DerenderChatRequest`` (``generate_response`` with
the complete token list). Returns a ``ChatCompletionResponse``.
Streaming: body is ``DerenderChatStreamRequest`` (one ``generate_chunk``
delta + optional ``stream_state``). Returns a ``DerenderChatStreamResponse``
(``chunk`` + ``stream_state``). The client carries ``stream_state`` between
successive calls, one per SSE chunk from ``/inference/v1/generate``.
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Derender API"
)
if isinstance(request, DerenderChatStreamRequest):
stream_result = await handler.derender_chat_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderChatStreamResponse(chunk=chunk, stream_state=stream_state)
return JSONResponse(content=response.model_dump())
result = await handler.derender_chat_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/derender",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def derender_completion(
request: DerenderCompletionRequestUnion,
raw_request: Request,
):
"""Derender a generate response into a CompletionResponse.
Accepts both non-streaming (``stream=false``, default) and streaming
(``stream=true``) request bodies on the same path.
Non-streaming: body is ``DerenderCompletionRequest``. Returns a
``CompletionResponse``.
Streaming: body is ``DerenderCompletionStreamRequest`` (one
``generate_chunk`` + optional ``stream_state``). Returns a
``DerenderCompletionStreamResponse`` (``chunk`` + ``stream_state``).
"""
handler = derender(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Derender API")
if isinstance(request, DerenderCompletionStreamRequest):
stream_result = await handler.derender_completion_stream_response(request)
if isinstance(stream_result, ErrorResponse):
return JSONResponse(
content=stream_result.model_dump(),
status_code=stream_result.error.code,
)
chunk, stream_state = stream_result
response = DerenderCompletionStreamResponse(
chunk=chunk, stream_state=stream_state
)
return JSONResponse(content=response.model_dump())
result = await handler.derender_completion_response(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -1,434 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
import vllm.envs as envs
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
ChatCompletionStreamResponse,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
CompletionStreamResponse,
)
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.renderers.online_derenderer import OnlineDerenderer
from ..token_in_token_out.mm_features import extract_mm_features
from ..token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateTokensResponse,
MultiModalFeatures,
)
logger = init_logger(__name__)
class ServingDerender(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
online_derenderer: "OnlineDerenderer",
*,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=models.model_config,
request_logger=request_logger,
)
self.online_derenderer = online_derenderer
def _validate_derender_bounds(
self,
generate_responses: list[GenerateTokensResponse],
) -> ErrorResponse | None:
"""Reject derender payloads that exceed resource bounds.
Runs before any tokenizer.decode() or parser invocation to prevent
CPU/memory exhaustion from oversized caller-supplied token structures.
"""
max_n = envs.VLLM_MAX_N_SEQUENCES
max_model_len = self.model_config.max_model_len
# See ModelConfig.max_logprobs for semantics and default value.
max_logprobs = self.model_config.max_logprobs
if len(generate_responses) > max_n:
return self.create_error_response(
f"generate_responses count ({len(generate_responses)}) "
f"exceeds server maximum ({max_n}). "
f"Set VLLM_MAX_N_SEQUENCES to increase this limit."
)
for gen in generate_responses:
if len(gen.choices) > max_n:
return self.create_error_response(
f"choices count ({len(gen.choices)}) in response "
f"'{gen.request_id}' exceeds server maximum ({max_n})."
)
for choice in gen.choices:
if choice.token_ids and len(choice.token_ids) > max_model_len:
return self.create_error_response(
f"token_ids length ({len(choice.token_ids)}) in "
f"choice {choice.index} exceeds "
f"max_model_len ({max_model_len})."
)
if choice.logprobs and choice.logprobs.content:
if len(choice.logprobs.content) > max_model_len:
return self.create_error_response(
f"logprobs.content length "
f"({len(choice.logprobs.content)}) in "
f"choice {choice.index} exceeds "
f"max_model_len ({max_model_len})."
)
for entry in choice.logprobs.content:
if (
max_logprobs >= 0
and entry.top_logprobs
and len(entry.top_logprobs) > max_logprobs
):
return self.create_error_response(
f"top_logprobs count "
f"({len(entry.top_logprobs)}) in "
f"choice {choice.index} exceeds "
f"max_logprobs ({max_logprobs})."
)
if gen.prompt_logprobs and len(gen.prompt_logprobs) > max_model_len:
return self.create_error_response(
f"prompt_logprobs length ({len(gen.prompt_logprobs)}) "
f"in response '{gen.request_id}' exceeds "
f"max_model_len ({max_model_len})."
)
return None
def _validate_prompt_token_ids(
self,
prompt_token_ids: list[list[int] | None],
) -> ErrorResponse | None:
"""Reject caller-supplied prompt_token_ids longer than max_model_len."""
max_model_len = self.model_config.max_model_len
for ids in prompt_token_ids:
if ids is not None and len(ids) > max_model_len:
return self.create_error_response(
f"prompt_token_ids length ({len(ids)}) exceeds "
f"max_model_len ({max_model_len})."
)
return None
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
bounds_error = self._validate_derender_bounds([request.generate_response])
if bounds_error is not None:
return bounds_error
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
if prompt_ids_error is not None:
return prompt_ids_error
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response,
request.chat_request,
request.prompt_token_ids,
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
model_name = request.model or self.models.model_name()
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
model_name,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=model_name,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
metrics=gen.metrics,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
bounds_error = self._validate_derender_bounds(request.generate_responses)
if bounds_error is not None:
return bounds_error
if request.prompt_token_ids is not None:
prompt_ids_error = self._validate_prompt_token_ids(request.prompt_token_ids)
if prompt_ids_error is not None:
return prompt_ids_error
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses,
request.prompt_tokens,
completion_request=request.completion_request,
prompt_token_ids=request.prompt_token_ids,
)
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
model_name = request.model or self.models.model_name()
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
model_name,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=model_name,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
# Metrics describe one prompt. Parallel samples are already
# suppressed by /generate; multi-prompt responses cannot be merged.
metrics=first.metrics if len(request.generate_responses) == 1 else None,
)
async def derender_chat_stream_response(
self,
request: DerenderChatStreamRequest,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_chat_response``.
Processes one ``GenerateStreamResponse`` chunk and returns the
derendered chunk together with the updated client carried state.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
if (
self.online_derenderer.parser is not None
and request.prompt_token_ids is None
):
return self.create_error_response(
"prompt_token_ids is required when a tool or reasoning "
"parser is configured. Without it parse_delta cannot tell "
"whether the prompt left reasoning open (e.g. a chat "
"template that pre-opens <think>) and would misclassify "
"reasoning content as plain content."
)
# Each streamed chunk contains at most one choice per SSE event.
# A single DerenderStreamState is threaded
# through every choice in the chunk, so >1 would corrupt detok/parser
# state across choices. See the matching check in
# OnlineDerenderer.derender_chat_stream which this fails ahead of to
# avoid touching the tokenizer at all on a malformed chunk.
num_choices = len(request.generate_chunk.choices)
if num_choices > 1:
return self.create_error_response(
f"derender_chat_stream expects at most one choice per chunk "
f"(got {num_choices})."
)
# prompt_token_ids is folded in here too. It is caller supplied and
# otherwise unbounded. A parser configured deployment rescans it
# in full on every chunk (is_reasoning_end /
# adjust_initial_state_from_prompt, see abstract_parser.parse_delta)
# since parser state cannot be carried across calls.
prompt_len = (
len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0
)
prior_len = (
len(request.stream_state.output_token_ids)
if request.stream_state is not None
else 0
)
delta_len = sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
)
max_model_len = self.model_config.max_model_len
if prompt_len + prior_len + delta_len > max_model_len:
return self.create_error_response(
f"prompt_token_ids length ({prompt_len}) plus "
f"output_token_ids length ({prior_len}) plus delta "
f"({delta_len}) exceeds max_model_len ({max_model_len})."
)
model_name = request.model or self.models.model_name()
try:
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
model=model_name,
generate_chunk=request.generate_chunk,
state=request.stream_state,
chat_request=request.chat_request,
prompt_tokens=request.prompt_tokens,
prompt_token_ids=request.prompt_token_ids,
)
except Exception as exc:
# The two ValueErrors derender_chat_stream can raise directly
# (missing chat_request, >1 choice per chunk) are already
# pre-checked above, so anything reaching here comes from the
# parser replaying entirely client controlled state. Hermes
# swallows its own streaming exceptions but finalize_generation
# and engine based parsers don't, so an unexpected parser
# failure on malformed (but well typed) state must surface as
# a 400 here rather than an unhandled 500.
logger.debug(
"derender stream replay failed: %s: %s", type(exc).__name__, exc
)
return self.create_error_response("invalid stream_state: derender failed")
logger.debug(
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
model_name,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
async def derender_completion_stream_response(
self,
request: DerenderCompletionStreamRequest,
) -> tuple[CompletionStreamResponse, DerenderStreamState] | ErrorResponse:
"""Streaming counterpart to ``derender_completion_response``.
Processes one ``GenerateStreamResponse`` chunk (one output sequence's
delta) and returns the derendered chunk and updated state.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
if prompt_ids_error is not None:
return prompt_ids_error
model_name = request.model or self.models.model_name()
try:
(
chunk,
updated_state,
) = await self.online_derenderer.derender_completion_stream(
model=model_name,
generate_chunk=request.generate_chunk,
state=request.stream_state,
prompt_tokens=request.prompt_tokens,
completion_request=request.completion_request,
prompt_token_ids=request.prompt_token_ids,
)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc})"
)
logger.debug(
"derender_completion_stream request_id=%s model=%s delta_tokens=%d",
request.generate_chunk.request_id,
model_name,
sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
),
)
return chunk, updated_state
@staticmethod
def _extract_mm_features(
engine_input: EngineInput,
) -> MultiModalFeatures | None:
return extract_mm_features(engine_input)
+4 -38
View File
@@ -25,35 +25,12 @@ else:
logger = init_logger(__name__)
def init_render_state(
state: "State",
request_logger: RequestLogger | None,
):
from .derender.serving import ServingDerender
from .render.serving import ServingRender
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
tool_server=state.tool_server,
)
state.serving_derender = ServingDerender(
state.openai_serving_models,
state.online_derenderer,
request_logger=request_logger,
)
def init_scale_out_state(
state: "State",
args: "Namespace",
engine_client: "EngineClient",
request_logger: RequestLogger | None,
):
init_render_state(state, request_logger)
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
state.serving_tokens = ServingTokens(
@@ -72,14 +49,10 @@ def register_scale_out_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
from .derender.api_router import router as derender_render
from .render.api_router import router as render_render
args = getattr(app.state, "args", None)
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
# serving these endpoints, so they always register them regardless of
# `--enable-scale-out`.
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
# `vllm serve --tokens-only` is dedicated to serving these endpoints, so it
# always registers them regardless of `--enable-scale-out`.
dedicated_mode = getattr(args, "tokens_only", False)
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
if not enabled:
message = (
@@ -96,11 +69,7 @@ def register_scale_out_api_routers(
from .token_in_token_out.api_router import router as token_router
for route in (
*render_render.routes,
*derender_render.routes,
*token_router.routes,
):
for route in token_router.routes:
if isinstance(route, APIRoute):
app.add_api_route(
route.path,
@@ -110,9 +79,6 @@ def register_scale_out_api_routers(
)
return
app.include_router(render_render)
app.include_router(derender_render)
if "generate" in supported_tasks:
from .token_in_token_out.api_router import (
attach_router as attach_disagg_router,
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
from ..token_in_token_out.protocol import GenerateRequest
from .serving import ServingRender
logger = init_logger(__name__)
router = APIRouter()
def render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
@router.post(
"/v1/chat/completions/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support Chat Completions Render API"
)
result = await handler.render_chat_request(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/messages/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_messages(request: AnthropicMessagesRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Messages Render API")
result = await handler.render_messages_request(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@router.post(
"/v1/completions/render",
dependencies=[Depends(validate_json_request)],
response_model=list[GenerateRequest],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_completion(request: CompletionRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Completions Render API")
result = await handler.render_completion_request(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=[item.model_dump() for item in result])
@router.post(
"/v1/responses/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_responses(request: ResponsesRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Responses Render API")
result = await handler.render_responses_request(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -1,385 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
from fastapi import Request
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.generate.base.protocol import MMPayload
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
MultiModalFeatures,
ReasoningParserKwargs,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.multimodal.parse import MultiModalDataParser
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
extract_prompt_len,
)
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.entrypoints.mcp.tool_server import ToolServer
def _hoist_transfer_params(
params: SamplingParams,
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
"""Move KV/EC transfer params out of `extra_args`.
Render emits them only as top-level `GenerateRequest` fields. Generate
copies them back into `extra_args` for the engine.
"""
if not params.extra_args:
return None, None
# to_sampling_params aliases request.vllm_xargs, so copy before popping.
extra = dict(params.extra_args)
kv = extra.pop("kv_transfer_params", None)
ec = extra.pop("ec_transfer_params", None)
params.extra_args = extra or None
return kv, ec
class ServingRender(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
online_renderer: "OnlineRenderer",
*,
request_logger: RequestLogger | None = None,
tool_server: "ToolServer | None" = None,
) -> None:
super().__init__(
models=models,
model_config=online_renderer.model_config,
request_logger=request_logger,
)
self.online_renderer = online_renderer
self.tool_server = tool_server
self._merge_inline_system = (
AnthropicServingMessages._should_merge_inline_system(online_renderer)
)
self._placeholder_metadata_parser: MultiModalDataParser | None = None
self._placeholder_metadata_parser_failed = False
self.default_sampling_params = (
online_renderer.model_config.get_diff_sampling_param()
)
mc = online_renderer.model_config
self.override_max_tokens = (
self.default_sampling_params.get("max_tokens")
if mc.generation_config not in ("auto", "vllm")
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
async def render_chat_request(
self,
request: ChatCompletionRequest,
raw_request: Request | None = None,
) -> GenerateRequest | ErrorResponse:
"""Validate the model and preprocess a chat completion request.
This is the authoritative implementation used directly by the
GPU-less render server and delegated to by OpenAIServingChat.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
logger.error("Error with model %s", error_check_ret)
return error_check_ret
if request.use_beam_search:
return self.create_error_response(
"Beam search is not supported by the render endpoint"
)
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
if isinstance(result, ErrorResponse):
return result
_, engine_inputs = result
if len(engine_inputs) != 1:
return self.create_error_response(
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
)
engine_input = engine_inputs[0]
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_completion_tokens
if request.max_completion_tokens is not None
else request.max_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=request.truncate_prompt_tokens,
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
# Resolve here what OpenAIServingChat passes to the engine:
# `_grammar_from_parser` (set by `adjust_request`) is not serialized.
reasoning_parser_kwargs = self._reasoning_parser_kwargs(request)
parser = None
if reasoning_parser_kwargs is not None:
assert self.online_renderer.parser is not None
parser = self.online_renderer.parser(
self.online_renderer.renderer.get_tokenizer(),
request.tools,
chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs,
model_config=self.model_config,
)
reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
request_id = (
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
)
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
model=request.model,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs=reasoning_parser_kwargs,
stream=bool(request.stream),
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
token_offsets=engine_input.get("prompt_token_offsets"),
)
async def render_messages_request(
self,
request: AnthropicMessagesRequest,
raw_request: Request | None = None,
) -> GenerateRequest | ErrorResponse:
"""Validate the model and preprocess an Anthropic Messages request.
Converts the request to the OpenAI chat format using the same
conversion as the /v1/messages server path, then delegates to
render_chat_request so the rendered tokens match the server exactly.
"""
chat_req = AnthropicServingMessages.to_chat_completion_request(
request, merge_inline_system=self._merge_inline_system
)
chat_req.mm_payload = request.mm_payload
return await self.render_chat_request(chat_req, raw_request)
async def render_completion_request(
self,
request: CompletionRequest,
raw_request: Request | None = None,
) -> list[GenerateRequest] | ErrorResponse:
"""Validate the model and preprocess a completion request.
This is the authoritative implementation used directly by the
GPU-less render server and delegated to by OpenAIServingCompletion.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.use_beam_search:
return self.create_error_response(
"Beam search is not supported by the render endpoint"
)
result = await self.online_renderer.render_completion(
request, skip_mm_cache=True
)
if isinstance(result, ErrorResponse):
return result
base_id = f"cmpl-{self._base_request_id(raw_request, request.request_id)}"
generate_requests: list[GenerateRequest] = []
for i, engine_input in enumerate(result):
prompt_components = extract_prompt_components(
self.model_config, engine_input
)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
token_ids = list(token_ids)
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=request.truncate_prompt_tokens,
)
params = request.to_sampling_params(
max_tokens, self.default_sampling_params
)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
request_id = base_id if len(result) == 1 else f"{base_id}-{i}"
generate_requests.append(
GenerateRequest(
request_id=request_id,
token_ids=token_ids,
features=self._extract_mm_features(
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
model=request.model,
stream=bool(request.stream),
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
token_offsets=engine_input.get("prompt_token_offsets"),
)
)
return generate_requests
async def render_responses_request(
self,
request: ResponsesRequest,
) -> GenerateRequest | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.previous_response_id is not None:
return self.create_error_response(
message=(
"previous_response_id is not supported by the stateless "
"render endpoint."
),
err_type="invalid_request_error",
param="previous_response_id",
)
result = await self.online_renderer.render_responses(
request,
previous_messages=None,
previous_response_outputs=None,
tool_server=self.tool_server,
skip_mm_cache=True,
)
if isinstance(result, ErrorResponse):
return result
engine_input = result.engine_input
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_output_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
kv_transfer_params, ec_transfer_params = _hoist_transfer_params(params)
return GenerateRequest(
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(
engine_input, mm_payload=request.mm_payload
),
sampling_params=params,
model=request.model,
reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
stream=bool(request.stream),
cache_salt=request.cache_salt,
priority=request.priority,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
token_offsets=engine_input.get("prompt_token_offsets"),
)
def _reasoning_parser_kwargs(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ReasoningParserKwargs | None:
parser_cls = self.online_renderer.parser
if parser_cls is None or parser_cls.reasoning_parser_cls is None:
return None
return ReasoningParserKwargs(
chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs(
request
)
)
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Fields the EC consumer still requires after embeddings transfer."""
if self._placeholder_metadata_parser_failed:
return set()
if self._placeholder_metadata_parser is None:
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
self._placeholder_metadata_parser = (
MULTIMODAL_REGISTRY.create_processor(
self.model_config
).info.data_parser
)
except Exception:
logger.debug(
"Could not load placeholder metadata fields; "
"mm_metadata will use keep_on_cpu fields only."
)
self._placeholder_metadata_parser_failed = True
return set()
return set(
self._placeholder_metadata_parser.placeholder_metadata_fields(modality)
)
def _extract_mm_features(
self,
engine_input: EngineInput,
*,
mm_payload: MMPayload = "tensors",
) -> MultiModalFeatures | None:
return extract_mm_features(
engine_input,
metadata_fields_for=self._placeholder_metadata_fields,
mm_payload=mm_payload,
)
@@ -108,8 +108,8 @@ def attach_router(app: FastAPI):
app.include_router(router)
if not getattr(app.state.args, "tokens_only", False):
return
# The RLHF dev router registers its own /abort_requests first. Registering
# this one too would only add a shadowed route with a duplicate operation ID.
# An endpoint plugin may already serve /abort_requests. Registering this
# one too would only add a shadowed route with a duplicate operation ID.
has_abort_route = any(
getattr(route, "path", None) == "/abort_requests" for route in app.routes
)
@@ -685,7 +685,6 @@ class DerenderStreamState(BaseModel):
entire accumulated text on each invocation, `parse_delta()` itself is
O(n) yielding a true worst case compute cost of O(n³) per generation.
No caching is performed. Work is bounded by `max_model_len`.
See `OnlineDerenderer._derender_chat_stream_parsed`.
The detokenization strategy carries the incremental decode offsets
directly rather than re-sending the whole token history each chunk.
@@ -823,7 +822,7 @@ class DerenderChatStreamRequest(BaseModel):
`<think>`). `prompt_tokens` is a usage count and cannot serve this
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
Rejected with a 400 when a tool or reasoning
parser is configured and this is omitted. Without it, `parse_delta`
cannot tell whether the prompt left reasoning open and would silently
misclassify reasoning content as plain content. On all paths it also
@@ -1,118 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
logger = init_logger(__name__)
def engine_client(request: Request) -> EngineClient:
return request.app.state.engine_client
router = APIRouter()
@router.post(
"/scale_elastic_ep",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"model": dict},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def scale_elastic_ep(raw_request: Request):
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail="Invalid JSON format") from e
new_data_parallel_size = body.get("new_data_parallel_size")
drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
if new_data_parallel_size is None:
raise HTTPException(
status_code=400, detail="new_data_parallel_size is required"
)
if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
raise HTTPException(
status_code=400,
detail="new_data_parallel_size must be a positive integer",
)
if not isinstance(drain_timeout, int) or drain_timeout <= 0:
raise HTTPException(
status_code=400, detail="drain_timeout must be a positive integer"
)
client = engine_client(raw_request)
try:
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
return JSONResponse(
{
"message": f"Scaled to {new_data_parallel_size} data parallel engines",
}
)
except TimeoutError as e:
raise HTTPException(
status_code=408,
detail="Scale failed due to request drain timeout "
f"after {drain_timeout} seconds",
) from e
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
except Exception as e:
logger.error("Scale failed: %s", e)
raise HTTPException(status_code=500, detail="Scale failed") from e
@router.post("/is_scaling_elastic_ep")
async def is_scaling_elastic_ep(raw_request: Request):
# External operation status comes from the shared store. Middleware gating
# remains process-local, so scaling requests must reach every old-rank API.
try:
status = await engine_client(raw_request).get_external_elastic_ep_status()
except Exception as e:
logger.warning("Failed to query external Elastic EP status: %s", e)
raise HTTPException(
status_code=503,
detail="External Elastic EP status is temporarily unavailable",
) from e
if status is None:
# Non-external EEP modes retain the process-local middleware state.
is_scaling = get_scaling_elastic_ep()
status = {
"phase": (
ExternalElasticEPScalePhase.COMMITTING.value
if is_scaling
else ExternalElasticEPScalePhase.IDLE.value
),
"epoch": None,
"error": None,
"requested_data_parallel_size": None,
}
else:
is_scaling = status["phase"] in (
ExternalElasticEPScalePhase.PREPARING.value,
ExternalElasticEPScalePhase.COMMITTING.value,
)
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
def attach_router(app: FastAPI):
app.include_router(router)
@@ -16,7 +16,3 @@ def register_instrumentator_api_routers(app: FastAPI):
from .metrics import attach_router as metrics_attach_router
metrics_attach_router(app)
from .offline_docs import attach_router as offline_docs_attach_router
offline_docs_attach_router(app)
@@ -1,50 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline FastAPI documentation support for air-gapped environments."""
import pathlib
from fastapi import FastAPI
from fastapi.openapi.docs import (
get_swagger_ui_html,
get_swagger_ui_oauth2_redirect_html,
)
from fastapi.staticfiles import StaticFiles
from vllm.logger import init_logger
logger = init_logger(__name__)
def attach_router(app: FastAPI) -> None:
"""Attach offline docs router if enabled via args."""
args = getattr(app.state, "args", None)
if args is None or not getattr(args, "enable_offline_docs", False):
return
static_dir = pathlib.Path(__file__).parent / "static"
if not static_dir.exists():
logger.warning(
"Static directory not found at %s. Offline docs will not be available.",
static_dir,
)
return
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
@app.get("/docs", include_in_schema=False)
async def custom_swagger_ui_html():
return get_swagger_ui_html(
openapi_url=app.openapi_url,
title=app.title + " - Swagger UI",
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
swagger_js_url="/static/swagger-ui-bundle.js",
swagger_css_url="/static/swagger-ui.css",
)
@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
async def swagger_ui_redirect():
return get_swagger_ui_oauth2_redirect_html()
logger.info("Offline documentation enabled with vendored static assets")
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -8,7 +8,7 @@ from starlette.datastructures import Headers
from starlette.responses import JSONResponse
from starlette.types import ASGIApp, Receive, Scope, Send
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
class AuthenticationMiddleware:
@@ -1,4 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""SageMaker-specific integration for vLLM."""
@@ -1,156 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from typing import Any
import pydantic
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, Response
from vllm.config import ModelConfig
from vllm.entrypoints.generate.factories import get_generate_invocation_types
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.instrumentator.basic import base
from vllm.entrypoints.serve.instrumentator.health import health
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.tasks import SupportedTask
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
# (requires typing_extensions >= 4.13)
RequestType = Any
GetHandlerFn = Callable[[Request], BaseServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
"""Snapshot handler levels for loggers that may be affected by third-party
logging configuration side effects (e.g. model_hosting_container_standards
calling configure_root_logger() at import time)."""
loggers = [logging.getLogger(), logging.getLogger("vllm")]
seen: set[int] = set()
snapshot: list[tuple[logging.Handler, int]] = []
for logger in loggers:
for handler in logger.handlers:
if id(handler) not in seen:
seen.add(id(handler))
snapshot.append((handler, handler.level))
return snapshot
def _restore_handler_levels(
snapshot: list[tuple[logging.Handler, int]],
) -> None:
"""Restore handler levels from a snapshot."""
for handler, level in snapshot:
handler.setLevel(level)
def attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Attach the SageMaker hosting endpoints to the API server.
Handler levels are snapshotted and restored because importing
model_hosting_container_standards may reconfigure root logging.
"""
snapshot = _snapshot_handler_levels()
try:
_attach_router(app, supported_tasks, model_config)
finally:
_restore_handler_levels(snapshot)
def _attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Register the SageMaker hosting routes (/ping, /invocations) on the
app."""
import model_hosting_container_standards.sagemaker as sagemaker_standards
router = APIRouter()
# NOTE: Construct the TypeAdapters only once
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
INVOCATION_VALIDATORS = [
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
for request_type, (get_handler, endpoint) in INVOCATION_TYPES
]
@router.post("/ping", response_class=Response)
@router.get("/ping", response_class=Response)
@sagemaker_standards.register_ping_handler
async def ping(raw_request: Request) -> Response:
"""Ping check. Endpoint required for SageMaker."""
return await health(raw_request)
@router.post(
"/invocations",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@sagemaker_standards.register_invocation_handler
@sagemaker_standards.stateful_session_manager()
@sagemaker_standards.inject_adapter_id(adapter_path="model")
async def invocations(raw_request: Request):
"""For SageMaker, routes requests based on the request type."""
try:
body = await raw_request.json()
except json.JSONDecodeError as e:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"JSON decode error: {e}",
) from e
valid_endpoints = [
(validator, endpoint)
for validator, (get_handler, endpoint) in INVOCATION_VALIDATORS
if get_handler(raw_request) is not None
]
for request_validator, endpoint in valid_endpoints:
try:
request = request_validator.validate_python(body)
except pydantic.ValidationError:
continue
return await endpoint(request, raw_request)
type_names = [
t.__name__ if isinstance(t := validator._type, type) else str(t)
for validator, _ in valid_endpoints
]
msg = f"Cannot find suitable handler for request. Expected one of: {type_names}"
res = base(raw_request).create_error_response(message=msg)
return JSONResponse(content=res.model_dump(), status_code=res.error.code)
app.include_router(router)
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
"""Bootstrap the app with the SageMaker hosting standards.
Handler levels are restored right after the import because importing
model_hosting_container_standards may reconfigure root logging, and
bootstrap must run with the original levels.
"""
snapshot = _snapshot_handler_levels()
try:
import model_hosting_container_standards.sagemaker as sagemaker_standards
app = sagemaker_standards.bootstrap(app)
finally:
_restore_handler_levels(snapshot)
return app
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ASGI middleware for WebSocket Prometheus metrics.
Modeled after prometheus-fastapi-instrumentator, this middleware
transparently instruments WebSocket endpoints with standard metrics
without requiring changes to handler code.
NOTE: This module intentionally has zero vllm imports so that it can
be extracted into a standalone package (similar to
prometheus-fastapi-instrumentator) in the future. Please keep it that way.
"""
import time
from collections.abc import Awaitable
from prometheus_client import Counter, Gauge, Histogram
from starlette.types import ASGIApp, Message, Receive, Scope, Send
# Standard WebSocket metric names (not vllm-specific, following
# the same convention as prometheus-fastapi-instrumentator).
_active_sessions = Gauge(
name="vllm:websocket_connections_active",
documentation="Number of currently active WebSocket connections.",
multiprocess_mode="livesum",
)
_total_sessions = Counter(
name="vllm:websocket_connections_total",
documentation="Total number of WebSocket connections.",
)
_session_duration = Histogram(
name="vllm:websocket_connection_duration_seconds",
documentation="Duration of WebSocket connections in seconds.",
buckets=[0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1800],
)
class WebSocketMetricsMiddleware:
"""Pure ASGI middleware that instruments WebSocket connections.
Tracks active connections (gauge), total connections (counter),
and connection duration (histogram) for all WebSocket endpoints.
Usage::
app.add_middleware(WebSocketMetricsMiddleware)
"""
def __init__(self, app: ASGIApp) -> None:
self.app = app
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
if scope["type"] != "websocket":
return self.app(scope, receive, send)
return self._handle_websocket(scope, receive, send)
async def _handle_websocket(
self, scope: Scope, receive: Receive, send: Send
) -> None:
start_time: float | None = None
async def send_wrapper(message: Message) -> None:
nonlocal start_time
if message["type"] == "websocket.accept":
start_time = time.monotonic()
_active_sessions.inc()
_total_sessions.inc()
await send(message)
try:
await self.app(scope, receive, send_wrapper)
finally:
if start_time is not None:
_active_sessions.dec()
_session_duration.observe(time.monotonic() - start_time)