A symbol scan over every tracked file found 123 top-level definitions in vllm/ with zero references, including references inside strings. All of them came from features this fork deleted earlier. - Delete dead attention helpers: MLADCPManager, split_prefill_chunks, the reshape helpers, compute_varlen_chunk_metadata, the LINEAR backend's unused BailingLinearAttentionBackend, and the sparse flash-attn entry points. - Delete the CPU and AMX op wrappers from _custom_ops.py: cpu_mla_decode, cpu_mla_extend, cpu_mla_get_scheduler_metadata, gather_mla_context_cache_cpu. - Delete dead quantization helpers: awq_pack, gptq_pack, gptq_quantize_weights, get_and_maybe_dequant_weights, marlin_quant_fp8_torch, the rand_marlin_weight_*_like pair, ocp_mx_weight_dtype_and_rows, get_fp8_block_weight_scale, input_to_float8. - Delete dead model helpers: resolve_visual_encoder_outputs, run_dp_sharded_vision_model, fast_topk, get_layer_index, is_pp_missing_parameter, skip_spec_layers, KimiK3MergedQKVGateLinear, selective_scan_fn, causal_conv1d_update_torch. - Delete dead utils: format_kib, format_mib, chunk_list, prev_power_of_2, split_host_port, join_host_port, make_zmq_path, make_async_with_semaphore, is_meta_module, is_non_overlapping_and_dense, supports_xpu_graph, create_kv_caches_with_random*, and the unused has_* capability probes. - Delete the unreachable deep-gemm mega-MoE warmup path, which imported vllm.models.deepseek_v4.nvidia.model and vllm.models.kimi_k3.nvidia.model, neither of which exists anymore. - Drop the imports and the two now-empty guards left behind. Verified statically: every changed file compiles, no unused import remains, and no deleted name is referenced in vllm, tests or csrc. The runtime gates still need the Python environment rebuilt. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
421 lines
19 KiB
Python
421 lines
19 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""This file contains the command line arguments for the vLLM's online server.
|
|
It is kept in a separate file for documentation purposes.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import ssl
|
|
from dataclasses import field
|
|
from typing import Any, Literal
|
|
|
|
from vllm.config import config
|
|
from vllm.engine.arg_utils import AsyncEngineArgs
|
|
from vllm.logger import init_logger
|
|
from vllm.renderers.chat_utils import (
|
|
ChatTemplateContentFormatOption,
|
|
validate_chat_template,
|
|
)
|
|
from vllm.tool_parsers import ToolParserManager
|
|
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
|
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
|
|
|
from .utils.constants import (
|
|
H11_MAX_HEADER_COUNT_DEFAULT,
|
|
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
@config
|
|
class BaseFrontendArgs:
|
|
"""Base arguments for the OpenAI-compatible frontend server.
|
|
|
|
This base class does not include host, port, and server-specific arguments
|
|
like SSL, CORS, and HTTP server settings. Those arguments are added by
|
|
the subclasses.
|
|
"""
|
|
|
|
chat_template: str | None = None
|
|
"""The file path to the chat template, or the template in single-line form
|
|
for the specified model."""
|
|
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
|
|
"""The format to render message content within a chat template.
|
|
|
|
* "string" will render the content as a string. Example: `"Hello World"`
|
|
* "openai" will render the content as a list of dictionaries, similar to
|
|
OpenAI schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
|
|
trust_request_chat_template: bool = False
|
|
"""Whether to trust the chat template provided in the request. If False,
|
|
the server will always use the chat template specified by `--chat-template`
|
|
or the ones from tokenizer."""
|
|
trust_request_mm_kwargs: bool = False
|
|
"""Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
|
|
and `media_io_kwargs`). If False, the server rejects non-empty values
|
|
because they can change multimodal preprocessing resource usage. Only
|
|
enable this when API clients are trusted."""
|
|
default_chat_template_kwargs: dict[str, Any] | None = None
|
|
"""Default keyword arguments to pass to the chat template renderer.
|
|
These will be merged with request-level chat_template_kwargs,
|
|
with request values taking precedence. Useful for setting default
|
|
behavior for reasoning models. Example: '{"enable_thinking": false}'
|
|
to disable thinking mode by default for Qwen3/DeepSeek models."""
|
|
response_role: str = "assistant"
|
|
"""The role name to return if `request.add_generation_prompt=true`."""
|
|
return_tokens_as_token_ids: bool = False
|
|
"""When `--max-logprobs` is specified, represents single tokens as
|
|
strings of the form 'token_id:{token_id}' so that tokens that are not
|
|
JSON-encodable can be identified."""
|
|
enable_auto_tool_choice: bool = False
|
|
"""Enable auto tool choice for supported models. Use `--tool-call-parser`
|
|
to specify which parser to use."""
|
|
exclude_tools_when_tool_choice_none: bool = False
|
|
"""If specified, exclude tool definitions in prompts when
|
|
tool_choice='none'."""
|
|
tool_call_parser: str | None = None
|
|
"""Select the tool call parser depending on the model that you're using.
|
|
This is used to parse the model-generated tool call into OpenAI API format.
|
|
Required for `--enable-auto-tool-choice`. You can choose any option from
|
|
the built-in parsers or register a plugin via `--tool-parser-plugin`."""
|
|
tool_parser_plugin: str = ""
|
|
"""Special the tool parser plugin write to parse the model-generated tool
|
|
into OpenAI API format, the name register in this plugin can be used in
|
|
`--tool-call-parser`."""
|
|
tool_strict_level: ToolStrictLevelName | None = None
|
|
"""Server-side floor for structural-tag based tool calling, applied on top
|
|
of the per-tool `strict` field. `auto` follows the request's tool choice
|
|
and per-tool strictness; `function` constrains the
|
|
tool-call envelope (markup and function name) for every request with
|
|
tools; `parameter` additionally pins argument schemas, as if every tool
|
|
were `strict: true`. When unset, the tool parser's default level applies,
|
|
which is `auto` for most parsers."""
|
|
tool_server: str | None = None
|
|
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
|
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
|
built-in demo tools (browser and Python code interpreter). WARNING:
|
|
The `demo` Python tool executes model-generated code in Docker without
|
|
network isolation by default. See the security guide for more
|
|
information."""
|
|
max_log_len: int | None = None
|
|
"""Max number of prompt characters or prompt ID numbers being printed in
|
|
log. The default of None means unlimited."""
|
|
enable_prompt_tokens_details: bool = False
|
|
"""If set to True, enable prompt_tokens_details in usage."""
|
|
enable_per_request_metrics: bool = False
|
|
"""If set to True, include per-request timing metrics in API responses."""
|
|
enable_server_load_tracking: bool = False
|
|
"""If set to True, enable tracking server_load_metrics in the app state."""
|
|
enable_force_include_usage: bool = False
|
|
"""If set to True, including usage on every request."""
|
|
sse_keep_alive_interval: int = 0
|
|
"""Send an SSE keep-alive comment line every this many seconds when a
|
|
`/v1/chat/completions` or `/v1/completions` streaming response is idle
|
|
(queued, prefill, or between tokens), to prevent reverse proxies/tunnels
|
|
with read timeouts from closing the connection. Defaults to 0, which
|
|
disables keep-alive comments entirely.
|
|
"""
|
|
enable_tokenizer_info_endpoint: bool = False
|
|
"""Enable the `/tokenizer_info` endpoint. May expose chat
|
|
templates and other tokenizer configuration."""
|
|
enable_log_outputs: bool = False
|
|
"""If set to True, log model outputs (generations). Requires
|
|
`--enable-log-requests`. Output text and finish reasons are logged at INFO,
|
|
while output token IDs are logged at DEBUG."""
|
|
enable_log_deltas: bool = True
|
|
"""If set to False, output deltas will not be logged. Relevant only if
|
|
--enable-log-outputs is set.
|
|
"""
|
|
log_error_stack: bool = False
|
|
"""If set to True, log the stack trace of error responses"""
|
|
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
|
"""Controls the ``system_fingerprint`` field on responses.
|
|
|
|
- ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes
|
|
server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an
|
|
8-char config hash.
|
|
- ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped.
|
|
- ``custom``: emits the literal string from ``--fingerprint-value``.
|
|
- ``none``: the field is omitted (serialized as ``null``).
|
|
"""
|
|
fingerprint_value: str | None = None
|
|
"""Literal fingerprint string used when ``--fingerprint-mode=custom``."""
|
|
|
|
@classmethod
|
|
def _customize_cli_kwargs(
|
|
cls,
|
|
frontend_kwargs: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Customize argparse kwargs before arguments are registered.
|
|
|
|
Subclasses should override this and call
|
|
``super()._customize_cli_kwargs(frontend_kwargs)`` first.
|
|
"""
|
|
# Special case: default_chat_template_kwargs needs json.loads type
|
|
frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
|
|
|
|
# Special case: Tool call parser shows built-in options.
|
|
valid_tool_parsers = list(ToolParserManager.list_registered())
|
|
parsers_str = ",".join(valid_tool_parsers)
|
|
frontend_kwargs["tool_call_parser"]["metavar"] = (
|
|
f"{{{parsers_str}}} or name registered in --tool-parser-plugin"
|
|
)
|
|
return frontend_kwargs
|
|
|
|
@classmethod
|
|
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
|
"""Register CLI arguments for this frontend class.
|
|
|
|
Subclasses should override ``_customize_cli_kwargs`` instead of
|
|
this method so that base-class postprocessing is always applied.
|
|
"""
|
|
from vllm.engine.arg_utils import get_kwargs
|
|
|
|
frontend_kwargs = get_kwargs(cls)
|
|
frontend_kwargs = cls._customize_cli_kwargs(frontend_kwargs)
|
|
|
|
group_name = cls.__name__.replace("Args", "")
|
|
frontend_group = parser.add_argument_group(
|
|
title=group_name,
|
|
description=cls.__doc__,
|
|
)
|
|
for key, value in frontend_kwargs.items():
|
|
extra_flags = value.pop("flags", [])
|
|
frontend_group.add_argument(
|
|
*extra_flags, f"--{key.replace('_', '-')}", **value
|
|
)
|
|
|
|
return parser
|
|
|
|
|
|
@config
|
|
class FrontendArgs(BaseFrontendArgs):
|
|
"""Arguments for the OpenAI-compatible frontend server."""
|
|
|
|
host: str | None = None
|
|
"""Host name."""
|
|
port: int = 8000
|
|
"""Port number."""
|
|
data_parallel_supervisor_port: int = 9256
|
|
"""HTTP port for aggregated health endpoints in multi-port external LB
|
|
mode."""
|
|
dp_supervisor_probe_interval_s: float = 5.0
|
|
"""Seconds between aggregated health probes in multi-port external LB mode."""
|
|
dp_supervisor_probe_timeout_s: float = 5.0
|
|
"""Seconds to wait between retries when a child health probe fails with a
|
|
connection error in multi-port external LB mode."""
|
|
dp_supervisor_probe_failure_threshold: int = 3
|
|
"""Number of consecutive connection-error retries before a child health
|
|
probe is declared failed in multi-port external LB mode."""
|
|
uds: str | None = None
|
|
"""Unix domain socket path. If set, host and port arguments are ignored."""
|
|
uvicorn_log_level: Literal[
|
|
"critical", "error", "warning", "info", "debug", "trace"
|
|
] = "info"
|
|
"""Log level for uvicorn."""
|
|
disable_uvicorn_access_log: bool = False
|
|
"""Disable uvicorn access log."""
|
|
disable_access_log_for_endpoints: str | None = None
|
|
"""Comma-separated list of endpoint paths to exclude from uvicorn access
|
|
logs. This is useful to reduce log noise from high-frequency endpoints
|
|
like health checks. Example: "/health,/metrics,/ping".
|
|
When set, access logs for requests to these paths will be suppressed
|
|
while keeping logs for other endpoints."""
|
|
allow_credentials: bool = False
|
|
"""Allow credentials."""
|
|
allowed_origins: list[str] = field(default_factory=list)
|
|
"""Allowed origins. Cross-origin browser access is disabled by default."""
|
|
allowed_methods: list[str] = field(default_factory=lambda: ["*"])
|
|
"""Allowed methods."""
|
|
allowed_headers: list[str] = field(default_factory=lambda: ["*"])
|
|
"""Allowed headers."""
|
|
api_key: list[str] | None = None
|
|
"""If provided, the server will require one of these keys to be presented in
|
|
the header.
|
|
|
|
Warning: this only authenticates endpoints under the `/v1`, `/v2`, and
|
|
`/inference` path prefixes. Other endpoints on the same server, including
|
|
`/invocations` (which exposes the same inference capabilities as `/v1`),
|
|
remain unauthenticated. Do not rely on `--api-key` alone to secure vLLM;
|
|
see
|
|
https://docs.vllm.ai/en/latest/usage/security.html#api-key-authentication-limitations
|
|
for what it does and does not protect."""
|
|
ssl_keyfile: str | None = None
|
|
"""The file path to the SSL key file."""
|
|
ssl_certfile: str | None = None
|
|
"""The file path to the SSL cert file."""
|
|
ssl_ca_certs: str | None = None
|
|
"""The CA certificates file."""
|
|
enable_ssl_refresh: bool = False
|
|
"""Refresh SSL Context when SSL certificate files change"""
|
|
ssl_cert_reqs: int = int(ssl.CERT_NONE)
|
|
"""Whether client certificate is required (see stdlib ssl module's)."""
|
|
ssl_ciphers: str | None = None
|
|
"""SSL cipher suites for HTTPS (TLS 1.2 and below only).
|
|
Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'"""
|
|
root_path: str | None = None
|
|
"""FastAPI root_path when app is behind a path based routing proxy."""
|
|
middleware: list[str] = field(default_factory=lambda: [])
|
|
"""Additional ASGI middleware to apply to the app. We accept multiple
|
|
--middleware arguments. The value should be an import path. If a function
|
|
is provided, vLLM will add it to the server using
|
|
`@app.middleware('http')`. If a class is provided, vLLM will
|
|
add it to the server using `app.add_middleware()`."""
|
|
enable_request_id_headers: bool = False
|
|
"""If specified, API server will add X-Request-Id header to responses."""
|
|
disable_fastapi_docs: bool = False
|
|
"""Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""
|
|
h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
|
|
"""Maximum size (bytes) of an incomplete HTTP event (header or body) for
|
|
h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB)."""
|
|
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
|
|
"""Maximum number of HTTP headers allowed in a request for h11 parser.
|
|
Helps mitigate header abuse. Default: 256."""
|
|
enable_flash_late_interaction: bool = True
|
|
"""If set, run pooling score MaxSim on GPU in the API server process.
|
|
Can significantly improve late-interaction scoring performance.
|
|
When disabled, the setting is also propagated to the engine's
|
|
`PoolerConfig`, so the engine-side scorer uses the reference MaxSim
|
|
path instead of the fused Triton kernel."""
|
|
|
|
@classmethod
|
|
def _customize_cli_kwargs(
|
|
cls,
|
|
frontend_kwargs: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
|
|
|
|
# Special case: allowed_origins, allowed_methods, allowed_headers all
|
|
# need json.loads type
|
|
# Should also remove nargs
|
|
frontend_kwargs["allowed_origins"]["type"] = json.loads
|
|
frontend_kwargs["allowed_methods"]["type"] = json.loads
|
|
frontend_kwargs["allowed_headers"]["type"] = json.loads
|
|
del frontend_kwargs["allowed_origins"]["nargs"]
|
|
del frontend_kwargs["allowed_methods"]["nargs"]
|
|
del frontend_kwargs["allowed_headers"]["nargs"]
|
|
|
|
# Special case: Middleware needs to append action
|
|
frontend_kwargs["middleware"]["action"] = "append"
|
|
frontend_kwargs["middleware"]["type"] = str
|
|
if "nargs" in frontend_kwargs["middleware"]:
|
|
del frontend_kwargs["middleware"]["nargs"]
|
|
frontend_kwargs["middleware"]["default"] = []
|
|
|
|
# Special case: disable_access_log_for_endpoints is a single
|
|
# comma-separated string, not a list
|
|
if "nargs" in frontend_kwargs["disable_access_log_for_endpoints"]:
|
|
del frontend_kwargs["disable_access_log_for_endpoints"]["nargs"]
|
|
|
|
return frontend_kwargs
|
|
|
|
|
|
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
|
"""Create the CLI argument parser used by the OpenAI API server.
|
|
|
|
We rely on the helper methods of `FrontendArgs` and `AsyncEngineArgs` to
|
|
register all arguments instead of manually enumerating them here. This
|
|
avoids code duplication and keeps the argument definitions in one place.
|
|
"""
|
|
parser.add_argument(
|
|
"model_tag",
|
|
type=str,
|
|
nargs="?",
|
|
help="The model tag to serve (optional if specified in config)",
|
|
)
|
|
parser.add_argument(
|
|
"--headless",
|
|
action="store_true",
|
|
default=False,
|
|
help="Run in headless mode. See multi-node data parallel "
|
|
"documentation for more details.",
|
|
)
|
|
parser.add_argument(
|
|
"--api-server-count",
|
|
"-asc",
|
|
type=int,
|
|
default=None,
|
|
help="How many API server processes to run. "
|
|
"Defaults to data_parallel_size if not specified.",
|
|
)
|
|
parser.add_argument(
|
|
"--config",
|
|
help="Read CLI options from a config file. "
|
|
"Must be a YAML with the following options: "
|
|
"https://docs.vllm.ai/en/latest/configuration/serve_args.html",
|
|
)
|
|
parser.add_argument(
|
|
"--grpc",
|
|
action="store_true",
|
|
default=False,
|
|
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
|
|
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc].",
|
|
)
|
|
parser = FrontendArgs.add_cli_args(parser)
|
|
parser = AsyncEngineArgs.add_cli_args(parser)
|
|
|
|
return parser
|
|
|
|
|
|
def validate_parsed_serve_args(args: argparse.Namespace):
|
|
"""Quick checks for model serve args that raise prior to loading."""
|
|
if hasattr(args, "subparser") and args.subparser != "serve":
|
|
return
|
|
|
|
# Ensure that the chat template is valid; raises if it likely isn't
|
|
validate_chat_template(args.chat_template)
|
|
if args.chat_template is not None and "hf" in (
|
|
args.tool_call_parser,
|
|
getattr(args, "reasoning_parser", None),
|
|
):
|
|
logger.warning(
|
|
"--chat-template is set; the hf parser still expects "
|
|
"the checkpoint's output format."
|
|
)
|
|
|
|
# Enable auto tool needs a tool call parser to be valid
|
|
if args.enable_auto_tool_choice and not args.tool_call_parser:
|
|
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
|
|
if args.enable_log_outputs and not args.enable_log_requests:
|
|
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
|
|
|
|
# SSE keep-alive interval must be zero (disabled) or a positive integer.
|
|
if getattr(args, "sse_keep_alive_interval", 0) < 0:
|
|
raise ValueError(
|
|
"Error: --sse-keep-alive-interval must be a non-negative integer "
|
|
"(0 disables keep-alive comments)."
|
|
)
|
|
|
|
if getattr(args, "enable_per_request_metrics", False) and getattr(
|
|
args, "disable_log_stats", False
|
|
):
|
|
raise ValueError(
|
|
"Error: --enable-per-request-metrics requires engine statistics "
|
|
"logging; remove --disable-log-stats to enable per-request metrics."
|
|
)
|
|
|
|
if args.data_parallel_multi_port_external_lb:
|
|
from vllm.entrypoints.launchers.dp_supervisor import (
|
|
validate_multi_port_external_lb_args,
|
|
)
|
|
|
|
validate_multi_port_external_lb_args(args)
|
|
|
|
|
|
def propagate_flash_late_interaction(args, engine_args) -> None:
|
|
"""Propagate `--disable-flash-late-interaction` into the engine config.
|
|
|
|
The frontend flag alone only disables the API-server scoring path;
|
|
mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the
|
|
engine-side scorer fall back to the reference MaxSim path too.
|
|
"""
|
|
if getattr(args, "enable_flash_late_interaction", True):
|
|
return
|
|
from vllm.config.pooler import PoolerConfig
|
|
|
|
if engine_args.pooler_config is None:
|
|
engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False)
|
|
else:
|
|
engine_args.pooler_config.enable_flash_late_interaction = False
|