Files
MOVZXandHinata 61fbe0b865 [Mini-VLLM] Remove unreferenced definitions found by static scan
A symbol scan over every tracked file found 123 top-level definitions in
vllm/ with zero references, including references inside strings. All of
them came from features this fork deleted earlier.

- Delete dead attention helpers: MLADCPManager, split_prefill_chunks, the
  reshape helpers, compute_varlen_chunk_metadata, the LINEAR backend's
  unused BailingLinearAttentionBackend, and the sparse flash-attn entry
  points.
- Delete the CPU and AMX op wrappers from _custom_ops.py: cpu_mla_decode,
  cpu_mla_extend, cpu_mla_get_scheduler_metadata,
  gather_mla_context_cache_cpu.
- Delete dead quantization helpers: awq_pack, gptq_pack,
  gptq_quantize_weights, get_and_maybe_dequant_weights,
  marlin_quant_fp8_torch, the rand_marlin_weight_*_like pair,
  ocp_mx_weight_dtype_and_rows, get_fp8_block_weight_scale,
  input_to_float8.
- Delete dead model helpers: resolve_visual_encoder_outputs,
  run_dp_sharded_vision_model, fast_topk, get_layer_index,
  is_pp_missing_parameter, skip_spec_layers, KimiK3MergedQKVGateLinear,
  selective_scan_fn, causal_conv1d_update_torch.
- Delete dead utils: format_kib, format_mib, chunk_list, prev_power_of_2,
  split_host_port, join_host_port, make_zmq_path, make_async_with_semaphore,
  is_meta_module, is_non_overlapping_and_dense, supports_xpu_graph,
  create_kv_caches_with_random*, and the unused has_* capability probes.
- Delete the unreachable deep-gemm mega-MoE warmup path, which imported
  vllm.models.deepseek_v4.nvidia.model and vllm.models.kimi_k3.nvidia.model,
  neither of which exists anymore.
- Drop the imports and the two now-empty guards left behind.

Verified statically: every changed file compiles, no unused import
remains, and no deleted name is referenced in vllm, tests or csrc. The
runtime gates still need the Python environment rebuilt.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-19 21:39:56 +07:00

421 lines
19 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This file contains the command line arguments for the vLLM's online server.
It is kept in a separate file for documentation purposes.
"""
import argparse
import json
import ssl
from dataclasses import field
from typing import Any, Literal
from vllm.config import config
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.logger import init_logger
from vllm.renderers.chat_utils import (
ChatTemplateContentFormatOption,
validate_chat_template,
)
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
from vllm.utils.argparse_utils import FlexibleArgumentParser
from .utils.constants import (
H11_MAX_HEADER_COUNT_DEFAULT,
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
)
logger = init_logger(__name__)
@config
class BaseFrontendArgs:
"""Base arguments for the OpenAI-compatible frontend server.
This base class does not include host, port, and server-specific arguments
like SSL, CORS, and HTTP server settings. Those arguments are added by
the subclasses.
"""
chat_template: str | None = None
"""The file path to the chat template, or the template in single-line form
for the specified model."""
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
"""The format to render message content within a chat template.
* "string" will render the content as a string. Example: `"Hello World"`
* "openai" will render the content as a list of dictionaries, similar to
OpenAI schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
trust_request_chat_template: bool = False
"""Whether to trust the chat template provided in the request. If False,
the server will always use the chat template specified by `--chat-template`
or the ones from tokenizer."""
trust_request_mm_kwargs: bool = False
"""Whether to trust per-request multimodal kwargs (`mm_processor_kwargs`
and `media_io_kwargs`). If False, the server rejects non-empty values
because they can change multimodal preprocessing resource usage. Only
enable this when API clients are trusted."""
default_chat_template_kwargs: dict[str, Any] | None = None
"""Default keyword arguments to pass to the chat template renderer.
These will be merged with request-level chat_template_kwargs,
with request values taking precedence. Useful for setting default
behavior for reasoning models. Example: '{"enable_thinking": false}'
to disable thinking mode by default for Qwen3/DeepSeek models."""
response_role: str = "assistant"
"""The role name to return if `request.add_generation_prompt=true`."""
return_tokens_as_token_ids: bool = False
"""When `--max-logprobs` is specified, represents single tokens as
strings of the form 'token_id:{token_id}' so that tokens that are not
JSON-encodable can be identified."""
enable_auto_tool_choice: bool = False
"""Enable auto tool choice for supported models. Use `--tool-call-parser`
to specify which parser to use."""
exclude_tools_when_tool_choice_none: bool = False
"""If specified, exclude tool definitions in prompts when
tool_choice='none'."""
tool_call_parser: str | None = None
"""Select the tool call parser depending on the model that you're using.
This is used to parse the model-generated tool call into OpenAI API format.
Required for `--enable-auto-tool-choice`. You can choose any option from
the built-in parsers or register a plugin via `--tool-parser-plugin`."""
tool_parser_plugin: str = ""
"""Special the tool parser plugin write to parse the model-generated tool
into OpenAI API format, the name register in this plugin can be used in
`--tool-call-parser`."""
tool_strict_level: ToolStrictLevelName | None = None
"""Server-side floor for structural-tag based tool calling, applied on top
of the per-tool `strict` field. `auto` follows the request's tool choice
and per-tool strictness; `function` constrains the
tool-call envelope (markup and function name) for every request with
tools; `parameter` additionally pins argument schemas, as if every tool
were `strict: true`. When unset, the tool parser's default level applies,
which is `auto` for most parsers."""
tool_server: str | None = None
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
built-in demo tools (browser and Python code interpreter). WARNING:
The `demo` Python tool executes model-generated code in Docker without
network isolation by default. See the security guide for more
information."""
max_log_len: int | None = None
"""Max number of prompt characters or prompt ID numbers being printed in
log. The default of None means unlimited."""
enable_prompt_tokens_details: bool = False
"""If set to True, enable prompt_tokens_details in usage."""
enable_per_request_metrics: bool = False
"""If set to True, include per-request timing metrics in API responses."""
enable_server_load_tracking: bool = False
"""If set to True, enable tracking server_load_metrics in the app state."""
enable_force_include_usage: bool = False
"""If set to True, including usage on every request."""
sse_keep_alive_interval: int = 0
"""Send an SSE keep-alive comment line every this many seconds when a
`/v1/chat/completions` or `/v1/completions` streaming response is idle
(queued, prefill, or between tokens), to prevent reverse proxies/tunnels
with read timeouts from closing the connection. Defaults to 0, which
disables keep-alive comments entirely.
"""
enable_tokenizer_info_endpoint: bool = False
"""Enable the `/tokenizer_info` endpoint. May expose chat
templates and other tokenizer configuration."""
enable_log_outputs: bool = False
"""If set to True, log model outputs (generations). Requires
`--enable-log-requests`. Output text and finish reasons are logged at INFO,
while output token IDs are logged at DEBUG."""
enable_log_deltas: bool = True
"""If set to False, output deltas will not be logged. Relevant only if
--enable-log-outputs is set.
"""
log_error_stack: bool = False
"""If set to True, log the stack trace of error responses"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
- ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes
server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an
8-char config hash.
- ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped.
- ``custom``: emits the literal string from ``--fingerprint-value``.
- ``none``: the field is omitted (serialized as ``null``).
"""
fingerprint_value: str | None = None
"""Literal fingerprint string used when ``--fingerprint-mode=custom``."""
@classmethod
def _customize_cli_kwargs(
cls,
frontend_kwargs: dict[str, Any],
) -> dict[str, Any]:
"""Customize argparse kwargs before arguments are registered.
Subclasses should override this and call
``super()._customize_cli_kwargs(frontend_kwargs)`` first.
"""
# Special case: default_chat_template_kwargs needs json.loads type
frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
# Special case: Tool call parser shows built-in options.
valid_tool_parsers = list(ToolParserManager.list_registered())
parsers_str = ",".join(valid_tool_parsers)
frontend_kwargs["tool_call_parser"]["metavar"] = (
f"{{{parsers_str}}} or name registered in --tool-parser-plugin"
)
return frontend_kwargs
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Register CLI arguments for this frontend class.
Subclasses should override ``_customize_cli_kwargs`` instead of
this method so that base-class postprocessing is always applied.
"""
from vllm.engine.arg_utils import get_kwargs
frontend_kwargs = get_kwargs(cls)
frontend_kwargs = cls._customize_cli_kwargs(frontend_kwargs)
group_name = cls.__name__.replace("Args", "")
frontend_group = parser.add_argument_group(
title=group_name,
description=cls.__doc__,
)
for key, value in frontend_kwargs.items():
extra_flags = value.pop("flags", [])
frontend_group.add_argument(
*extra_flags, f"--{key.replace('_', '-')}", **value
)
return parser
@config
class FrontendArgs(BaseFrontendArgs):
"""Arguments for the OpenAI-compatible frontend server."""
host: str | None = None
"""Host name."""
port: int = 8000
"""Port number."""
data_parallel_supervisor_port: int = 9256
"""HTTP port for aggregated health endpoints in multi-port external LB
mode."""
dp_supervisor_probe_interval_s: float = 5.0
"""Seconds between aggregated health probes in multi-port external LB mode."""
dp_supervisor_probe_timeout_s: float = 5.0
"""Seconds to wait between retries when a child health probe fails with a
connection error in multi-port external LB mode."""
dp_supervisor_probe_failure_threshold: int = 3
"""Number of consecutive connection-error retries before a child health
probe is declared failed in multi-port external LB mode."""
uds: str | None = None
"""Unix domain socket path. If set, host and port arguments are ignored."""
uvicorn_log_level: Literal[
"critical", "error", "warning", "info", "debug", "trace"
] = "info"
"""Log level for uvicorn."""
disable_uvicorn_access_log: bool = False
"""Disable uvicorn access log."""
disable_access_log_for_endpoints: str | None = None
"""Comma-separated list of endpoint paths to exclude from uvicorn access
logs. This is useful to reduce log noise from high-frequency endpoints
like health checks. Example: "/health,/metrics,/ping".
When set, access logs for requests to these paths will be suppressed
while keeping logs for other endpoints."""
allow_credentials: bool = False
"""Allow credentials."""
allowed_origins: list[str] = field(default_factory=list)
"""Allowed origins. Cross-origin browser access is disabled by default."""
allowed_methods: list[str] = field(default_factory=lambda: ["*"])
"""Allowed methods."""
allowed_headers: list[str] = field(default_factory=lambda: ["*"])
"""Allowed headers."""
api_key: list[str] | None = None
"""If provided, the server will require one of these keys to be presented in
the header.
Warning: this only authenticates endpoints under the `/v1`, `/v2`, and
`/inference` path prefixes. Other endpoints on the same server, including
`/invocations` (which exposes the same inference capabilities as `/v1`),
remain unauthenticated. Do not rely on `--api-key` alone to secure vLLM;
see
https://docs.vllm.ai/en/latest/usage/security.html#api-key-authentication-limitations
for what it does and does not protect."""
ssl_keyfile: str | None = None
"""The file path to the SSL key file."""
ssl_certfile: str | None = None
"""The file path to the SSL cert file."""
ssl_ca_certs: str | None = None
"""The CA certificates file."""
enable_ssl_refresh: bool = False
"""Refresh SSL Context when SSL certificate files change"""
ssl_cert_reqs: int = int(ssl.CERT_NONE)
"""Whether client certificate is required (see stdlib ssl module's)."""
ssl_ciphers: str | None = None
"""SSL cipher suites for HTTPS (TLS 1.2 and below only).
Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'"""
root_path: str | None = None
"""FastAPI root_path when app is behind a path based routing proxy."""
middleware: list[str] = field(default_factory=lambda: [])
"""Additional ASGI middleware to apply to the app. We accept multiple
--middleware arguments. The value should be an import path. If a function
is provided, vLLM will add it to the server using
`@app.middleware('http')`. If a class is provided, vLLM will
add it to the server using `app.add_middleware()`."""
enable_request_id_headers: bool = False
"""If specified, API server will add X-Request-Id header to responses."""
disable_fastapi_docs: bool = False
"""Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""
h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
"""Maximum size (bytes) of an incomplete HTTP event (header or body) for
h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB)."""
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
"""Maximum number of HTTP headers allowed in a request for h11 parser.
Helps mitigate header abuse. Default: 256."""
enable_flash_late_interaction: bool = True
"""If set, run pooling score MaxSim on GPU in the API server process.
Can significantly improve late-interaction scoring performance.
When disabled, the setting is also propagated to the engine's
`PoolerConfig`, so the engine-side scorer uses the reference MaxSim
path instead of the fused Triton kernel."""
@classmethod
def _customize_cli_kwargs(
cls,
frontend_kwargs: dict[str, Any],
) -> dict[str, Any]:
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
# Special case: allowed_origins, allowed_methods, allowed_headers all
# need json.loads type
# Should also remove nargs
frontend_kwargs["allowed_origins"]["type"] = json.loads
frontend_kwargs["allowed_methods"]["type"] = json.loads
frontend_kwargs["allowed_headers"]["type"] = json.loads
del frontend_kwargs["allowed_origins"]["nargs"]
del frontend_kwargs["allowed_methods"]["nargs"]
del frontend_kwargs["allowed_headers"]["nargs"]
# Special case: Middleware needs to append action
frontend_kwargs["middleware"]["action"] = "append"
frontend_kwargs["middleware"]["type"] = str
if "nargs" in frontend_kwargs["middleware"]:
del frontend_kwargs["middleware"]["nargs"]
frontend_kwargs["middleware"]["default"] = []
# Special case: disable_access_log_for_endpoints is a single
# comma-separated string, not a list
if "nargs" in frontend_kwargs["disable_access_log_for_endpoints"]:
del frontend_kwargs["disable_access_log_for_endpoints"]["nargs"]
return frontend_kwargs
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Create the CLI argument parser used by the OpenAI API server.
We rely on the helper methods of `FrontendArgs` and `AsyncEngineArgs` to
register all arguments instead of manually enumerating them here. This
avoids code duplication and keeps the argument definitions in one place.
"""
parser.add_argument(
"model_tag",
type=str,
nargs="?",
help="The model tag to serve (optional if specified in config)",
)
parser.add_argument(
"--headless",
action="store_true",
default=False,
help="Run in headless mode. See multi-node data parallel "
"documentation for more details.",
)
parser.add_argument(
"--api-server-count",
"-asc",
type=int,
default=None,
help="How many API server processes to run. "
"Defaults to data_parallel_size if not specified.",
)
parser.add_argument(
"--config",
help="Read CLI options from a config file. "
"Must be a YAML with the following options: "
"https://docs.vllm.ai/en/latest/configuration/serve_args.html",
)
parser.add_argument(
"--grpc",
action="store_true",
default=False,
help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
"of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc].",
)
parser = FrontendArgs.add_cli_args(parser)
parser = AsyncEngineArgs.add_cli_args(parser)
return parser
def validate_parsed_serve_args(args: argparse.Namespace):
"""Quick checks for model serve args that raise prior to loading."""
if hasattr(args, "subparser") and args.subparser != "serve":
return
# Ensure that the chat template is valid; raises if it likely isn't
validate_chat_template(args.chat_template)
if args.chat_template is not None and "hf" in (
args.tool_call_parser,
getattr(args, "reasoning_parser", None),
):
logger.warning(
"--chat-template is set; the hf parser still expects "
"the checkpoint's output format."
)
# Enable auto tool needs a tool call parser to be valid
if args.enable_auto_tool_choice and not args.tool_call_parser:
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
if args.enable_log_outputs and not args.enable_log_requests:
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
# SSE keep-alive interval must be zero (disabled) or a positive integer.
if getattr(args, "sse_keep_alive_interval", 0) < 0:
raise ValueError(
"Error: --sse-keep-alive-interval must be a non-negative integer "
"(0 disables keep-alive comments)."
)
if getattr(args, "enable_per_request_metrics", False) and getattr(
args, "disable_log_stats", False
):
raise ValueError(
"Error: --enable-per-request-metrics requires engine statistics "
"logging; remove --disable-log-stats to enable per-request metrics."
)
if args.data_parallel_multi_port_external_lb:
from vllm.entrypoints.launchers.dp_supervisor import (
validate_multi_port_external_lb_args,
)
validate_multi_port_external_lb_args(args)
def propagate_flash_late_interaction(args, engine_args) -> None:
"""Propagate `--disable-flash-late-interaction` into the engine config.
The frontend flag alone only disables the API-server scoring path;
mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the
engine-side scorer fall back to the reference MaxSim path too.
"""
if getattr(args, "enable_flash_late_interaction", True):
return
from vllm.config.pooler import PoolerConfig
if engine_args.pooler_config is None:
engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False)
else:
engine_args.pooler_config.enable_flash_late_interaction = False