[Frontend] Move cli_args.py and dp_supervisor.py out openai folder (#53659)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-08-25 07:02:32 +00:00
committed by GitHub
parent c74dd2d208
commit 735daf8ef6
14 changed files with 42 additions and 33 deletions
-463
View File
@@ -1,463 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
This file contains the command line arguments for the vLLM's
OpenAI-compatible server. It is kept in a separate file for documentation
purposes.
"""
import argparse
import json
import ssl
from collections.abc import Sequence
from dataclasses import field
from typing import Any, Literal
import vllm.envs as envs
from vllm.config import config
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
from vllm.entrypoints.chat_utils import (
ChatTemplateContentFormatOption,
validate_chat_template,
)
from vllm.entrypoints.launchers.utils.constants import (
H11_MAX_HEADER_COUNT_DEFAULT,
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.tool_parsers import ToolParserManager
from vllm.utils.argparse_utils import FlexibleArgumentParser
class LoRAParserAction(argparse.Action):
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: str | Sequence[str] | None,
option_string: str | None = None,
):
if values is None:
values = []
if isinstance(values, str):
raise TypeError("Expected values to be a list")
lora_list: list[LoRAModulePath] = []
for item in values:
if item in [None, ""]: # Skip if item is None or empty string
continue
if "=" in item and "," not in item: # Old format: name=path
name, path = item.split("=")
lora_list.append(LoRAModulePath(name, path))
else: # Assume JSON format
try:
lora_dict = json.loads(item)
lora = LoRAModulePath(**lora_dict)
lora_list.append(lora)
except json.JSONDecodeError:
parser.error(f"Invalid JSON format for --lora-modules: {item}")
except TypeError as e:
parser.error(
f"Invalid fields for --lora-modules: {item} - {str(e)}"
)
setattr(namespace, self.dest, lora_list)
@config
class BaseFrontendArgs:
"""Base arguments for the OpenAI-compatible frontend server.
This base class does not include host, port, and server-specific arguments
like SSL, CORS, and HTTP server settings. Those arguments are added by
the subclasses.
"""
lora_modules: list[LoRAModulePath] | None = None
"""LoRA modules configurations in either 'name=path' format or JSON format
or JSON list format. Example (old format): `'name=path'` Example (new
format): `{\"name\": \"name\", \"path\": \"lora_path\",
\"base_model_name\": \"id\"}`"""
chat_template: str | None = None
"""The file path to the chat template, or the template in single-line form
for the specified model."""
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
"""The format to render message content within a chat template.
* "string" will render the content as a string. Example: `"Hello World"`
* "openai" will render the content as a list of dictionaries, similar to
OpenAI schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
trust_request_chat_template: bool = False
"""Whether to trust the chat template provided in the request. If False,
the server will always use the chat template specified by `--chat-template`
or the ones from tokenizer."""
default_chat_template_kwargs: dict[str, Any] | None = None
"""Default keyword arguments to pass to the chat template renderer.
These will be merged with request-level chat_template_kwargs,
with request values taking precedence. Useful for setting default
behavior for reasoning models. Example: '{"enable_thinking": false}'
to disable thinking mode by default for Qwen3/DeepSeek models."""
response_role: str = "assistant"
"""The role name to return if `request.add_generation_prompt=true`."""
return_tokens_as_token_ids: bool = False
"""When `--max-logprobs` is specified, represents single tokens as
strings of the form 'token_id:{token_id}' so that tokens that are not
JSON-encodable can be identified."""
enable_auto_tool_choice: bool = False
"""Enable auto tool choice for supported models. Use `--tool-call-parser`
to specify which parser to use."""
exclude_tools_when_tool_choice_none: bool = False
"""If specified, exclude tool definitions in prompts when
tool_choice='none'."""
tool_call_parser: str | None = None
"""Select the tool call parser depending on the model that you're using.
This is used to parse the model-generated tool call into OpenAI API format.
Required for `--enable-auto-tool-choice`. You can choose any option from
the built-in parsers or register a plugin via `--tool-parser-plugin`."""
tool_parser_plugin: str = ""
"""Special the tool parser plugin write to parse the model-generated tool
into OpenAI API format, the name register in this plugin can be used in
`--tool-call-parser`."""
tool_server: str | None = None
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
built-in demo tools (browser and Python code interpreter). WARNING:
The `demo` Python tool executes model-generated code in Docker without
network isolation by default. See the security guide for more
information."""
log_config_file: str | None = envs.VLLM_LOGGING_CONFIG_PATH
"""Path to logging config JSON file for both vllm and uvicorn"""
max_log_len: int | None = None
"""Max number of prompt characters or prompt ID numbers being printed in
log. The default of None means unlimited."""
enable_prompt_tokens_details: bool = False
"""If set to True, enable prompt_tokens_details in usage."""
enable_per_request_metrics: bool = False
"""If set to True, include per-request timing metrics in API responses."""
enable_server_load_tracking: bool = False
"""If set to True, enable tracking server_load_metrics in the app state."""
enable_force_include_usage: bool = False
"""If set to True, including usage on every request."""
sse_keep_alive_interval: int = 0
"""Send an SSE keep-alive comment line every this many seconds when a
`/v1/chat/completions` or `/v1/completions` streaming response is idle
(queued, prefill, or between tokens), to prevent reverse proxies/tunnels
with read timeouts from closing the connection. Defaults to 0, which
disables keep-alive comments entirely.
"""
enable_tokenizer_info_endpoint: bool = False
"""Enable the `/tokenizer_info` endpoint. May expose chat
templates and other tokenizer configuration."""
enable_log_outputs: bool = False
"""If set to True, log model outputs (generations). Requires
`--enable-log-requests`. Output text and finish reasons are logged at INFO,
while output token IDs are logged at DEBUG."""
enable_log_deltas: bool = True
"""If set to False, output deltas will not be logged. Relevant only if
--enable-log-outputs is set.
"""
cohere_is_reasoning_model: bool = True
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
reasoning Command-family model. When True (default), the assistant's
chain-of-thought is surfaced as a ``thinking`` content block (or
``content-*`` events on the stream). When False, reasoning is
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
events) whenever the model emits tool calls, matching older non-
reasoning Command models. Has no effect on the non-Cohere
endpoints."""
cohere_format: str = "cmd4"
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
format to render: ``cmd4`` (current Command A+ models; default) or
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
wrong format silently produces a prompt the model wasn't trained
on, which most commonly manifests as the model emitting text but no
citations / tool calls / thinking blocks. Equivalent to passing
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
explicit request-level ``chat_template_kwargs.cohere_format`` takes
priority."""
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
"""If set to True, log the stack trace of error responses"""
tokens_only: bool = False
"""
If set to True, only enable the Tokens In<>Out endpoint.
This is intended for use in a Disaggregated Everything setup.
"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
- ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes
server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an
8-char config hash.
- ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped.
- ``custom``: emits the literal string from ``--fingerprint-value``.
- ``none``: the field is omitted (serialized as ``null``).
"""
fingerprint_value: str | None = None
"""Literal fingerprint string used when ``--fingerprint-mode=custom``."""
@classmethod
def _customize_cli_kwargs(
cls,
frontend_kwargs: dict[str, Any],
) -> dict[str, Any]:
"""Customize argparse kwargs before arguments are registered.
Subclasses should override this and call
``super()._customize_cli_kwargs(frontend_kwargs)`` first.
"""
# Special case: default_chat_template_kwargs needs json.loads type
frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
# Special case: LoRA modules need custom parser action and
# optional_type(str)
frontend_kwargs["lora_modules"]["type"] = optional_type(str)
frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
# Special case: Tool call parser shows built-in options.
valid_tool_parsers = list(ToolParserManager.list_registered())
parsers_str = ",".join(valid_tool_parsers)
frontend_kwargs["tool_call_parser"]["metavar"] = (
f"{{{parsers_str}}} or name registered in --tool-parser-plugin"
)
return frontend_kwargs
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Register CLI arguments for this frontend class.
Subclasses should override ``_customize_cli_kwargs`` instead of
this method so that base-class postprocessing is always applied.
"""
from vllm.engine.arg_utils import get_kwargs
frontend_kwargs = get_kwargs(cls)
frontend_kwargs = cls._customize_cli_kwargs(frontend_kwargs)
group_name = cls.__name__.replace("Args", "")
frontend_group = parser.add_argument_group(
title=group_name,
description=cls.__doc__,
)
for key, value in frontend_kwargs.items():
extra_flags = value.pop("flags", [])
frontend_group.add_argument(
*extra_flags, f"--{key.replace('_', '-')}", **value
)
return parser
@config
class FrontendArgs(BaseFrontendArgs):
"""Arguments for the OpenAI-compatible frontend server."""
host: str | None = None
"""Host name."""
port: int = 8000
"""Port number."""
data_parallel_supervisor_port: int = 9256
"""HTTP port for aggregated health endpoints in multi-port external LB
mode."""
dp_supervisor_probe_interval_s: float = 5.0
"""Seconds between aggregated health probes in multi-port external LB mode."""
dp_supervisor_probe_timeout_s: float = 5.0
"""Seconds to wait between retries when a child health probe fails with a
connection error in multi-port external LB mode."""
dp_supervisor_probe_failure_threshold: int = 3
"""Number of consecutive connection-error retries before a child health
probe is declared failed in multi-port external LB mode."""
uds: str | None = None
"""Unix domain socket path. If set, host and port arguments are ignored."""
uvicorn_log_level: Literal[
"critical", "error", "warning", "info", "debug", "trace"
] = "info"
"""Log level for uvicorn."""
disable_uvicorn_access_log: bool = False
"""Disable uvicorn access log."""
disable_access_log_for_endpoints: str | None = None
"""Comma-separated list of endpoint paths to exclude from uvicorn access
logs. This is useful to reduce log noise from high-frequency endpoints
like health checks. Example: "/health,/metrics,/ping".
When set, access logs for requests to these paths will be suppressed
while keeping logs for other endpoints."""
allow_credentials: bool = False
"""Allow credentials."""
allowed_origins: list[str] = field(default_factory=lambda: ["*"])
"""Allowed origins."""
allowed_methods: list[str] = field(default_factory=lambda: ["*"])
"""Allowed methods."""
allowed_headers: list[str] = field(default_factory=lambda: ["*"])
"""Allowed headers."""
api_key: list[str] | None = None
"""If provided, the server will require one of these keys to be presented in
the header.
Warning: this only authenticates endpoints under the `/v1`, `/v2`, and
`/inference` path prefixes. Other endpoints on the same server, including
`/invocations` (which exposes the same inference capabilities as `/v1`),
remain unauthenticated. Do not rely on `--api-key` alone to secure vLLM;
see
https://docs.vllm.ai/en/latest/usage/security.html#api-key-authentication-limitations
for what it does and does not protect."""
ssl_keyfile: str | None = None
"""The file path to the SSL key file."""
ssl_certfile: str | None = None
"""The file path to the SSL cert file."""
ssl_ca_certs: str | None = None
"""The CA certificates file."""
enable_ssl_refresh: bool = False
"""Refresh SSL Context when SSL certificate files change"""
ssl_cert_reqs: int = int(ssl.CERT_NONE)
"""Whether client certificate is required (see stdlib ssl module's)."""
ssl_ciphers: str | None = None
"""SSL cipher suites for HTTPS (TLS 1.2 and below only).
Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'"""
root_path: str | None = None
"""FastAPI root_path when app is behind a path based routing proxy."""
middleware: list[str] = field(default_factory=lambda: [])
"""Additional ASGI middleware to apply to the app. We accept multiple
--middleware arguments. The value should be an import path. If a function
is provided, vLLM will add it to the server using
`@app.middleware('http')`. If a class is provided, vLLM will
add it to the server using `app.add_middleware()`."""
enable_request_id_headers: bool = False
"""If specified, API server will add X-Request-Id header to responses."""
disable_fastapi_docs: bool = False
"""Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""
h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
"""Maximum size (bytes) of an incomplete HTTP event (header or body) for
h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB)."""
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
"""Maximum number of HTTP headers allowed in a request for h11 parser.
Helps mitigate header abuse. Default: 256."""
enable_offline_docs: bool = False
"""
Enable offline FastAPI documentation for air-gapped environments.
Uses vendored static assets bundled with vLLM.
"""
enable_flash_late_interaction: bool = True
"""If set, run pooling score MaxSim on GPU in the API server process.
Can significantly improve late-interaction scoring performance."""
@classmethod
def _customize_cli_kwargs(
cls,
frontend_kwargs: dict[str, Any],
) -> dict[str, Any]:
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
# Special case: allowed_origins, allowed_methods, allowed_headers all
# need json.loads type
# Should also remove nargs
frontend_kwargs["allowed_origins"]["type"] = json.loads
frontend_kwargs["allowed_methods"]["type"] = json.loads
frontend_kwargs["allowed_headers"]["type"] = json.loads
del frontend_kwargs["allowed_origins"]["nargs"]
del frontend_kwargs["allowed_methods"]["nargs"]
del frontend_kwargs["allowed_headers"]["nargs"]
# Special case: Middleware needs to append action
frontend_kwargs["middleware"]["action"] = "append"
frontend_kwargs["middleware"]["type"] = str
if "nargs" in frontend_kwargs["middleware"]:
del frontend_kwargs["middleware"]["nargs"]
frontend_kwargs["middleware"]["default"] = []
# Special case: disable_access_log_for_endpoints is a single
# comma-separated string, not a list
if "nargs" in frontend_kwargs["disable_access_log_for_endpoints"]:
del frontend_kwargs["disable_access_log_for_endpoints"]["nargs"]
return frontend_kwargs
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Create the CLI argument parser used by the OpenAI API server.
We rely on the helper methods of `FrontendArgs` and `AsyncEngineArgs` to
register all arguments instead of manually enumerating them here. This
avoids code duplication and keeps the argument definitions in one place.
"""
parser.add_argument(
"model_tag",
type=str,
nargs="?",
help="The model tag to serve (optional if specified in config)",
)
parser.add_argument(
"--headless",
action="store_true",
default=False,
help="Run in headless mode. See multi-node data parallel "
"documentation for more details.",
)
parser.add_argument(
"--api-server-count",
"-asc",
type=int,
default=None,
help="How many API server processes to run. "
"Defaults to data_parallel_size if not specified.",
)
parser.add_argument(
"--config",
help="Read CLI options from a config file. "
"Must be a YAML with the following options: "
"https://docs.vllm.ai/en/latest/configuration/serve_args.html",
)
parser.add_argument(
"--grpc",
action="store_true",
default=False,
help="Launch a gRPC server instead of the HTTP OpenAI-compatible "
"server. Requires: pip install vllm[grpc].",
)
parser = FrontendArgs.add_cli_args(parser)
parser = AsyncEngineArgs.add_cli_args(parser)
return parser
def validate_parsed_serve_args(args: argparse.Namespace):
"""Quick checks for model serve args that raise prior to loading."""
# `vllm launch <component>` builds its parser with make_arg_parser too (see
# LaunchSubcommandBase.add_cli_args), so its args are serve args as well.
if hasattr(args, "subparser") and args.subparser not in ("serve", "launch"):
return
# Ensure that the chat template is valid; raises if it likely isn't
validate_chat_template(args.chat_template)
# Enable auto tool needs a tool call parser to be valid
if args.enable_auto_tool_choice and not args.tool_call_parser:
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
if args.enable_log_outputs and not args.enable_log_requests:
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
# SSE keep-alive interval must be zero (disabled) or a positive integer.
if getattr(args, "sse_keep_alive_interval", 0) < 0:
raise ValueError(
"Error: --sse-keep-alive-interval must be a non-negative integer "
"(0 disables keep-alive comments)."
)
if getattr(args, "enable_per_request_metrics", False) and getattr(
args, "disable_log_stats", False
):
raise ValueError(
"Error: --enable-per-request-metrics requires engine statistics "
"logging; remove --disable-log-stats to enable per-request metrics."
)
if args.data_parallel_multi_port_external_lb:
from vllm.entrypoints.openai.dp_supervisor import (
validate_multi_port_external_lb_args,
)
validate_multi_port_external_lb_args(args)
def create_parser_for_docs() -> FlexibleArgumentParser:
parser_for_docs = FlexibleArgumentParser(
prog="-m vllm.entrypoints.launchers.api_server.entry"
)
return make_arg_parser(parser_for_docs)
-571
View File
@@ -1,571 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import asyncio
import contextlib
import copy
import multiprocessing
import os
import signal
import time
from functools import partial
from http import HTTPStatus
from multiprocessing.process import BaseProcess
import aiohttp
import psutil
import uvicorn
import uvloop
from fastapi import FastAPI, Response
import vllm.envs as envs
from vllm.entrypoints.launchers.launcher import NoSignalServer
from vllm.entrypoints.launchers.utils.server_utils import get_uvicorn_log_config
from vllm.logger import init_logger
from vllm.utils.system_utils import (
decorate_logs,
kill_process_tree,
set_process_title,
)
from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
logger = init_logger(__name__)
CHILD_EXIT_GRACE_S = 5.0
def infer_multi_port_external_lb_start_rank(args: argparse.Namespace) -> int:
start_rank = getattr(args, "data_parallel_start_rank", None)
if start_rank is not None:
return start_rank
node_rank = getattr(args, "node_rank", 0) or 0
local_size = getattr(args, "data_parallel_size_local", 0) or 0
return node_rank * local_size
def validate_multi_port_external_lb_args(args: argparse.Namespace) -> None:
if getattr(args, "grpc", False):
raise ValueError(
"Error: --data-parallel-multi-port-external-lb does not support --grpc"
)
if args.uds is not None:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb does not support --uds"
)
if bool(args.ssl_keyfile) != bool(args.ssl_certfile):
raise ValueError(
"Error: --ssl-keyfile and --ssl-certfile must be provided together"
)
if args.api_server_count not in (None, 1):
raise ValueError(
"Error: --data-parallel-multi-port-external-lb currently requires "
"--api-server-count=1"
)
if args.data_parallel_rank is not None:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb manages child "
"--data-parallel-rank values internally"
)
if args.data_parallel_external_lb or args.data_parallel_hybrid_lb:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb cannot be combined with "
"--data-parallel-external-lb or --data-parallel-hybrid-lb"
)
if args.data_parallel_size < 2:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb requires "
"--data-parallel-size > 1"
)
local_size = args.data_parallel_size_local
if local_size is None or local_size < 2:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb requires "
"--data-parallel-size-local >= 2"
)
if local_size > args.data_parallel_size:
raise ValueError(
"Error: --data-parallel-size-local cannot exceed --data-parallel-size"
)
if args.data_parallel_size % local_size != 0:
raise ValueError(
"Error: --data-parallel-size must be divisible by "
"--data-parallel-size-local"
)
start_rank = infer_multi_port_external_lb_start_rank(args)
if start_rank + local_size > args.data_parallel_size:
raise ValueError(
"Error: multi-port supervised ranks would exceed --data-parallel-size"
)
supervisor_port = args.data_parallel_supervisor_port
child_port_min = args.port
child_port_max = args.port + local_size - 1
if child_port_min <= supervisor_port <= child_port_max:
raise ValueError(
f"Error: --data-parallel-supervisor-port {supervisor_port} "
f"overlaps with child rank ports {child_port_min}-{child_port_max}"
)
def _build_vllm_dp_server_args(
args: argparse.Namespace, local_rank: int
) -> argparse.Namespace:
child_args = copy.copy(args)
child_args.port = args.port + local_rank
child_args.data_parallel_rank = (
infer_multi_port_external_lb_start_rank(args) + local_rank
)
child_args.data_parallel_start_rank = None
child_args.data_parallel_size_local = 1
child_args.data_parallel_external_lb = True
child_args.data_parallel_hybrid_lb = False
child_args.data_parallel_multi_port_external_lb = False
child_args.data_parallel_supervisor_port = None
child_args.api_server_count = 1
child_args.device_ids = _build_device_ids(args, local_rank)
return child_args
def _build_device_ids(args: argparse.Namespace, local_rank: int) -> list[int | str]:
"""Build the --device-ids value for a DP child process.
The child resolves these against its own inherited device-control env
var (e.g. CUDA_VISIBLE_DEVICES), so integer IDs must stay env-relative
here rather than being translated to physical IDs.
"""
devices_per_rank = args.tensor_parallel_size * args.pipeline_parallel_size
start = local_rank * devices_per_rank
stop = start + devices_per_rank
device_ids = getattr(args, "device_ids", None)
if device_ids is not None:
if stop > len(device_ids):
raise ValueError(
f"--device-ids has {len(device_ids)} entries, but DP rank "
f"{local_rank} needs devices [{start}, {stop})"
)
return device_ids[start:stop]
return list(range(start, stop))
def _child_base_url(args: argparse.Namespace, port: int) -> str:
host = args.host or "127.0.0.1"
if host == "0.0.0.0":
host = "127.0.0.1"
elif host == "::":
host = "::1"
scheme = "https" if args.ssl_keyfile and args.ssl_certfile else "http"
return f"{scheme}://{host}:{port}"
def _join_processes_with_timeout(processes: list[BaseProcess], timeout: float) -> None:
deadline = time.monotonic() + timeout
for process in processes:
remaining = deadline - time.monotonic()
if remaining <= 0:
break
process.join(timeout=remaining)
async def _probe_endpoint(
session: aiohttp.ClientSession,
args: argparse.Namespace,
port: int,
path: str,
conn_err_failure_threshold: int = 3,
conn_err_retry_delay: float = 5.0,
) -> bool:
"""
Probe /health endpoint for 200 status.
If there is a connection error, retry every N seconds.
"""
for iteration in range(conn_err_failure_threshold):
try:
probe_ssl = None
if args.ssl_keyfile and args.ssl_certfile:
# Probes target node-local child servers over loopback, so skip
# certificate verification to avoid SAN/hostname mismatches for
# localhost/127.0.0.1 deployments.
probe_ssl = False
async with session.get(
_child_base_url(args, port) + path, ssl=probe_ssl
) as response:
# vLLM returns 503 on EngineDeadError, so we should return
# immediately if vLLM responds with a non-200 status code.
return response.status == HTTPStatus.OK
except (aiohttp.ClientError, asyncio.TimeoutError) as e:
# Allow retry of connection errors.
logger.debug(
"Probe attempt %d/%d failed on port %d: %r",
iteration + 1,
conn_err_failure_threshold,
port,
e,
)
if iteration < conn_err_failure_threshold - 1:
await asyncio.sleep(conn_err_retry_delay)
return False
def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:
app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
app.state.supervisor = supervisor
def _status_response(ok: bool) -> Response:
return Response(
status_code=(HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE)
)
@app.get("/health", include_in_schema=False)
async def health() -> Response:
return _status_response(app.state.supervisor.is_ready)
@app.get("/ready", include_in_schema=False)
@app.get("/readyz", include_in_schema=False)
async def ready() -> Response:
return _status_response(app.state.supervisor.is_ready)
return app
def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
from vllm.entrypoints.launchers.api_server.entry import run_server
uvloop.run(run_server(child_args))
def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
from vllm.entrypoints.cli.serve import run_multi_api_server
run_multi_api_server(child_args)
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
"""
Entrypoint function for the vLLM DP Server.
"""
# Create a fresh process group for the vLLM DP Server,
# so that CTRL-C is propagated cleanly.
os.setpgrp()
name = f"APIServer_DP{child_args.data_parallel_rank}"
set_process_title(name)
decorate_logs(name)
if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH:
_run_rust_vllm_dp_server(child_args)
else:
_run_python_vllm_dp_server(child_args)
class DPSupervisor:
def __init__(self, args: argparse.Namespace):
validate_multi_port_external_lb_args(args)
self.args = args
self.supervisor_port = args.data_parallel_supervisor_port
self.child_ports = [
args.port + local_rank
for local_rank in range(args.data_parallel_size_local)
]
self._is_ready = False
self._processes: list[BaseProcess] = []
self._shutdown_event = asyncio.Event()
self._shutdown_signal = signal.SIGTERM
@property
def is_ready(self) -> bool:
return self._is_ready and not self._shutdown_event.is_set()
async def run(self) -> None:
loop = asyncio.get_running_loop()
decorate_logs("DPSupervisor")
# K8s sends SIGTERM for shutdown - begin graceful termination.
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, partial(self._handle_signal, sig))
supervisor_server: uvicorn.Server | None = None
supervisor_server_task: asyncio.Task[None] | None = None
try:
# Launch vLLM DP Servers and begin monitoring them.
self._start_children()
monitor_task = asyncio.create_task(
self._monitor_children(), name="dp-monitor"
)
# Only start the DPSupervisor server once the vLLM DP Servers are
# ready. This avoids the supervisor /health endpoint returning 503
# to external load balancer probes while the engines initialize.
await self._wait_until_ready(monitor_task)
if self.is_ready and not monitor_task.done():
supervisor_server, supervisor_server_task = await self._start_server()
await monitor_task
finally:
self._is_ready = False
await self._shutdown_children()
# Shutdown the DP Supervisor server.
if supervisor_server is not None and supervisor_server_task is not None:
supervisor_server.should_exit = True
await supervisor_server_task
async def _start_server(self) -> tuple[uvicorn.Server, asyncio.Task[None]]:
"""
Launch the DPSupervisor HTTP server.
Called only after the vLLM DP Servers are ready so that /health does
not return 503 to external probes while the engines are initializing.
"""
app = _build_dp_supervisor_app(self)
host = self.args.host or "0.0.0.0"
uvicorn_kwargs: dict = {}
log_config = get_uvicorn_log_config(self.args)
if log_config is not None:
uvicorn_kwargs["log_config"] = log_config
config = uvicorn.Config(
app,
host=host,
port=self.supervisor_port,
log_level=self.args.uvicorn_log_level,
access_log=not self.args.disable_uvicorn_access_log,
ssl_keyfile=self.args.ssl_keyfile,
ssl_certfile=self.args.ssl_certfile,
ssl_ca_certs=self.args.ssl_ca_certs,
ssl_cert_reqs=self.args.ssl_cert_reqs,
ssl_ciphers=self.args.ssl_ciphers,
**uvicorn_kwargs,
)
supervisor_server = NoSignalServer(config)
supervisor_server_task = asyncio.create_task(
supervisor_server.serve(),
name="dp-supervisor",
)
supervisor_server_task.add_done_callback(
lambda _task: self._shutdown_event.set()
)
# Ensure DPSupervisor task starts on the event loop.
while not supervisor_server.started:
if supervisor_server_task.done():
supervisor_server_task.result()
raise RuntimeError("DPSupervisor exited before startup.")
await asyncio.sleep(0.05)
logger.info("Started DPSupervisor on %s:%d", host, self.supervisor_port)
return supervisor_server, supervisor_server_task
async def _wait_until_ready(self, monitor_task: asyncio.Task[None]) -> None:
"""
Block until the vLLM DP Servers are ready or shutdown is triggered.
Returns early if monitoring stops (e.g. a DP Server dies during
startup), in which case the supervisor server is never started.
"""
logger.info("Waiting for vLLM DP Servers to become ready.")
while not self._is_ready and not self._shutdown_event.is_set():
if monitor_task.done():
return
await asyncio.sleep(0.05)
def _handle_signal(self, signum: int) -> None:
"""
Signal handler that is added to the event loop.
This catches the SIGTERM from K8s and begins graceful shutdown,
by setting the _shutdown_event(), which is watched by the main
coroutine monitoring the vLLM DP Servers.
"""
if self._shutdown_event.is_set():
return
self._shutdown_signal = signal.Signals(signum)
logger.info(
"DPSupervisor received %s, shutting down.",
self._shutdown_signal.name,
)
self._shutdown_event.set()
self._is_ready = False
def _start_children(self) -> None:
"""
Launch vLLM DP Servers on separate GPUs.
"""
logger.info("Launching vLLM DP Servers")
context = multiprocessing.get_context("spawn")
for local_rank in range(self.args.data_parallel_size_local):
child_args = _build_vllm_dp_server_args(self.args, local_rank)
process = context.Process(
target=_run_vllm_dp_server,
name=f"APIServer_DPRank_{child_args.data_parallel_rank}",
args=(child_args,),
)
process.start()
self._processes.append(process)
async def _probe_all_children(self) -> None:
"""
Background coroutine: probes all child endpoints on each interval.
Exits when any server becomes unhealthy after being ready, signalling
_monitor_children to initiate shutdown.
"""
timeout = aiohttp.ClientTimeout(total=self.args.dp_supervisor_probe_timeout_s)
async with aiohttp.ClientSession(timeout=timeout) as session:
while not self._shutdown_event.is_set():
threshold = (
self.args.dp_supervisor_probe_failure_threshold
if self._is_ready
else 1
)
results = await asyncio.gather(
*(
_probe_endpoint(
session,
self.args,
port,
"/health",
conn_err_failure_threshold=threshold,
conn_err_retry_delay=self.args.dp_supervisor_probe_interval_s,
)
for port in self.child_ports
),
return_exceptions=True,
)
all_healthy = all(r is True for r in results)
if all_healthy:
# If all healthy, we are ready to receive requests.
# This conditional avoids a potential race condition
# where shutdown is set, THEN the probe returns true.
if not self._shutdown_event.is_set():
self._is_ready = True
elif self._is_ready:
# Once ready, any failure in the probe means vLLM is dead.
num_unhealthy = sum(1 for r in results if r is not True)
logger.info(
"DPSupervisor probe found %s unhealthy DP Servers.",
num_unhealthy,
)
self._is_ready = False
self._shutdown_event.set()
return
with contextlib.suppress(asyncio.TimeoutError):
logger.debug(
"Waiting for %s seconds before next probe",
self.args.dp_supervisor_probe_interval_s,
)
await asyncio.wait_for(
self._shutdown_event.wait(),
timeout=self.args.dp_supervisor_probe_interval_s,
)
async def _monitor_children(self) -> None:
"""
Main coroutine task that monitors the children vLLM servers.
Before the vLLM servers are /ready:
- if the pid is dead, we will shut down
- if the probe fails, we try again after dp_supervisor_probe_interval_s
After the vLLM servers are /ready:
- if the pid is dead, we will shut down
- if the probe fails, we will shut down
"""
probe_task = asyncio.create_task(
self._probe_all_children(), name="dp-health-probe"
)
try:
while not self._shutdown_event.is_set():
# 1. Check for dead processes
n_failed = len([p for p in self._processes if not p.is_alive()])
if n_failed > 0:
logger.info("DPSupervisor found %s exited DP Servers.", n_failed)
break
# 2. Check if the probe background task crashed or failed.
if probe_task.done():
# Extract exception if it crashed, or log failure
exc = probe_task.exception() if not probe_task.cancelled() else None
logger.info("DPSupervisor probe task stopped. Exception: %s", exc)
break
# Sleep for probe_interval seconds or until a shutdown.
with contextlib.suppress(asyncio.TimeoutError):
logger.debug(
"Waiting for %s seconds before next monitor",
self.args.dp_supervisor_probe_interval_s,
)
await asyncio.wait_for(
self._shutdown_event.wait(),
timeout=self.args.dp_supervisor_probe_interval_s,
)
finally:
# Cleanup probe task if needed.
if not probe_task.done():
probe_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await probe_task
async def _shutdown_children(self) -> None:
"""Terminate the vLLM DP servers."""
process_timeout = get_engine_process_shutdown_timeout(
self.args.shutdown_timeout, self.args.shutdown_timeout
)
assert process_timeout is not None
timeout = process_timeout + CHILD_EXIT_GRACE_S
try:
logger.info(
"DPSupervisor forwarding %s to DP Servers.",
self._shutdown_signal.name,
)
for process in self._processes:
pid = process.pid
if not process.is_alive() or pid is None:
continue
with contextlib.suppress(ProcessLookupError, OSError):
os.kill(pid, self._shutdown_signal)
try:
await asyncio.to_thread(
_join_processes_with_timeout, self._processes, timeout
)
except asyncio.CancelledError:
logger.warning("Shutdown await cancelled")
raise
finally:
for process in self._processes:
pid = process.pid
if not process.is_alive() or pid is None:
continue
logger.warning(
"DP server %s did not exit within %.1fs; force killing.",
process.name,
timeout,
)
with contextlib.suppress(
ProcessLookupError,
OSError,
psutil.NoSuchProcess,
psutil.AccessDenied,
):
kill_process_tree(pid)
def run_dp_supervisor(args: argparse.Namespace) -> None:
uvloop.run(DPSupervisor(args).run())