[Frontend] Move cli_args.py and dp_supervisor.py out openai folder (#53659)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,463 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
This file contains the command line arguments for the vLLM's
|
||||
OpenAI-compatible server. It is kept in a separate file for documentation
|
||||
purposes.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import ssl
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import field
|
||||
from typing import Any, Literal
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import config
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
validate_chat_template,
|
||||
)
|
||||
from vllm.entrypoints.launchers.utils.constants import (
|
||||
H11_MAX_HEADER_COUNT_DEFAULT,
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class LoRAParserAction(argparse.Action):
|
||||
def __call__(
|
||||
self,
|
||||
parser: argparse.ArgumentParser,
|
||||
namespace: argparse.Namespace,
|
||||
values: str | Sequence[str] | None,
|
||||
option_string: str | None = None,
|
||||
):
|
||||
if values is None:
|
||||
values = []
|
||||
if isinstance(values, str):
|
||||
raise TypeError("Expected values to be a list")
|
||||
|
||||
lora_list: list[LoRAModulePath] = []
|
||||
for item in values:
|
||||
if item in [None, ""]: # Skip if item is None or empty string
|
||||
continue
|
||||
if "=" in item and "," not in item: # Old format: name=path
|
||||
name, path = item.split("=")
|
||||
lora_list.append(LoRAModulePath(name, path))
|
||||
else: # Assume JSON format
|
||||
try:
|
||||
lora_dict = json.loads(item)
|
||||
lora = LoRAModulePath(**lora_dict)
|
||||
lora_list.append(lora)
|
||||
except json.JSONDecodeError:
|
||||
parser.error(f"Invalid JSON format for --lora-modules: {item}")
|
||||
except TypeError as e:
|
||||
parser.error(
|
||||
f"Invalid fields for --lora-modules: {item} - {str(e)}"
|
||||
)
|
||||
setattr(namespace, self.dest, lora_list)
|
||||
|
||||
|
||||
@config
|
||||
class BaseFrontendArgs:
|
||||
"""Base arguments for the OpenAI-compatible frontend server.
|
||||
|
||||
This base class does not include host, port, and server-specific arguments
|
||||
like SSL, CORS, and HTTP server settings. Those arguments are added by
|
||||
the subclasses.
|
||||
"""
|
||||
|
||||
lora_modules: list[LoRAModulePath] | None = None
|
||||
"""LoRA modules configurations in either 'name=path' format or JSON format
|
||||
or JSON list format. Example (old format): `'name=path'` Example (new
|
||||
format): `{\"name\": \"name\", \"path\": \"lora_path\",
|
||||
\"base_model_name\": \"id\"}`"""
|
||||
chat_template: str | None = None
|
||||
"""The file path to the chat template, or the template in single-line form
|
||||
for the specified model."""
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
|
||||
"""The format to render message content within a chat template.
|
||||
|
||||
* "string" will render the content as a string. Example: `"Hello World"`
|
||||
* "openai" will render the content as a list of dictionaries, similar to
|
||||
OpenAI schema. Example: `[{"type": "text", "text": "Hello world!"}]`"""
|
||||
trust_request_chat_template: bool = False
|
||||
"""Whether to trust the chat template provided in the request. If False,
|
||||
the server will always use the chat template specified by `--chat-template`
|
||||
or the ones from tokenizer."""
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None
|
||||
"""Default keyword arguments to pass to the chat template renderer.
|
||||
These will be merged with request-level chat_template_kwargs,
|
||||
with request values taking precedence. Useful for setting default
|
||||
behavior for reasoning models. Example: '{"enable_thinking": false}'
|
||||
to disable thinking mode by default for Qwen3/DeepSeek models."""
|
||||
response_role: str = "assistant"
|
||||
"""The role name to return if `request.add_generation_prompt=true`."""
|
||||
return_tokens_as_token_ids: bool = False
|
||||
"""When `--max-logprobs` is specified, represents single tokens as
|
||||
strings of the form 'token_id:{token_id}' so that tokens that are not
|
||||
JSON-encodable can be identified."""
|
||||
enable_auto_tool_choice: bool = False
|
||||
"""Enable auto tool choice for supported models. Use `--tool-call-parser`
|
||||
to specify which parser to use."""
|
||||
exclude_tools_when_tool_choice_none: bool = False
|
||||
"""If specified, exclude tool definitions in prompts when
|
||||
tool_choice='none'."""
|
||||
tool_call_parser: str | None = None
|
||||
"""Select the tool call parser depending on the model that you're using.
|
||||
This is used to parse the model-generated tool call into OpenAI API format.
|
||||
Required for `--enable-auto-tool-choice`. You can choose any option from
|
||||
the built-in parsers or register a plugin via `--tool-parser-plugin`."""
|
||||
tool_parser_plugin: str = ""
|
||||
"""Special the tool parser plugin write to parse the model-generated tool
|
||||
into OpenAI API format, the name register in this plugin can be used in
|
||||
`--tool-call-parser`."""
|
||||
tool_server: str | None = None
|
||||
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
||||
built-in demo tools (browser and Python code interpreter). WARNING:
|
||||
The `demo` Python tool executes model-generated code in Docker without
|
||||
network isolation by default. See the security guide for more
|
||||
information."""
|
||||
log_config_file: str | None = envs.VLLM_LOGGING_CONFIG_PATH
|
||||
"""Path to logging config JSON file for both vllm and uvicorn"""
|
||||
max_log_len: int | None = None
|
||||
"""Max number of prompt characters or prompt ID numbers being printed in
|
||||
log. The default of None means unlimited."""
|
||||
enable_prompt_tokens_details: bool = False
|
||||
"""If set to True, enable prompt_tokens_details in usage."""
|
||||
enable_per_request_metrics: bool = False
|
||||
"""If set to True, include per-request timing metrics in API responses."""
|
||||
enable_server_load_tracking: bool = False
|
||||
"""If set to True, enable tracking server_load_metrics in the app state."""
|
||||
enable_force_include_usage: bool = False
|
||||
"""If set to True, including usage on every request."""
|
||||
sse_keep_alive_interval: int = 0
|
||||
"""Send an SSE keep-alive comment line every this many seconds when a
|
||||
`/v1/chat/completions` or `/v1/completions` streaming response is idle
|
||||
(queued, prefill, or between tokens), to prevent reverse proxies/tunnels
|
||||
with read timeouts from closing the connection. Defaults to 0, which
|
||||
disables keep-alive comments entirely.
|
||||
"""
|
||||
enable_tokenizer_info_endpoint: bool = False
|
||||
"""Enable the `/tokenizer_info` endpoint. May expose chat
|
||||
templates and other tokenizer configuration."""
|
||||
enable_log_outputs: bool = False
|
||||
"""If set to True, log model outputs (generations). Requires
|
||||
`--enable-log-requests`. Output text and finish reasons are logged at INFO,
|
||||
while output token IDs are logged at DEBUG."""
|
||||
enable_log_deltas: bool = True
|
||||
"""If set to False, output deltas will not be logged. Relevant only if
|
||||
--enable-log-outputs is set.
|
||||
"""
|
||||
cohere_is_reasoning_model: bool = True
|
||||
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
|
||||
reasoning Command-family model. When True (default), the assistant's
|
||||
chain-of-thought is surfaced as a ``thinking`` content block (or
|
||||
``content-*`` events on the stream). When False, reasoning is
|
||||
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
|
||||
events) whenever the model emits tool calls, matching older non-
|
||||
reasoning Command models. Has no effect on the non-Cohere
|
||||
endpoints."""
|
||||
cohere_format: str = "cmd4"
|
||||
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
|
||||
format to render: ``cmd4`` (current Command A+ models; default) or
|
||||
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
|
||||
wrong format silently produces a prompt the model wasn't trained
|
||||
on, which most commonly manifests as the model emitting text but no
|
||||
citations / tool calls / thinking blocks. Equivalent to passing
|
||||
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
|
||||
explicit request-level ``chat_template_kwargs.cohere_format`` takes
|
||||
priority."""
|
||||
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
"""
|
||||
If set to True, only enable the Tokens In<>Out endpoint.
|
||||
This is intended for use in a Disaggregated Everything setup.
|
||||
"""
|
||||
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
||||
"""Controls the ``system_fingerprint`` field on responses.
|
||||
|
||||
- ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes
|
||||
server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an
|
||||
8-char config hash.
|
||||
- ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped.
|
||||
- ``custom``: emits the literal string from ``--fingerprint-value``.
|
||||
- ``none``: the field is omitted (serialized as ``null``).
|
||||
"""
|
||||
fingerprint_value: str | None = None
|
||||
"""Literal fingerprint string used when ``--fingerprint-mode=custom``."""
|
||||
|
||||
@classmethod
|
||||
def _customize_cli_kwargs(
|
||||
cls,
|
||||
frontend_kwargs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Customize argparse kwargs before arguments are registered.
|
||||
|
||||
Subclasses should override this and call
|
||||
``super()._customize_cli_kwargs(frontend_kwargs)`` first.
|
||||
"""
|
||||
# Special case: default_chat_template_kwargs needs json.loads type
|
||||
frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
|
||||
|
||||
# Special case: LoRA modules need custom parser action and
|
||||
# optional_type(str)
|
||||
frontend_kwargs["lora_modules"]["type"] = optional_type(str)
|
||||
frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
|
||||
|
||||
# Special case: Tool call parser shows built-in options.
|
||||
valid_tool_parsers = list(ToolParserManager.list_registered())
|
||||
parsers_str = ",".join(valid_tool_parsers)
|
||||
frontend_kwargs["tool_call_parser"]["metavar"] = (
|
||||
f"{{{parsers_str}}} or name registered in --tool-parser-plugin"
|
||||
)
|
||||
return frontend_kwargs
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Register CLI arguments for this frontend class.
|
||||
|
||||
Subclasses should override ``_customize_cli_kwargs`` instead of
|
||||
this method so that base-class postprocessing is always applied.
|
||||
"""
|
||||
from vllm.engine.arg_utils import get_kwargs
|
||||
|
||||
frontend_kwargs = get_kwargs(cls)
|
||||
frontend_kwargs = cls._customize_cli_kwargs(frontend_kwargs)
|
||||
|
||||
group_name = cls.__name__.replace("Args", "")
|
||||
frontend_group = parser.add_argument_group(
|
||||
title=group_name,
|
||||
description=cls.__doc__,
|
||||
)
|
||||
for key, value in frontend_kwargs.items():
|
||||
extra_flags = value.pop("flags", [])
|
||||
frontend_group.add_argument(
|
||||
*extra_flags, f"--{key.replace('_', '-')}", **value
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@config
|
||||
class FrontendArgs(BaseFrontendArgs):
|
||||
"""Arguments for the OpenAI-compatible frontend server."""
|
||||
|
||||
host: str | None = None
|
||||
"""Host name."""
|
||||
port: int = 8000
|
||||
"""Port number."""
|
||||
data_parallel_supervisor_port: int = 9256
|
||||
"""HTTP port for aggregated health endpoints in multi-port external LB
|
||||
mode."""
|
||||
dp_supervisor_probe_interval_s: float = 5.0
|
||||
"""Seconds between aggregated health probes in multi-port external LB mode."""
|
||||
dp_supervisor_probe_timeout_s: float = 5.0
|
||||
"""Seconds to wait between retries when a child health probe fails with a
|
||||
connection error in multi-port external LB mode."""
|
||||
dp_supervisor_probe_failure_threshold: int = 3
|
||||
"""Number of consecutive connection-error retries before a child health
|
||||
probe is declared failed in multi-port external LB mode."""
|
||||
uds: str | None = None
|
||||
"""Unix domain socket path. If set, host and port arguments are ignored."""
|
||||
uvicorn_log_level: Literal[
|
||||
"critical", "error", "warning", "info", "debug", "trace"
|
||||
] = "info"
|
||||
"""Log level for uvicorn."""
|
||||
disable_uvicorn_access_log: bool = False
|
||||
"""Disable uvicorn access log."""
|
||||
disable_access_log_for_endpoints: str | None = None
|
||||
"""Comma-separated list of endpoint paths to exclude from uvicorn access
|
||||
logs. This is useful to reduce log noise from high-frequency endpoints
|
||||
like health checks. Example: "/health,/metrics,/ping".
|
||||
When set, access logs for requests to these paths will be suppressed
|
||||
while keeping logs for other endpoints."""
|
||||
allow_credentials: bool = False
|
||||
"""Allow credentials."""
|
||||
allowed_origins: list[str] = field(default_factory=lambda: ["*"])
|
||||
"""Allowed origins."""
|
||||
allowed_methods: list[str] = field(default_factory=lambda: ["*"])
|
||||
"""Allowed methods."""
|
||||
allowed_headers: list[str] = field(default_factory=lambda: ["*"])
|
||||
"""Allowed headers."""
|
||||
api_key: list[str] | None = None
|
||||
"""If provided, the server will require one of these keys to be presented in
|
||||
the header.
|
||||
|
||||
Warning: this only authenticates endpoints under the `/v1`, `/v2`, and
|
||||
`/inference` path prefixes. Other endpoints on the same server, including
|
||||
`/invocations` (which exposes the same inference capabilities as `/v1`),
|
||||
remain unauthenticated. Do not rely on `--api-key` alone to secure vLLM;
|
||||
see
|
||||
https://docs.vllm.ai/en/latest/usage/security.html#api-key-authentication-limitations
|
||||
for what it does and does not protect."""
|
||||
ssl_keyfile: str | None = None
|
||||
"""The file path to the SSL key file."""
|
||||
ssl_certfile: str | None = None
|
||||
"""The file path to the SSL cert file."""
|
||||
ssl_ca_certs: str | None = None
|
||||
"""The CA certificates file."""
|
||||
enable_ssl_refresh: bool = False
|
||||
"""Refresh SSL Context when SSL certificate files change"""
|
||||
ssl_cert_reqs: int = int(ssl.CERT_NONE)
|
||||
"""Whether client certificate is required (see stdlib ssl module's)."""
|
||||
ssl_ciphers: str | None = None
|
||||
"""SSL cipher suites for HTTPS (TLS 1.2 and below only).
|
||||
Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'"""
|
||||
root_path: str | None = None
|
||||
"""FastAPI root_path when app is behind a path based routing proxy."""
|
||||
middleware: list[str] = field(default_factory=lambda: [])
|
||||
"""Additional ASGI middleware to apply to the app. We accept multiple
|
||||
--middleware arguments. The value should be an import path. If a function
|
||||
is provided, vLLM will add it to the server using
|
||||
`@app.middleware('http')`. If a class is provided, vLLM will
|
||||
add it to the server using `app.add_middleware()`."""
|
||||
enable_request_id_headers: bool = False
|
||||
"""If specified, API server will add X-Request-Id header to responses."""
|
||||
disable_fastapi_docs: bool = False
|
||||
"""Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""
|
||||
h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
|
||||
"""Maximum size (bytes) of an incomplete HTTP event (header or body) for
|
||||
h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB)."""
|
||||
h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
|
||||
"""Maximum number of HTTP headers allowed in a request for h11 parser.
|
||||
Helps mitigate header abuse. Default: 256."""
|
||||
enable_offline_docs: bool = False
|
||||
"""
|
||||
Enable offline FastAPI documentation for air-gapped environments.
|
||||
Uses vendored static assets bundled with vLLM.
|
||||
"""
|
||||
enable_flash_late_interaction: bool = True
|
||||
"""If set, run pooling score MaxSim on GPU in the API server process.
|
||||
Can significantly improve late-interaction scoring performance."""
|
||||
|
||||
@classmethod
|
||||
def _customize_cli_kwargs(
|
||||
cls,
|
||||
frontend_kwargs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
|
||||
|
||||
# Special case: allowed_origins, allowed_methods, allowed_headers all
|
||||
# need json.loads type
|
||||
# Should also remove nargs
|
||||
frontend_kwargs["allowed_origins"]["type"] = json.loads
|
||||
frontend_kwargs["allowed_methods"]["type"] = json.loads
|
||||
frontend_kwargs["allowed_headers"]["type"] = json.loads
|
||||
del frontend_kwargs["allowed_origins"]["nargs"]
|
||||
del frontend_kwargs["allowed_methods"]["nargs"]
|
||||
del frontend_kwargs["allowed_headers"]["nargs"]
|
||||
|
||||
# Special case: Middleware needs to append action
|
||||
frontend_kwargs["middleware"]["action"] = "append"
|
||||
frontend_kwargs["middleware"]["type"] = str
|
||||
if "nargs" in frontend_kwargs["middleware"]:
|
||||
del frontend_kwargs["middleware"]["nargs"]
|
||||
frontend_kwargs["middleware"]["default"] = []
|
||||
|
||||
# Special case: disable_access_log_for_endpoints is a single
|
||||
# comma-separated string, not a list
|
||||
if "nargs" in frontend_kwargs["disable_access_log_for_endpoints"]:
|
||||
del frontend_kwargs["disable_access_log_for_endpoints"]["nargs"]
|
||||
|
||||
return frontend_kwargs
|
||||
|
||||
|
||||
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Create the CLI argument parser used by the OpenAI API server.
|
||||
|
||||
We rely on the helper methods of `FrontendArgs` and `AsyncEngineArgs` to
|
||||
register all arguments instead of manually enumerating them here. This
|
||||
avoids code duplication and keeps the argument definitions in one place.
|
||||
"""
|
||||
parser.add_argument(
|
||||
"model_tag",
|
||||
type=str,
|
||||
nargs="?",
|
||||
help="The model tag to serve (optional if specified in config)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--headless",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Run in headless mode. See multi-node data parallel "
|
||||
"documentation for more details.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-server-count",
|
||||
"-asc",
|
||||
type=int,
|
||||
default=None,
|
||||
help="How many API server processes to run. "
|
||||
"Defaults to data_parallel_size if not specified.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
help="Read CLI options from a config file. "
|
||||
"Must be a YAML with the following options: "
|
||||
"https://docs.vllm.ai/en/latest/configuration/serve_args.html",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--grpc",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Launch a gRPC server instead of the HTTP OpenAI-compatible "
|
||||
"server. Requires: pip install vllm[grpc].",
|
||||
)
|
||||
parser = FrontendArgs.add_cli_args(parser)
|
||||
parser = AsyncEngineArgs.add_cli_args(parser)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"""Quick checks for model serve args that raise prior to loading."""
|
||||
# `vllm launch <component>` builds its parser with make_arg_parser too (see
|
||||
# LaunchSubcommandBase.add_cli_args), so its args are serve args as well.
|
||||
if hasattr(args, "subparser") and args.subparser not in ("serve", "launch"):
|
||||
return
|
||||
|
||||
# Ensure that the chat template is valid; raises if it likely isn't
|
||||
validate_chat_template(args.chat_template)
|
||||
|
||||
# Enable auto tool needs a tool call parser to be valid
|
||||
if args.enable_auto_tool_choice and not args.tool_call_parser:
|
||||
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
|
||||
if args.enable_log_outputs and not args.enable_log_requests:
|
||||
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
|
||||
|
||||
# SSE keep-alive interval must be zero (disabled) or a positive integer.
|
||||
if getattr(args, "sse_keep_alive_interval", 0) < 0:
|
||||
raise ValueError(
|
||||
"Error: --sse-keep-alive-interval must be a non-negative integer "
|
||||
"(0 disables keep-alive comments)."
|
||||
)
|
||||
|
||||
if getattr(args, "enable_per_request_metrics", False) and getattr(
|
||||
args, "disable_log_stats", False
|
||||
):
|
||||
raise ValueError(
|
||||
"Error: --enable-per-request-metrics requires engine statistics "
|
||||
"logging; remove --disable-log-stats to enable per-request metrics."
|
||||
)
|
||||
|
||||
if args.data_parallel_multi_port_external_lb:
|
||||
from vllm.entrypoints.openai.dp_supervisor import (
|
||||
validate_multi_port_external_lb_args,
|
||||
)
|
||||
|
||||
validate_multi_port_external_lb_args(args)
|
||||
|
||||
|
||||
def create_parser_for_docs() -> FlexibleArgumentParser:
|
||||
parser_for_docs = FlexibleArgumentParser(
|
||||
prog="-m vllm.entrypoints.launchers.api_server.entry"
|
||||
)
|
||||
return make_arg_parser(parser_for_docs)
|
||||
@@ -1,571 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from functools import partial
|
||||
from http import HTTPStatus
|
||||
from multiprocessing.process import BaseProcess
|
||||
|
||||
import aiohttp
|
||||
import psutil
|
||||
import uvicorn
|
||||
import uvloop
|
||||
from fastapi import FastAPI, Response
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.launchers.launcher import NoSignalServer
|
||||
from vllm.entrypoints.launchers.utils.server_utils import get_uvicorn_log_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.system_utils import (
|
||||
decorate_logs,
|
||||
kill_process_tree,
|
||||
set_process_title,
|
||||
)
|
||||
from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
CHILD_EXIT_GRACE_S = 5.0
|
||||
|
||||
|
||||
def infer_multi_port_external_lb_start_rank(args: argparse.Namespace) -> int:
|
||||
start_rank = getattr(args, "data_parallel_start_rank", None)
|
||||
if start_rank is not None:
|
||||
return start_rank
|
||||
|
||||
node_rank = getattr(args, "node_rank", 0) or 0
|
||||
local_size = getattr(args, "data_parallel_size_local", 0) or 0
|
||||
return node_rank * local_size
|
||||
|
||||
|
||||
def validate_multi_port_external_lb_args(args: argparse.Namespace) -> None:
|
||||
if getattr(args, "grpc", False):
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb does not support --grpc"
|
||||
)
|
||||
if args.uds is not None:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb does not support --uds"
|
||||
)
|
||||
if bool(args.ssl_keyfile) != bool(args.ssl_certfile):
|
||||
raise ValueError(
|
||||
"Error: --ssl-keyfile and --ssl-certfile must be provided together"
|
||||
)
|
||||
if args.api_server_count not in (None, 1):
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb currently requires "
|
||||
"--api-server-count=1"
|
||||
)
|
||||
if args.data_parallel_rank is not None:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb manages child "
|
||||
"--data-parallel-rank values internally"
|
||||
)
|
||||
if args.data_parallel_external_lb or args.data_parallel_hybrid_lb:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb cannot be combined with "
|
||||
"--data-parallel-external-lb or --data-parallel-hybrid-lb"
|
||||
)
|
||||
if args.data_parallel_size < 2:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb requires "
|
||||
"--data-parallel-size > 1"
|
||||
)
|
||||
|
||||
local_size = args.data_parallel_size_local
|
||||
if local_size is None or local_size < 2:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb requires "
|
||||
"--data-parallel-size-local >= 2"
|
||||
)
|
||||
if local_size > args.data_parallel_size:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-size-local cannot exceed --data-parallel-size"
|
||||
)
|
||||
if args.data_parallel_size % local_size != 0:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-size must be divisible by "
|
||||
"--data-parallel-size-local"
|
||||
)
|
||||
|
||||
start_rank = infer_multi_port_external_lb_start_rank(args)
|
||||
if start_rank + local_size > args.data_parallel_size:
|
||||
raise ValueError(
|
||||
"Error: multi-port supervised ranks would exceed --data-parallel-size"
|
||||
)
|
||||
|
||||
supervisor_port = args.data_parallel_supervisor_port
|
||||
child_port_min = args.port
|
||||
child_port_max = args.port + local_size - 1
|
||||
if child_port_min <= supervisor_port <= child_port_max:
|
||||
raise ValueError(
|
||||
f"Error: --data-parallel-supervisor-port {supervisor_port} "
|
||||
f"overlaps with child rank ports {child_port_min}-{child_port_max}"
|
||||
)
|
||||
|
||||
|
||||
def _build_vllm_dp_server_args(
|
||||
args: argparse.Namespace, local_rank: int
|
||||
) -> argparse.Namespace:
|
||||
child_args = copy.copy(args)
|
||||
child_args.port = args.port + local_rank
|
||||
child_args.data_parallel_rank = (
|
||||
infer_multi_port_external_lb_start_rank(args) + local_rank
|
||||
)
|
||||
child_args.data_parallel_start_rank = None
|
||||
child_args.data_parallel_size_local = 1
|
||||
child_args.data_parallel_external_lb = True
|
||||
child_args.data_parallel_hybrid_lb = False
|
||||
child_args.data_parallel_multi_port_external_lb = False
|
||||
child_args.data_parallel_supervisor_port = None
|
||||
child_args.api_server_count = 1
|
||||
child_args.device_ids = _build_device_ids(args, local_rank)
|
||||
return child_args
|
||||
|
||||
|
||||
def _build_device_ids(args: argparse.Namespace, local_rank: int) -> list[int | str]:
|
||||
"""Build the --device-ids value for a DP child process.
|
||||
|
||||
The child resolves these against its own inherited device-control env
|
||||
var (e.g. CUDA_VISIBLE_DEVICES), so integer IDs must stay env-relative
|
||||
here rather than being translated to physical IDs.
|
||||
"""
|
||||
devices_per_rank = args.tensor_parallel_size * args.pipeline_parallel_size
|
||||
start = local_rank * devices_per_rank
|
||||
stop = start + devices_per_rank
|
||||
device_ids = getattr(args, "device_ids", None)
|
||||
if device_ids is not None:
|
||||
if stop > len(device_ids):
|
||||
raise ValueError(
|
||||
f"--device-ids has {len(device_ids)} entries, but DP rank "
|
||||
f"{local_rank} needs devices [{start}, {stop})"
|
||||
)
|
||||
return device_ids[start:stop]
|
||||
return list(range(start, stop))
|
||||
|
||||
|
||||
def _child_base_url(args: argparse.Namespace, port: int) -> str:
|
||||
host = args.host or "127.0.0.1"
|
||||
if host == "0.0.0.0":
|
||||
host = "127.0.0.1"
|
||||
elif host == "::":
|
||||
host = "::1"
|
||||
scheme = "https" if args.ssl_keyfile and args.ssl_certfile else "http"
|
||||
return f"{scheme}://{host}:{port}"
|
||||
|
||||
|
||||
def _join_processes_with_timeout(processes: list[BaseProcess], timeout: float) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
for process in processes:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
process.join(timeout=remaining)
|
||||
|
||||
|
||||
async def _probe_endpoint(
|
||||
session: aiohttp.ClientSession,
|
||||
args: argparse.Namespace,
|
||||
port: int,
|
||||
path: str,
|
||||
conn_err_failure_threshold: int = 3,
|
||||
conn_err_retry_delay: float = 5.0,
|
||||
) -> bool:
|
||||
"""
|
||||
Probe /health endpoint for 200 status.
|
||||
|
||||
If there is a connection error, retry every N seconds.
|
||||
"""
|
||||
for iteration in range(conn_err_failure_threshold):
|
||||
try:
|
||||
probe_ssl = None
|
||||
if args.ssl_keyfile and args.ssl_certfile:
|
||||
# Probes target node-local child servers over loopback, so skip
|
||||
# certificate verification to avoid SAN/hostname mismatches for
|
||||
# localhost/127.0.0.1 deployments.
|
||||
probe_ssl = False
|
||||
async with session.get(
|
||||
_child_base_url(args, port) + path, ssl=probe_ssl
|
||||
) as response:
|
||||
# vLLM returns 503 on EngineDeadError, so we should return
|
||||
# immediately if vLLM responds with a non-200 status code.
|
||||
return response.status == HTTPStatus.OK
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError) as e:
|
||||
# Allow retry of connection errors.
|
||||
logger.debug(
|
||||
"Probe attempt %d/%d failed on port %d: %r",
|
||||
iteration + 1,
|
||||
conn_err_failure_threshold,
|
||||
port,
|
||||
e,
|
||||
)
|
||||
|
||||
if iteration < conn_err_failure_threshold - 1:
|
||||
await asyncio.sleep(conn_err_retry_delay)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:
|
||||
app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
|
||||
app.state.supervisor = supervisor
|
||||
|
||||
def _status_response(ok: bool) -> Response:
|
||||
return Response(
|
||||
status_code=(HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE)
|
||||
)
|
||||
|
||||
@app.get("/health", include_in_schema=False)
|
||||
async def health() -> Response:
|
||||
return _status_response(app.state.supervisor.is_ready)
|
||||
|
||||
@app.get("/ready", include_in_schema=False)
|
||||
@app.get("/readyz", include_in_schema=False)
|
||||
async def ready() -> Response:
|
||||
return _status_response(app.state.supervisor.is_ready)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
from vllm.entrypoints.launchers.api_server.entry import run_server
|
||||
|
||||
uvloop.run(run_server(child_args))
|
||||
|
||||
|
||||
def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
from vllm.entrypoints.cli.serve import run_multi_api_server
|
||||
|
||||
run_multi_api_server(child_args)
|
||||
|
||||
|
||||
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
"""
|
||||
Entrypoint function for the vLLM DP Server.
|
||||
"""
|
||||
# Create a fresh process group for the vLLM DP Server,
|
||||
# so that CTRL-C is propagated cleanly.
|
||||
os.setpgrp()
|
||||
|
||||
name = f"APIServer_DP{child_args.data_parallel_rank}"
|
||||
set_process_title(name)
|
||||
decorate_logs(name)
|
||||
if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH:
|
||||
_run_rust_vllm_dp_server(child_args)
|
||||
else:
|
||||
_run_python_vllm_dp_server(child_args)
|
||||
|
||||
|
||||
class DPSupervisor:
|
||||
def __init__(self, args: argparse.Namespace):
|
||||
validate_multi_port_external_lb_args(args)
|
||||
self.args = args
|
||||
self.supervisor_port = args.data_parallel_supervisor_port
|
||||
self.child_ports = [
|
||||
args.port + local_rank
|
||||
for local_rank in range(args.data_parallel_size_local)
|
||||
]
|
||||
self._is_ready = False
|
||||
self._processes: list[BaseProcess] = []
|
||||
self._shutdown_event = asyncio.Event()
|
||||
self._shutdown_signal = signal.SIGTERM
|
||||
|
||||
@property
|
||||
def is_ready(self) -> bool:
|
||||
return self._is_ready and not self._shutdown_event.is_set()
|
||||
|
||||
async def run(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
decorate_logs("DPSupervisor")
|
||||
|
||||
# K8s sends SIGTERM for shutdown - begin graceful termination.
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, partial(self._handle_signal, sig))
|
||||
|
||||
supervisor_server: uvicorn.Server | None = None
|
||||
supervisor_server_task: asyncio.Task[None] | None = None
|
||||
try:
|
||||
# Launch vLLM DP Servers and begin monitoring them.
|
||||
self._start_children()
|
||||
monitor_task = asyncio.create_task(
|
||||
self._monitor_children(), name="dp-monitor"
|
||||
)
|
||||
|
||||
# Only start the DPSupervisor server once the vLLM DP Servers are
|
||||
# ready. This avoids the supervisor /health endpoint returning 503
|
||||
# to external load balancer probes while the engines initialize.
|
||||
await self._wait_until_ready(monitor_task)
|
||||
if self.is_ready and not monitor_task.done():
|
||||
supervisor_server, supervisor_server_task = await self._start_server()
|
||||
|
||||
await monitor_task
|
||||
finally:
|
||||
self._is_ready = False
|
||||
await self._shutdown_children()
|
||||
|
||||
# Shutdown the DP Supervisor server.
|
||||
if supervisor_server is not None and supervisor_server_task is not None:
|
||||
supervisor_server.should_exit = True
|
||||
await supervisor_server_task
|
||||
|
||||
async def _start_server(self) -> tuple[uvicorn.Server, asyncio.Task[None]]:
|
||||
"""
|
||||
Launch the DPSupervisor HTTP server.
|
||||
|
||||
Called only after the vLLM DP Servers are ready so that /health does
|
||||
not return 503 to external probes while the engines are initializing.
|
||||
"""
|
||||
app = _build_dp_supervisor_app(self)
|
||||
host = self.args.host or "0.0.0.0"
|
||||
|
||||
uvicorn_kwargs: dict = {}
|
||||
log_config = get_uvicorn_log_config(self.args)
|
||||
if log_config is not None:
|
||||
uvicorn_kwargs["log_config"] = log_config
|
||||
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host=host,
|
||||
port=self.supervisor_port,
|
||||
log_level=self.args.uvicorn_log_level,
|
||||
access_log=not self.args.disable_uvicorn_access_log,
|
||||
ssl_keyfile=self.args.ssl_keyfile,
|
||||
ssl_certfile=self.args.ssl_certfile,
|
||||
ssl_ca_certs=self.args.ssl_ca_certs,
|
||||
ssl_cert_reqs=self.args.ssl_cert_reqs,
|
||||
ssl_ciphers=self.args.ssl_ciphers,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
supervisor_server = NoSignalServer(config)
|
||||
supervisor_server_task = asyncio.create_task(
|
||||
supervisor_server.serve(),
|
||||
name="dp-supervisor",
|
||||
)
|
||||
supervisor_server_task.add_done_callback(
|
||||
lambda _task: self._shutdown_event.set()
|
||||
)
|
||||
|
||||
# Ensure DPSupervisor task starts on the event loop.
|
||||
while not supervisor_server.started:
|
||||
if supervisor_server_task.done():
|
||||
supervisor_server_task.result()
|
||||
raise RuntimeError("DPSupervisor exited before startup.")
|
||||
await asyncio.sleep(0.05)
|
||||
logger.info("Started DPSupervisor on %s:%d", host, self.supervisor_port)
|
||||
return supervisor_server, supervisor_server_task
|
||||
|
||||
async def _wait_until_ready(self, monitor_task: asyncio.Task[None]) -> None:
|
||||
"""
|
||||
Block until the vLLM DP Servers are ready or shutdown is triggered.
|
||||
|
||||
Returns early if monitoring stops (e.g. a DP Server dies during
|
||||
startup), in which case the supervisor server is never started.
|
||||
"""
|
||||
logger.info("Waiting for vLLM DP Servers to become ready.")
|
||||
while not self._is_ready and not self._shutdown_event.is_set():
|
||||
if monitor_task.done():
|
||||
return
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
def _handle_signal(self, signum: int) -> None:
|
||||
"""
|
||||
Signal handler that is added to the event loop.
|
||||
|
||||
This catches the SIGTERM from K8s and begins graceful shutdown,
|
||||
by setting the _shutdown_event(), which is watched by the main
|
||||
coroutine monitoring the vLLM DP Servers.
|
||||
"""
|
||||
|
||||
if self._shutdown_event.is_set():
|
||||
return
|
||||
|
||||
self._shutdown_signal = signal.Signals(signum)
|
||||
logger.info(
|
||||
"DPSupervisor received %s, shutting down.",
|
||||
self._shutdown_signal.name,
|
||||
)
|
||||
|
||||
self._shutdown_event.set()
|
||||
self._is_ready = False
|
||||
|
||||
def _start_children(self) -> None:
|
||||
"""
|
||||
Launch vLLM DP Servers on separate GPUs.
|
||||
"""
|
||||
logger.info("Launching vLLM DP Servers")
|
||||
context = multiprocessing.get_context("spawn")
|
||||
for local_rank in range(self.args.data_parallel_size_local):
|
||||
child_args = _build_vllm_dp_server_args(self.args, local_rank)
|
||||
process = context.Process(
|
||||
target=_run_vllm_dp_server,
|
||||
name=f"APIServer_DPRank_{child_args.data_parallel_rank}",
|
||||
args=(child_args,),
|
||||
)
|
||||
process.start()
|
||||
self._processes.append(process)
|
||||
|
||||
async def _probe_all_children(self) -> None:
|
||||
"""
|
||||
Background coroutine: probes all child endpoints on each interval.
|
||||
|
||||
Exits when any server becomes unhealthy after being ready, signalling
|
||||
_monitor_children to initiate shutdown.
|
||||
"""
|
||||
timeout = aiohttp.ClientTimeout(total=self.args.dp_supervisor_probe_timeout_s)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
while not self._shutdown_event.is_set():
|
||||
threshold = (
|
||||
self.args.dp_supervisor_probe_failure_threshold
|
||||
if self._is_ready
|
||||
else 1
|
||||
)
|
||||
results = await asyncio.gather(
|
||||
*(
|
||||
_probe_endpoint(
|
||||
session,
|
||||
self.args,
|
||||
port,
|
||||
"/health",
|
||||
conn_err_failure_threshold=threshold,
|
||||
conn_err_retry_delay=self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
for port in self.child_ports
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
all_healthy = all(r is True for r in results)
|
||||
|
||||
if all_healthy:
|
||||
# If all healthy, we are ready to receive requests.
|
||||
# This conditional avoids a potential race condition
|
||||
# where shutdown is set, THEN the probe returns true.
|
||||
if not self._shutdown_event.is_set():
|
||||
self._is_ready = True
|
||||
elif self._is_ready:
|
||||
# Once ready, any failure in the probe means vLLM is dead.
|
||||
num_unhealthy = sum(1 for r in results if r is not True)
|
||||
logger.info(
|
||||
"DPSupervisor probe found %s unhealthy DP Servers.",
|
||||
num_unhealthy,
|
||||
)
|
||||
self._is_ready = False
|
||||
self._shutdown_event.set()
|
||||
return
|
||||
|
||||
with contextlib.suppress(asyncio.TimeoutError):
|
||||
logger.debug(
|
||||
"Waiting for %s seconds before next probe",
|
||||
self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
await asyncio.wait_for(
|
||||
self._shutdown_event.wait(),
|
||||
timeout=self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
|
||||
async def _monitor_children(self) -> None:
|
||||
"""
|
||||
Main coroutine task that monitors the children vLLM servers.
|
||||
|
||||
Before the vLLM servers are /ready:
|
||||
- if the pid is dead, we will shut down
|
||||
- if the probe fails, we try again after dp_supervisor_probe_interval_s
|
||||
|
||||
After the vLLM servers are /ready:
|
||||
- if the pid is dead, we will shut down
|
||||
- if the probe fails, we will shut down
|
||||
"""
|
||||
probe_task = asyncio.create_task(
|
||||
self._probe_all_children(), name="dp-health-probe"
|
||||
)
|
||||
|
||||
try:
|
||||
while not self._shutdown_event.is_set():
|
||||
# 1. Check for dead processes
|
||||
n_failed = len([p for p in self._processes if not p.is_alive()])
|
||||
if n_failed > 0:
|
||||
logger.info("DPSupervisor found %s exited DP Servers.", n_failed)
|
||||
break
|
||||
|
||||
# 2. Check if the probe background task crashed or failed.
|
||||
if probe_task.done():
|
||||
# Extract exception if it crashed, or log failure
|
||||
exc = probe_task.exception() if not probe_task.cancelled() else None
|
||||
logger.info("DPSupervisor probe task stopped. Exception: %s", exc)
|
||||
break
|
||||
|
||||
# Sleep for probe_interval seconds or until a shutdown.
|
||||
with contextlib.suppress(asyncio.TimeoutError):
|
||||
logger.debug(
|
||||
"Waiting for %s seconds before next monitor",
|
||||
self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
await asyncio.wait_for(
|
||||
self._shutdown_event.wait(),
|
||||
timeout=self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
|
||||
finally:
|
||||
# Cleanup probe task if needed.
|
||||
if not probe_task.done():
|
||||
probe_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await probe_task
|
||||
|
||||
async def _shutdown_children(self) -> None:
|
||||
"""Terminate the vLLM DP servers."""
|
||||
process_timeout = get_engine_process_shutdown_timeout(
|
||||
self.args.shutdown_timeout, self.args.shutdown_timeout
|
||||
)
|
||||
assert process_timeout is not None
|
||||
timeout = process_timeout + CHILD_EXIT_GRACE_S
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
"DPSupervisor forwarding %s to DP Servers.",
|
||||
self._shutdown_signal.name,
|
||||
)
|
||||
for process in self._processes:
|
||||
pid = process.pid
|
||||
if not process.is_alive() or pid is None:
|
||||
continue
|
||||
with contextlib.suppress(ProcessLookupError, OSError):
|
||||
os.kill(pid, self._shutdown_signal)
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
_join_processes_with_timeout, self._processes, timeout
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
logger.warning("Shutdown await cancelled")
|
||||
raise
|
||||
finally:
|
||||
for process in self._processes:
|
||||
pid = process.pid
|
||||
if not process.is_alive() or pid is None:
|
||||
continue
|
||||
logger.warning(
|
||||
"DP server %s did not exit within %.1fs; force killing.",
|
||||
process.name,
|
||||
timeout,
|
||||
)
|
||||
with contextlib.suppress(
|
||||
ProcessLookupError,
|
||||
OSError,
|
||||
psutil.NoSuchProcess,
|
||||
psutil.AccessDenied,
|
||||
):
|
||||
kill_process_tree(pid)
|
||||
|
||||
|
||||
def run_dp_supervisor(args: argparse.Namespace) -> None:
|
||||
uvloop.run(DPSupervisor(args).run())
|
||||
Reference in New Issue
Block a user