[Frontend] DP Supervisor (#40841)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: robertgshaw2-redhat <robertgshaw2@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Wentao Ye
2026-05-21 20:33:16 -07:00
committed by GitHub
co-authored by robertgshaw2-redhat Robert Shaw Nick Hill
parent 60af5c16ee
commit 0ddd7dd656
7 changed files with 1168 additions and 17 deletions
@@ -0,0 +1,610 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for DPSupervisor: unit tests and lifecycle integration tests.
Lifecycle integration tests replace child vLLM servers with lightweight
aiohttp "fake" servers controlled by the test, so the suite runs without GPUs.
_start_children is monkeypatched to install FakeProcess objects (with
controllable liveness/timing) alongside those fake HTTP servers.
Port allocation (kept far from default vLLM ports to avoid conflicts):
Supervisor : 19256
Children : 18000, 18001
"""
from __future__ import annotations
import argparse
import asyncio
import contextlib
import os
import signal
import time
from types import SimpleNamespace
import aiohttp
import pytest
import uvicorn
from fastapi import FastAPI, Response
import vllm.entrypoints.openai.dp_supervisor as dp_sup
from vllm.entrypoints.openai.dp_supervisor import (
CHILD_EXIT_GRACE_S,
DPSupervisor,
_build_vllm_dp_server_args,
infer_multi_port_external_lb_start_rank,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_SUPERVISOR_PORT = 19256
_CHILD_PORT_BASE = 18000
_N_CHILDREN = 2
_PROBE_INTERVAL = 1.0
_POLL_INTERVAL = 1.0
# ---------------------------------------------------------------------------
# Args factories
# ---------------------------------------------------------------------------
def _make_unit_args(**overrides) -> argparse.Namespace:
"""Minimal args for unit tests (no real network activity)."""
base = {
"host": None,
"port": 8000,
"data_parallel_multi_port_external_lb": True,
"data_parallel_supervisor_port": 9256,
"dp_supervisor_probe_interval_s": 5.0,
"dp_supervisor_probe_timeout_s": 5.0,
"dp_supervisor_probe_failure_threshold": 3,
"data_parallel_size": 8,
"data_parallel_size_local": 4,
"data_parallel_start_rank": None,
"data_parallel_rank": None,
"data_parallel_external_lb": False,
"data_parallel_hybrid_lb": False,
"api_server_count": None,
"headless": False,
"grpc": False,
"uds": None,
"ssl_keyfile": None,
"ssl_certfile": None,
"ssl_ca_certs": None,
"node_rank": 1,
"tensor_parallel_size": 1,
"pipeline_parallel_size": 1,
"uvicorn_log_level": "info",
"shutdown_timeout": 5.0,
}
base.update(overrides)
return argparse.Namespace(**base)
def _make_args(**overrides) -> argparse.Namespace:
"""Args for lifecycle integration tests (real loopback servers)."""
base: dict = dict(
host="127.0.0.1",
port=_CHILD_PORT_BASE,
data_parallel_multi_port_external_lb=True,
data_parallel_supervisor_port=_SUPERVISOR_PORT,
dp_supervisor_probe_interval_s=_PROBE_INTERVAL,
dp_supervisor_probe_timeout_s=1.0,
dp_supervisor_probe_failure_threshold=3,
data_parallel_size=_N_CHILDREN,
data_parallel_size_local=_N_CHILDREN,
data_parallel_start_rank=0,
data_parallel_rank=None,
data_parallel_external_lb=False,
data_parallel_hybrid_lb=False,
api_server_count=None,
headless=False,
grpc=False,
uds=None,
ssl_keyfile=None,
ssl_certfile=None,
ssl_ca_certs=None,
node_rank=0,
tensor_parallel_size=1,
pipeline_parallel_size=1,
uvicorn_log_level="warning",
shutdown_timeout=0.0,
)
base.update(overrides)
return argparse.Namespace(**base)
# ---------------------------------------------------------------------------
# Unit tests
# ---------------------------------------------------------------------------
def test_infer_multi_port_external_lb_start_rank_uses_node_rank():
args = _make_unit_args()
assert infer_multi_port_external_lb_start_rank(args) == 4
def test_build_multi_port_external_lb_child_args_sets_external_rank_server():
args = _make_unit_args(data_parallel_start_rank=8, api_server_count=None)
child_args = _build_vllm_dp_server_args(args, local_rank=2)
assert child_args.port == 8002
assert child_args.data_parallel_rank == 10
assert child_args.data_parallel_size_local == 1
assert child_args.data_parallel_external_lb is True
assert child_args.data_parallel_hybrid_lb is False
assert child_args.data_parallel_multi_port_external_lb is False
assert child_args.api_server_count == 1
def test_aggregates_health():
supervisor = DPSupervisor(_make_unit_args())
supervisor._is_ready = True
assert supervisor.is_ready is True
def test_handles_shutdown_event():
supervisor = DPSupervisor(_make_unit_args())
supervisor._is_ready = True
supervisor._shutdown_event.set()
assert supervisor.is_ready is False
@pytest.mark.asyncio
async def test_handles_child_exit(
monkeypatch: pytest.MonkeyPatch,
):
supervisor = DPSupervisor(_make_unit_args())
supervisor._processes = [
SimpleNamespace(
name="APIServer_DPRank_4", exitcode=None, is_alive=lambda: True
),
SimpleNamespace(name="APIServer_DPRank_5", exitcode=17, is_alive=lambda: False),
]
async def fake_probe(*_args, **_kwargs) -> bool:
return True
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
await supervisor._monitor_children()
assert supervisor._is_ready is False
@pytest.mark.asyncio
async def test_handles_probe_failure(
monkeypatch: pytest.MonkeyPatch,
):
supervisor = DPSupervisor(_make_unit_args(dp_supervisor_probe_interval_s=0.0))
supervisor.child_ports = [8000]
probe_results = iter([True, False])
async def fake_probe(*_args, **_kwargs) -> bool:
return next(probe_results)
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
await supervisor._monitor_children()
assert supervisor._is_ready is False
@pytest.mark.asyncio
async def test_shutdown_if_supervisor_server_error_on_startup(
monkeypatch: pytest.MonkeyPatch,
):
class FakeLoop:
def add_signal_handler(self, *_args, **_kwargs):
pass
def remove_signal_handler(self, *_args, **_kwargs):
pass
class FakeServer:
def __init__(self, _config):
self.started = False
self.should_exit = False
async def serve(self):
raise ValueError("supervisor boom")
async def fake_shutdown_children(self):
return None
monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop())
monkeypatch.setattr(dp_sup.uvicorn, "Server", FakeServer)
monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children)
supervisor = DPSupervisor(_make_unit_args())
with pytest.raises(ValueError, match="supervisor boom"):
await supervisor.run()
# ---------------------------------------------------------------------------
# Lifecycle integration tests – MockVLLMServer
# ---------------------------------------------------------------------------
class MockVLLMServer:
"""
Minimal FastAPI server that mimics one vLLM replica.
GET /health returns 200 when healthy, 503 otherwise.
Health state is toggled by the test via set_healthy().
"""
def __init__(self, port: int, drain_seconds: float = 0.0) -> None:
self.port = port
self._healthy = False
self._drain_seconds = drain_seconds
self._server: uvicorn.Server | None = None
self._serve_task: asyncio.Task | None = None
async def start(self) -> None:
app = FastAPI()
@app.get("/health")
async def health() -> Response:
print(f"MockServer {self.port}: /health: {self._healthy}")
return Response(status_code=200 if self._healthy else 503)
@app.get("/set_healthy")
async def set_healthy() -> Response:
print(f"MockServer {self.port}: /set_healthy")
self._healthy = True
return Response(status_code=200)
@app.get("/set_unhealthy")
async def set_unhealthy() -> Response:
print(f"MockServer {self.port}: /set_unhealthy")
self._healthy = False
return Response(status_code=200)
@app.get("/kill")
async def kill() -> Response:
print(f"MockServer {self.port}: /kill")
os.kill(os.getpid(), signal.SIGKILL)
config = uvicorn.Config(
app,
host="127.0.0.1",
port=self.port,
log_level="warning",
lifespan="off",
)
self._server = uvicorn.Server(config)
# Configure request draining if needed.
# Uvicorn's capture_signals() installs signal.signal(SIGTERM, self.handle_exit),
# which sets should_exit=True immediately. Override handle_exit on the instance
# so capture_signals() picks up our version that drains first.
if self._drain_seconds > 0:
self._shutdown_event = asyncio.Event()
loop = asyncio.get_running_loop()
async def _drain_and_stop() -> None:
await self._shutdown_event.wait()
print(f"MockServer {self.port}: draining for {self._drain_seconds}s.")
await asyncio.sleep(self._drain_seconds)
print("Setting should_exit")
if self._server is not None:
self._server.should_exit = True
self._drain_task = asyncio.create_task(_drain_and_stop())
def _custom_handle_exit(sig: int, frame: object) -> None:
print("Got SIGTERM, setting shutdown.")
if not self._shutdown_event.is_set():
loop.call_soon_threadsafe(self._shutdown_event.set)
self._server.handle_exit = _custom_handle_exit
self._serve_task = asyncio.create_task(self._server.serve())
while not self._server.started:
await asyncio.sleep(0.01)
print(f"Mock DP Server on port {self.port} started")
await self._serve_task
def launch_mock_vllm(child_args: argparse.Namespace, env_updates: dict[str, str]):
logger.info("Launching mock vLLM on port %s", child_args.port)
mock_vllm = MockVLLMServer(port=child_args.port)
asyncio.run(mock_vllm.start())
def launch_mock_vllm_with_drain(
child_args: argparse.Namespace, env_updates: dict[str, str]
):
logger.info("Launching mock vLLM with 15s drain on port %s", child_args.port)
mock_vllm = MockVLLMServer(port=child_args.port, drain_seconds=10.0)
asyncio.run(mock_vllm.start())
# ---------------------------------------------------------------------------
# Lifecycle test helpers
# ---------------------------------------------------------------------------
async def _poll_supervisor_health(expected_status: int) -> bool:
"""
Poll GET /health on the supervisor until expected_status is seen.
A connection error is treated as 503-equivalent when expected_status != 200.
"""
url = f"http://127.0.0.1:{_SUPERVISOR_PORT}/health"
async with aiohttp.ClientSession() as session:
try:
async with session.get(url) as resp:
if resp.status != expected_status:
print(f"expected: {expected_status=}, got: {resp.status=}")
return False
return True
except aiohttp.ClientError:
if expected_status != -1:
print(f"expected: {expected_status=}, got: aiohttp.ClientError")
return False
return True
async def _poll_until_api_server_running(port: int, retries: int = 10) -> None:
url = f"http://127.0.0.1:{port}/health"
async with aiohttp.ClientSession() as session:
for _ in range(retries):
try:
async with session.get(url) as resp:
if resp.status != 200:
return
await asyncio.sleep(1.0)
except aiohttp.ClientError:
print("Test detected not started yet, sleeping for 1s")
await asyncio.sleep(1.0)
async def _set_healthy(port: int) -> None:
url = f"http://127.0.0.1:{port}/set_healthy"
async with aiohttp.ClientSession() as session, session.get(url) as resp:
assert resp.status == 200
async def _set_unhealthy(port: int) -> None:
url = f"http://127.0.0.1:{port}/set_unhealthy"
async with aiohttp.ClientSession() as session, session.get(url) as resp:
assert resp.status == 200
async def _kill_server(port: int) -> None:
url = f"http://127.0.0.1:{port}/kill"
try:
async with aiohttp.ClientSession() as session, session.get(url) as resp:
assert resp.status != 200
except Exception as e:
assert isinstance(e, aiohttp.ClientConnectorError)
@contextlib.asynccontextmanager
async def _run_supervisor(
args: argparse.Namespace,
monkeypatch: pytest.MonkeyPatch,
launch_fn=None,
):
if launch_fn is None:
launch_fn = launch_mock_vllm
monkeypatch.setattr(dp_sup, "_run_vllm_dp_server", launch_fn)
supervisor = DPSupervisor(args)
task = asyncio.create_task(supervisor.run())
await asyncio.sleep(1.0)
try:
yield supervisor, task
finally:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
# ---------------------------------------------------------------------------
# Lifecycle integration tests
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_basic_lifecycle(monkeypatch):
"""
A) Supervisor /health returns 503 while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) SIGTERM and shutdown
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
for port in vllm_server_ports:
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
await _poll_until_api_server_running(port)
await _set_healthy(vllm_server_ports[0])
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
print("/health is 503 --- expected!")
for port in vllm_server_ports:
await _set_healthy(port)
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
os.kill(os.getpid(), signal.SIGTERM)
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
print("everything was cleaned up!")
@pytest.mark.asyncio
async def test_failed_startup(monkeypatch):
"""
A) One of the vLLM servers crashes during startup.
B) DPSupervisor detects this, and cleans up resources.
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
for port in vllm_server_ports:
await _poll_until_api_server_running(port)
await _kill_server(port)
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
@pytest.mark.asyncio
async def test_becomes_unhealthy(monkeypatch):
"""
A) Supervisor /health returns 503 while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) Child process becomes unhealthy.
D) Detected and shutdown.
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
for port in vllm_server_ports:
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
await _poll_until_api_server_running(port)
await _set_healthy(vllm_server_ports[0])
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
print("/health is 503 --- expected!")
for port in vllm_server_ports:
await _set_healthy(port)
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
await _set_unhealthy(port)
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
print("everything was cleaned up!")
@pytest.mark.asyncio
async def test_dp_server_fails(monkeypatch):
"""
A) Supervisor /health returns 503 while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) Child process fails.
D) Detected and shutdown.
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
for port in vllm_server_ports:
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
await _poll_until_api_server_running(port)
await _set_healthy(vllm_server_ports[0])
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(503)
assert not supervisor.is_ready
print("/health is 503 --- expected!")
for port in vllm_server_ports:
await _set_healthy(port)
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
dp_mock_server_process = supervisor._processes[0]
os.kill(dp_mock_server_process.pid, signal.SIGKILL)
await asyncio.sleep(1.0)
assert not dp_mock_server_process.is_alive()
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
print("everything was cleaned up!")
@pytest.mark.asyncio
async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch):
"""
Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
request drain). The supervisor is configured with shutdown_timeout=10,
so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The
children exit naturally within that window, so no force-kill should occur
and the measured wall-clock time must be >= 10s.
"""
_DRAIN_SECONDS = 10.0
_SHUTDOWN_TIMEOUT = 10.0
args = _make_args(shutdown_timeout=_SHUTDOWN_TIMEOUT)
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(
args, monkeypatch, launch_fn=launch_mock_vllm_with_drain
) as (supervisor, _task):
for port in vllm_server_ports:
await _poll_until_api_server_running(port)
for port in vllm_server_ports:
await _set_healthy(port)
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(200)
assert supervisor.is_ready
start_t = time.perf_counter()
os.kill(os.getpid(), signal.SIGTERM)
print(f"DRAINING FOR {_DRAIN_SECONDS}")
await asyncio.wait_for(_task, timeout=_DRAIN_SECONDS + CHILD_EXIT_GRACE_S + 5.0)
elapsed = time.perf_counter() - start_t
assert elapsed >= _DRAIN_SECONDS, (
f"Supervisor exited after only {elapsed:.1f}s; "
f"expected >= {_DRAIN_SECONDS}s for request draining"
)
for p in supervisor._processes:
assert not p.is_alive()
print(f"Supervisor waited {elapsed:.1f}s for children to drain — expected!")
+10
View File
@@ -476,6 +476,7 @@ class EngineArgs:
data_parallel_rpc_port: int | None = None
data_parallel_hybrid_lb: bool = False
data_parallel_external_lb: bool = False
data_parallel_multi_port_external_lb: bool = False
data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter
@@ -1043,6 +1044,15 @@ class EngineArgs:
"-dpe",
**parallel_kwargs["data_parallel_external_lb"],
)
parallel_group.add_argument(
"--data-parallel-multi-port-external-lb",
"-dpm",
action="store_true",
default=False,
help="Run a node-local supervisor that launches one external-LB API "
"server per local data parallel rank and exposes aggregated health on "
"a supervisor port.",
)
parallel_group.add_argument(
"--enable-expert-parallel",
"-ep",
+30 -14
View File
@@ -12,6 +12,9 @@ import vllm.envs as envs
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.openai.api_server import run_server, setup_server
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
from vllm.entrypoints.openai.dp_supervisor import (
run_dp_supervisor,
)
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.usage.usage_lib import UsageContext
@@ -66,31 +69,41 @@ class ServeSubcommand(CLISubcommand):
args.api_server_count = 0
# Detect LB mode for defaulting api_server_count.
# Multi-port: --data-parallel-multi-port-external-lb
# External LB: --data-parallel-external-lb or --data-parallel-rank
# Hybrid LB: --data-parallel-hybrid-lb or --data-parallel-start-rank
is_external_lb = (
args.data_parallel_external_lb or args.data_parallel_rank is not None
)
is_hybrid_lb = (
args.data_parallel_hybrid_lb or args.data_parallel_start_rank is not None
)
if is_external_lb and is_hybrid_lb:
# If --data_parallel_multi_port_external_lb and --data_parallel_hybrid_lb
# are unset, default to hybrid if --data-parallel-start-rank is set
is_hybrid_lb = is_multi_port = False
if (
not args.data_parallel_hybrid_lb
and not args.data_parallel_multi_port_external_lb
):
is_hybrid_lb = args.data_parallel_start_rank is not None
else:
is_hybrid_lb = args.data_parallel_hybrid_lb
is_multi_port = args.data_parallel_multi_port_external_lb
if sum([is_multi_port, is_external_lb, is_hybrid_lb]) > 1:
raise ValueError(
"Cannot use both external and hybrid data parallel load "
"balancing modes. External LB is enabled via "
"--data-parallel-external-lb or --data-parallel-rank. "
"Hybrid LB is enabled via --data-parallel-hybrid-lb or "
"--data-parallel-start-rank. Use one mode or the other."
"Cannot use more than one data parallel load balancing mode. "
"Choose one of: --data-parallel-multi-port-external-lb, "
"--data-parallel-external-lb (or --data-parallel-rank), "
"--data-parallel-hybrid-lb (or --data-parallel-start-rank)."
)
# Default api_server_count if not explicitly set.
# - Rust frontend: Use 1 (not applicable as it's multithreaded)
# - External LB: Leave as 1 (external LB handles distribution)
# - Multi-port: 1 (supervisor spawns one server per local DP rank)
# - Rust frontend: 1 (not applicable as it's multithreaded)
# - External LB: 1 (external LB handles distribution)
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
# - Internal LB: Use full DP size
if args.api_server_count is None:
if is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
args.api_server_count = 1
elif is_hybrid_lb:
args.api_server_count = args.data_parallel_size_local or 1
@@ -109,7 +122,8 @@ class ServeSubcommand(CLISubcommand):
)
elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
logger.warning(
"Ignoring --api-server-count=%d when using rust front-end process"
"Ignoring --api-server-count=%d when using rust front-end process",
args.api_server_count,
)
args.api_server_count = 1
@@ -122,7 +136,9 @@ class ServeSubcommand(CLISubcommand):
)
args.api_server_count = 1
if args.api_server_count < 1:
if is_multi_port:
run_dp_supervisor(args)
elif args.api_server_count < 1:
run_headless(args)
elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
run_multi_api_server(args)
+1 -2
View File
@@ -665,8 +665,7 @@ async def build_and_serve_renderer(
async def run_server(args, **uvicorn_kwargs) -> None:
"""Run a single-worker API server."""
# Add process-specific prefix to stdout and stderr.
decorate_logs("APIServer")
decorate_logs("APIServer", skip_if_decorated=True)
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
# own signal handlers. Once uvicorn is running it replaces this.
+18
View File
@@ -229,6 +229,17 @@ class FrontendArgs(BaseFrontendArgs):
"""Host name."""
port: int = 8000
"""Port number."""
data_parallel_supervisor_port: int = 9256
"""HTTP port for aggregated health endpoints in multi-port external LB
mode."""
dp_supervisor_probe_interval_s: float = 5.0
"""Seconds between aggregated health probes in multi-port external LB mode."""
dp_supervisor_probe_timeout_s: float = 5.0
"""Seconds to wait between retries when a child health probe fails with a
connection error in multi-port external LB mode."""
dp_supervisor_probe_failure_threshold: int = 3
"""Number of consecutive connection-error retries before a child health
probe is declared failed in multi-port external LB mode."""
uds: str | None = None
"""Unix domain socket path. If set, host and port arguments are ignored."""
uvicorn_log_level: Literal[
@@ -387,6 +398,13 @@ def validate_parsed_serve_args(args: argparse.Namespace):
if args.enable_log_outputs and not args.enable_log_requests:
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
if args.data_parallel_multi_port_external_lb:
from vllm.entrypoints.openai.dp_supervisor import (
validate_multi_port_external_lb_args,
)
validate_multi_port_external_lb_args(args)
def create_parser_for_docs() -> FlexibleArgumentParser:
parser_for_docs = FlexibleArgumentParser(
+493
View File
@@ -0,0 +1,493 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import asyncio
import contextlib
import copy
import multiprocessing
import os
import signal
import time
from functools import partial
from http import HTTPStatus
from multiprocessing.process import BaseProcess
import aiohttp
import psutil
import uvicorn
import uvloop
from fastapi import FastAPI, Response
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.system_utils import (
decorate_logs,
kill_process_tree,
set_process_title,
update_environment_variables,
)
logger = init_logger(__name__)
CHILD_EXIT_GRACE_S = 5.0
def infer_multi_port_external_lb_start_rank(args: argparse.Namespace) -> int:
start_rank = getattr(args, "data_parallel_start_rank", None)
if start_rank is not None:
return start_rank
node_rank = getattr(args, "node_rank", 0) or 0
local_size = getattr(args, "data_parallel_size_local", 0) or 0
return node_rank * local_size
def validate_multi_port_external_lb_args(args: argparse.Namespace) -> None:
if getattr(args, "grpc", False):
raise ValueError(
"Error: --data-parallel-multi-port-external-lb does not support --grpc"
)
if args.uds is not None:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb does not support --uds"
)
if any((args.ssl_keyfile, args.ssl_certfile, args.ssl_ca_certs)):
raise ValueError(
"Error: --data-parallel-multi-port-external-lb does not support HTTPS yet"
)
if args.api_server_count not in (None, 1):
raise ValueError(
"Error: --data-parallel-multi-port-external-lb currently requires "
"--api-server-count=1"
)
if args.data_parallel_rank is not None:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb manages child "
"--data-parallel-rank values internally"
)
if args.data_parallel_external_lb or args.data_parallel_hybrid_lb:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb cannot be combined with "
"--data-parallel-external-lb or --data-parallel-hybrid-lb"
)
if args.data_parallel_size < 2:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb requires "
"--data-parallel-size > 1"
)
local_size = args.data_parallel_size_local
if local_size is None or local_size < 2:
raise ValueError(
"Error: --data-parallel-multi-port-external-lb requires "
"--data-parallel-size-local >= 2"
)
if local_size > args.data_parallel_size:
raise ValueError(
"Error: --data-parallel-size-local cannot exceed --data-parallel-size"
)
if args.data_parallel_size % local_size != 0:
raise ValueError(
"Error: --data-parallel-size must be divisible by "
"--data-parallel-size-local"
)
start_rank = infer_multi_port_external_lb_start_rank(args)
if start_rank + local_size > args.data_parallel_size:
raise ValueError(
"Error: multi-port supervised ranks would exceed --data-parallel-size"
)
supervisor_port = args.data_parallel_supervisor_port
child_port_min = args.port
child_port_max = args.port + local_size - 1
if child_port_min <= supervisor_port <= child_port_max:
raise ValueError(
f"Error: --data-parallel-supervisor-port {supervisor_port} "
f"overlaps with child rank ports {child_port_min}-{child_port_max}"
)
def _build_vllm_dp_server_args(
args: argparse.Namespace, local_rank: int
) -> argparse.Namespace:
child_args = copy.copy(args)
child_args.port = args.port + local_rank
child_args.data_parallel_rank = (
infer_multi_port_external_lb_start_rank(args) + local_rank
)
child_args.data_parallel_start_rank = None
child_args.data_parallel_size_local = 1
child_args.data_parallel_external_lb = True
child_args.data_parallel_hybrid_lb = False
child_args.data_parallel_multi_port_external_lb = False
child_args.data_parallel_supervisor_port = None
child_args.api_server_count = 1
return child_args
def _build_vllm_dp_server_env(
args: argparse.Namespace, local_rank: int
) -> dict[str, str]:
# set visible devices for the child process
devices_per_rank = args.tensor_parallel_size * args.pipeline_parallel_size
start = local_rank * devices_per_rank
stop = start + devices_per_rank
device_env = current_platform.device_control_env_var
visible_devices = ",".join(
str(current_platform.device_id_to_physical_device_id(idx))
for idx in range(start, stop)
)
return {device_env: visible_devices}
def _child_base_url(args: argparse.Namespace, port: int) -> str:
host = args.host or "127.0.0.1"
if host == "0.0.0.0":
host = "127.0.0.1"
elif host == "::":
host = "::1"
return f"http://{host}:{port}"
def _join_processes_with_timeout(processes: list[BaseProcess], timeout: float) -> None:
deadline = time.monotonic() + timeout
for process in processes:
remaining = deadline - time.monotonic()
if remaining <= 0:
break
process.join(timeout=remaining)
async def _probe_endpoint(
session: aiohttp.ClientSession,
args: argparse.Namespace,
port: int,
path: str,
conn_err_failure_threshold: int = 3,
conn_err_retry_delay: float = 5.0,
) -> bool:
"""
Probe /health endpoint for 200 status.
If there is a connection error, retry every N seconds.
"""
for iteration in range(conn_err_failure_threshold):
try:
async with session.get(_child_base_url(args, port) + path) as response:
# vLLM returns 503 on EngineDeadError, so we should return
# immediately if vLLM responds with a non-200 status code.
return response.status == HTTPStatus.OK
except (aiohttp.ClientError, asyncio.TimeoutError) as e:
# Allow retry of connection errors.
logger.debug(
"Probe attempt %d/%d failed on port %d: %r",
iteration + 1,
conn_err_failure_threshold,
port,
e,
)
if iteration < conn_err_failure_threshold - 1:
await asyncio.sleep(conn_err_retry_delay)
return False
def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:
app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
app.state.supervisor = supervisor
def _status_response(ok: bool) -> Response:
return Response(
status_code=(HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE)
)
@app.get("/health", include_in_schema=False)
async def health() -> Response:
return _status_response(app.state.supervisor.is_ready)
@app.get("/ready", include_in_schema=False)
@app.get("/readyz", include_in_schema=False)
async def ready() -> Response:
return _status_response(app.state.supervisor.is_ready)
return app
def _run_vllm_dp_server(
child_args: argparse.Namespace, env_updates: dict[str, str]
) -> None:
"""
Entrypoint function for the vLLM DP Server.
"""
from vllm.entrypoints.openai.api_server import run_server
# Create a fresh process group for the vLLM DP Server,
# so that CTRL-C is propagated cleanly.
os.setpgrp()
name = f"APIServer_DP{child_args.data_parallel_rank}"
update_environment_variables(env_updates)
set_process_title(name)
decorate_logs(name)
uvloop.run(run_server(child_args))
class DPSupervisor:
def __init__(self, args: argparse.Namespace):
validate_multi_port_external_lb_args(args)
self.args = args
self.supervisor_port = args.data_parallel_supervisor_port
self.child_ports = [
args.port + local_rank
for local_rank in range(args.data_parallel_size_local)
]
self._is_ready = False
self._processes: list[BaseProcess] = []
self._shutdown_event = asyncio.Event()
self._shutdown_signal = signal.SIGTERM
@property
def is_ready(self) -> bool:
return self._is_ready and not self._shutdown_event.is_set()
async def run(self) -> None:
loop = asyncio.get_running_loop()
# K8s sends SIGTERM for shutdown - begin graceful termination.
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, partial(self._handle_signal, sig))
# Launch DPSupervisor Server.
app = _build_dp_supervisor_app(self)
decorate_logs("DPSupervisor")
host = self.args.host or "0.0.0.0"
config = uvicorn.Config(
app,
host=host,
port=self.supervisor_port,
log_level=self.args.uvicorn_log_level,
)
supervisor_server = uvicorn.Server(config)
supervisor_server_task = asyncio.create_task(
supervisor_server.serve(),
name="dp-supervisor",
)
supervisor_server_task.add_done_callback(
lambda _task: self._shutdown_event.set()
)
# Ensure DPSupervisor task starts on the event loop.
while not supervisor_server.started:
if supervisor_server_task.done():
supervisor_server_task.result()
raise RuntimeError("DPSupervisor exited before startup.")
await asyncio.sleep(0.05)
logger.info("Started DPSupervisor on %s:%d", host, self.supervisor_port)
# Launch and Monitor vLLM Server Processes.
try:
self._start_children()
await self._monitor_children()
finally:
self._is_ready = False
await self._shutdown_children()
# Shutdown the DP Supervisor server.
supervisor_server.should_exit = True
await supervisor_server_task
def _handle_signal(self, signum: int) -> None:
"""
Signal handler that is added to the event loop.
This catches the SIGTERM from K8s and begins graceful shutdown,
by setting the _shutdown_event(), which is watched by the main
coroutine monitoring the vLLM DP Servers.
"""
if self._shutdown_event.is_set():
return
self._shutdown_signal = signal.Signals(signum)
logger.info(
"DPSupervisor received %s, shutting down.",
self._shutdown_signal.name,
)
self._shutdown_event.set()
self._is_ready = False
def _start_children(self) -> None:
"""
Launch vLLM DP Servers on separate GPUs.
"""
logger.info("Launching vLLM DP Servers")
context = multiprocessing.get_context("spawn")
for local_rank in range(self.args.data_parallel_size_local):
child_args = _build_vllm_dp_server_args(self.args, local_rank)
child_env = _build_vllm_dp_server_env(self.args, local_rank)
process = context.Process(
target=_run_vllm_dp_server,
name=f"APIServer_DPRank_{child_args.data_parallel_rank}",
args=(child_args, child_env),
)
process.start()
self._processes.append(process)
async def _probe_all_children(self) -> None:
"""
Background coroutine: probes all child endpoints on each interval.
Exits when any server becomes unhealthy after being ready, signalling
_monitor_children to initiate shutdown.
"""
timeout = aiohttp.ClientTimeout(total=self.args.dp_supervisor_probe_timeout_s)
async with aiohttp.ClientSession(timeout=timeout) as session:
while not self._shutdown_event.is_set():
threshold = (
self.args.dp_supervisor_probe_failure_threshold
if self._is_ready
else 1
)
results = await asyncio.gather(
*(
_probe_endpoint(
session,
self.args,
port,
"/health",
conn_err_failure_threshold=threshold,
conn_err_retry_delay=self.args.dp_supervisor_probe_interval_s,
)
for port in self.child_ports
),
return_exceptions=True,
)
all_healthy = all(r is True for r in results)
if all_healthy:
# If all healthy, we are ready to receive requests.
# This conditional avoids a potential race condition
# where shutdown is set, THEN the probe returns true.
if not self._shutdown_event.is_set():
self._is_ready = True
elif self._is_ready:
# Once ready, any failure in the probe means vLLM is dead.
num_unhealthy = sum(1 for r in results if r is not True)
logger.info(
"DPSupervisor probe found %s unhealthy DP Servers.",
num_unhealthy,
)
self._is_ready = False
self._shutdown_event.set()
return
with contextlib.suppress(asyncio.TimeoutError):
logger.debug(
"Waiting for %s seconds before next probe",
self.args.dp_supervisor_probe_interval_s,
)
await asyncio.wait_for(
self._shutdown_event.wait(),
timeout=self.args.dp_supervisor_probe_interval_s,
)
async def _monitor_children(self) -> None:
"""
Main coroutine task that monitors the children vLLM servers.
Before the vLLM servers are /ready:
- if the pid is dead, we will shut down
- if the probe fails, we try again after dp_supervisor_probe_interval_s
After the vLLM servers are /ready:
- if the pid is dead, we will shut down
- if the probe fails, we will shut down
"""
probe_task = asyncio.create_task(
self._probe_all_children(), name="dp-health-probe"
)
try:
while not self._shutdown_event.is_set():
# 1. Check for dead processes
n_failed = len([p for p in self._processes if not p.is_alive()])
if n_failed > 0:
logger.info("DPSupervisor found %s exited DP Servers.", n_failed)
break
# 2. Check if the probe background task crashed or failed.
if probe_task.done():
# Extract exception if it crashed, or log failure
exc = probe_task.exception() if not probe_task.cancelled() else None
logger.info("DPSupervisor probe task stopped. Exception: %s", exc)
break
# Sleep for probe_interval seconds or until a shutdown.
with contextlib.suppress(asyncio.TimeoutError):
logger.debug(
"Waiting for %s seconds before next monitor",
self.args.dp_supervisor_probe_interval_s,
)
await asyncio.wait_for(
self._shutdown_event.wait(),
timeout=self.args.dp_supervisor_probe_interval_s,
)
finally:
# Cleanup probe task if needed.
if not probe_task.done():
probe_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await probe_task
async def _shutdown_children(self) -> None:
"""Terminate the vLLM DP servers."""
timeout = self.args.shutdown_timeout + CHILD_EXIT_GRACE_S
try:
logger.info(
"DPSupervisor forwarding %s to DP Servers.",
self._shutdown_signal.name,
)
for process in self._processes:
pid = process.pid
if not process.is_alive() or pid is None:
continue
with contextlib.suppress(ProcessLookupError, OSError):
os.kill(pid, self._shutdown_signal)
try:
await asyncio.to_thread(
_join_processes_with_timeout, self._processes, timeout
)
except asyncio.CancelledError:
logger.warning("Shutdown await cancelled")
raise
finally:
for process in self._processes:
pid = process.pid
if not process.is_alive() or pid is None:
continue
logger.warning(
"DP server %s did not exit within %.1fs; force killing.",
process.name,
timeout,
)
with contextlib.suppress(
ProcessLookupError,
OSError,
psutil.NoSuchProcess,
psutil.AccessDenied,
):
kill_process_tree(pid)
def run_dp_supervisor(args: argparse.Namespace) -> None:
uvloop.run(DPSupervisor(args).run())
+6 -1
View File
@@ -234,12 +234,17 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
file.write = write_with_prefix # type: ignore[method-assign]
def decorate_logs(process_name: str | None = None) -> None:
def decorate_logs(
process_name: str | None = None, *, skip_if_decorated: bool = False
) -> None:
"""Decorate stdout/stderr with process name and PID prefix."""
# Respect VLLM_CONFIGURE_LOGGING environment variable
if not envs.VLLM_CONFIGURE_LOGGING:
return
if skip_if_decorated and hasattr(sys.stdout, "_original_write"):
return
if process_name is None:
process_name = get_mp_context().current_process().name