From 0ddd7dd6564f5e403a15bd7c973c7d358ec82454 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 21 May 2026 23:33:16 -0400 Subject: [PATCH] [Frontend] DP Supervisor (#40841) Signed-off-by: yewentao256 Signed-off-by: Robert Shaw Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: robertgshaw2-redhat Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Nick Hill --- .../entrypoints/openai/test_dp_supervisor.py | 610 ++++++++++++++++++ vllm/engine/arg_utils.py | 10 + vllm/entrypoints/cli/serve.py | 44 +- vllm/entrypoints/openai/api_server.py | 3 +- vllm/entrypoints/openai/cli_args.py | 18 + vllm/entrypoints/openai/dp_supervisor.py | 493 ++++++++++++++ vllm/utils/system_utils.py | 7 +- 7 files changed, 1168 insertions(+), 17 deletions(-) create mode 100644 tests/entrypoints/openai/test_dp_supervisor.py create mode 100644 vllm/entrypoints/openai/dp_supervisor.py diff --git a/tests/entrypoints/openai/test_dp_supervisor.py b/tests/entrypoints/openai/test_dp_supervisor.py new file mode 100644 index 0000000000..0b678b2262 --- /dev/null +++ b/tests/entrypoints/openai/test_dp_supervisor.py @@ -0,0 +1,610 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Tests for DPSupervisor: unit tests and lifecycle integration tests. + +Lifecycle integration tests replace child vLLM servers with lightweight +aiohttp "fake" servers controlled by the test, so the suite runs without GPUs. +_start_children is monkeypatched to install FakeProcess objects (with +controllable liveness/timing) alongside those fake HTTP servers. + +Port allocation (kept far from default vLLM ports to avoid conflicts): + Supervisor : 19256 + Children : 18000, 18001 +""" + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import os +import signal +import time +from types import SimpleNamespace + +import aiohttp +import pytest +import uvicorn +from fastapi import FastAPI, Response + +import vllm.entrypoints.openai.dp_supervisor as dp_sup +from vllm.entrypoints.openai.dp_supervisor import ( + CHILD_EXIT_GRACE_S, + DPSupervisor, + _build_vllm_dp_server_args, + infer_multi_port_external_lb_start_rank, +) +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_SUPERVISOR_PORT = 19256 +_CHILD_PORT_BASE = 18000 +_N_CHILDREN = 2 +_PROBE_INTERVAL = 1.0 +_POLL_INTERVAL = 1.0 + + +# --------------------------------------------------------------------------- +# Args factories +# --------------------------------------------------------------------------- + + +def _make_unit_args(**overrides) -> argparse.Namespace: + """Minimal args for unit tests (no real network activity).""" + base = { + "host": None, + "port": 8000, + "data_parallel_multi_port_external_lb": True, + "data_parallel_supervisor_port": 9256, + "dp_supervisor_probe_interval_s": 5.0, + "dp_supervisor_probe_timeout_s": 5.0, + "dp_supervisor_probe_failure_threshold": 3, + "data_parallel_size": 8, + "data_parallel_size_local": 4, + "data_parallel_start_rank": None, + "data_parallel_rank": None, + "data_parallel_external_lb": False, + "data_parallel_hybrid_lb": False, + "api_server_count": None, + "headless": False, + "grpc": False, + "uds": None, + "ssl_keyfile": None, + "ssl_certfile": None, + "ssl_ca_certs": None, + "node_rank": 1, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "uvicorn_log_level": "info", + "shutdown_timeout": 5.0, + } + base.update(overrides) + return argparse.Namespace(**base) + + +def _make_args(**overrides) -> argparse.Namespace: + """Args for lifecycle integration tests (real loopback servers).""" + base: dict = dict( + host="127.0.0.1", + port=_CHILD_PORT_BASE, + data_parallel_multi_port_external_lb=True, + data_parallel_supervisor_port=_SUPERVISOR_PORT, + dp_supervisor_probe_interval_s=_PROBE_INTERVAL, + dp_supervisor_probe_timeout_s=1.0, + dp_supervisor_probe_failure_threshold=3, + data_parallel_size=_N_CHILDREN, + data_parallel_size_local=_N_CHILDREN, + data_parallel_start_rank=0, + data_parallel_rank=None, + data_parallel_external_lb=False, + data_parallel_hybrid_lb=False, + api_server_count=None, + headless=False, + grpc=False, + uds=None, + ssl_keyfile=None, + ssl_certfile=None, + ssl_ca_certs=None, + node_rank=0, + tensor_parallel_size=1, + pipeline_parallel_size=1, + uvicorn_log_level="warning", + shutdown_timeout=0.0, + ) + base.update(overrides) + return argparse.Namespace(**base) + + +# --------------------------------------------------------------------------- +# Unit tests +# --------------------------------------------------------------------------- + + +def test_infer_multi_port_external_lb_start_rank_uses_node_rank(): + args = _make_unit_args() + assert infer_multi_port_external_lb_start_rank(args) == 4 + + +def test_build_multi_port_external_lb_child_args_sets_external_rank_server(): + args = _make_unit_args(data_parallel_start_rank=8, api_server_count=None) + child_args = _build_vllm_dp_server_args(args, local_rank=2) + + assert child_args.port == 8002 + assert child_args.data_parallel_rank == 10 + assert child_args.data_parallel_size_local == 1 + assert child_args.data_parallel_external_lb is True + assert child_args.data_parallel_hybrid_lb is False + assert child_args.data_parallel_multi_port_external_lb is False + assert child_args.api_server_count == 1 + + +def test_aggregates_health(): + supervisor = DPSupervisor(_make_unit_args()) + supervisor._is_ready = True + assert supervisor.is_ready is True + + +def test_handles_shutdown_event(): + supervisor = DPSupervisor(_make_unit_args()) + supervisor._is_ready = True + supervisor._shutdown_event.set() + assert supervisor.is_ready is False + + +@pytest.mark.asyncio +async def test_handles_child_exit( + monkeypatch: pytest.MonkeyPatch, +): + supervisor = DPSupervisor(_make_unit_args()) + supervisor._processes = [ + SimpleNamespace( + name="APIServer_DPRank_4", exitcode=None, is_alive=lambda: True + ), + SimpleNamespace(name="APIServer_DPRank_5", exitcode=17, is_alive=lambda: False), + ] + + async def fake_probe(*_args, **_kwargs) -> bool: + return True + + monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe) + + await supervisor._monitor_children() + assert supervisor._is_ready is False + + +@pytest.mark.asyncio +async def test_handles_probe_failure( + monkeypatch: pytest.MonkeyPatch, +): + supervisor = DPSupervisor(_make_unit_args(dp_supervisor_probe_interval_s=0.0)) + supervisor.child_ports = [8000] + probe_results = iter([True, False]) + + async def fake_probe(*_args, **_kwargs) -> bool: + return next(probe_results) + + monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe) + + await supervisor._monitor_children() + assert supervisor._is_ready is False + + +@pytest.mark.asyncio +async def test_shutdown_if_supervisor_server_error_on_startup( + monkeypatch: pytest.MonkeyPatch, +): + class FakeLoop: + def add_signal_handler(self, *_args, **_kwargs): + pass + + def remove_signal_handler(self, *_args, **_kwargs): + pass + + class FakeServer: + def __init__(self, _config): + self.started = False + self.should_exit = False + + async def serve(self): + raise ValueError("supervisor boom") + + async def fake_shutdown_children(self): + return None + + monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop()) + monkeypatch.setattr(dp_sup.uvicorn, "Server", FakeServer) + monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children) + + supervisor = DPSupervisor(_make_unit_args()) + + with pytest.raises(ValueError, match="supervisor boom"): + await supervisor.run() + + +# --------------------------------------------------------------------------- +# Lifecycle integration tests – MockVLLMServer +# --------------------------------------------------------------------------- + + +class MockVLLMServer: + """ + Minimal FastAPI server that mimics one vLLM replica. + GET /health returns 200 when healthy, 503 otherwise. + Health state is toggled by the test via set_healthy(). + """ + + def __init__(self, port: int, drain_seconds: float = 0.0) -> None: + self.port = port + self._healthy = False + self._drain_seconds = drain_seconds + self._server: uvicorn.Server | None = None + self._serve_task: asyncio.Task | None = None + + async def start(self) -> None: + app = FastAPI() + + @app.get("/health") + async def health() -> Response: + print(f"MockServer {self.port}: /health: {self._healthy}") + return Response(status_code=200 if self._healthy else 503) + + @app.get("/set_healthy") + async def set_healthy() -> Response: + print(f"MockServer {self.port}: /set_healthy") + self._healthy = True + return Response(status_code=200) + + @app.get("/set_unhealthy") + async def set_unhealthy() -> Response: + print(f"MockServer {self.port}: /set_unhealthy") + self._healthy = False + return Response(status_code=200) + + @app.get("/kill") + async def kill() -> Response: + print(f"MockServer {self.port}: /kill") + os.kill(os.getpid(), signal.SIGKILL) + + config = uvicorn.Config( + app, + host="127.0.0.1", + port=self.port, + log_level="warning", + lifespan="off", + ) + self._server = uvicorn.Server(config) + + # Configure request draining if needed. + # Uvicorn's capture_signals() installs signal.signal(SIGTERM, self.handle_exit), + # which sets should_exit=True immediately. Override handle_exit on the instance + # so capture_signals() picks up our version that drains first. + if self._drain_seconds > 0: + self._shutdown_event = asyncio.Event() + loop = asyncio.get_running_loop() + + async def _drain_and_stop() -> None: + await self._shutdown_event.wait() + print(f"MockServer {self.port}: draining for {self._drain_seconds}s.") + await asyncio.sleep(self._drain_seconds) + print("Setting should_exit") + if self._server is not None: + self._server.should_exit = True + + self._drain_task = asyncio.create_task(_drain_and_stop()) + + def _custom_handle_exit(sig: int, frame: object) -> None: + print("Got SIGTERM, setting shutdown.") + if not self._shutdown_event.is_set(): + loop.call_soon_threadsafe(self._shutdown_event.set) + + self._server.handle_exit = _custom_handle_exit + + self._serve_task = asyncio.create_task(self._server.serve()) + while not self._server.started: + await asyncio.sleep(0.01) + print(f"Mock DP Server on port {self.port} started") + + await self._serve_task + + +def launch_mock_vllm(child_args: argparse.Namespace, env_updates: dict[str, str]): + logger.info("Launching mock vLLM on port %s", child_args.port) + mock_vllm = MockVLLMServer(port=child_args.port) + asyncio.run(mock_vllm.start()) + + +def launch_mock_vllm_with_drain( + child_args: argparse.Namespace, env_updates: dict[str, str] +): + logger.info("Launching mock vLLM with 15s drain on port %s", child_args.port) + mock_vllm = MockVLLMServer(port=child_args.port, drain_seconds=10.0) + asyncio.run(mock_vllm.start()) + + +# --------------------------------------------------------------------------- +# Lifecycle test helpers +# --------------------------------------------------------------------------- + + +async def _poll_supervisor_health(expected_status: int) -> bool: + """ + Poll GET /health on the supervisor until expected_status is seen. + A connection error is treated as 503-equivalent when expected_status != 200. + """ + url = f"http://127.0.0.1:{_SUPERVISOR_PORT}/health" + async with aiohttp.ClientSession() as session: + try: + async with session.get(url) as resp: + if resp.status != expected_status: + print(f"expected: {expected_status=}, got: {resp.status=}") + return False + return True + except aiohttp.ClientError: + if expected_status != -1: + print(f"expected: {expected_status=}, got: aiohttp.ClientError") + return False + return True + + +async def _poll_until_api_server_running(port: int, retries: int = 10) -> None: + url = f"http://127.0.0.1:{port}/health" + async with aiohttp.ClientSession() as session: + for _ in range(retries): + try: + async with session.get(url) as resp: + if resp.status != 200: + return + await asyncio.sleep(1.0) + except aiohttp.ClientError: + print("Test detected not started yet, sleeping for 1s") + await asyncio.sleep(1.0) + + +async def _set_healthy(port: int) -> None: + url = f"http://127.0.0.1:{port}/set_healthy" + async with aiohttp.ClientSession() as session, session.get(url) as resp: + assert resp.status == 200 + + +async def _set_unhealthy(port: int) -> None: + url = f"http://127.0.0.1:{port}/set_unhealthy" + async with aiohttp.ClientSession() as session, session.get(url) as resp: + assert resp.status == 200 + + +async def _kill_server(port: int) -> None: + url = f"http://127.0.0.1:{port}/kill" + try: + async with aiohttp.ClientSession() as session, session.get(url) as resp: + assert resp.status != 200 + except Exception as e: + assert isinstance(e, aiohttp.ClientConnectorError) + + +@contextlib.asynccontextmanager +async def _run_supervisor( + args: argparse.Namespace, + monkeypatch: pytest.MonkeyPatch, + launch_fn=None, +): + if launch_fn is None: + launch_fn = launch_mock_vllm + monkeypatch.setattr(dp_sup, "_run_vllm_dp_server", launch_fn) + supervisor = DPSupervisor(args) + task = asyncio.create_task(supervisor.run()) + await asyncio.sleep(1.0) + try: + yield supervisor, task + finally: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +# --------------------------------------------------------------------------- +# Lifecycle integration tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_basic_lifecycle(monkeypatch): + """ + A) Supervisor /health returns 503 while children are unhealthy. + B) /health returns 200 once every child reports healthy. + C) SIGTERM and shutdown + """ + args = _make_args() + + vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)] + + async with _run_supervisor(args, monkeypatch) as (supervisor, _task): + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + + for port in vllm_server_ports: + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + await _poll_until_api_server_running(port) + + await _set_healthy(vllm_server_ports[0]) + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + print("/health is 503 --- expected!") + + for port in vllm_server_ports: + await _set_healthy(port) + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(200) + assert supervisor.is_ready + print("/health is 200 --- expected!") + + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(200) + assert supervisor.is_ready + print("/health is 200 --- expected!") + + os.kill(os.getpid(), signal.SIGTERM) + + await asyncio.wait_for(_task, timeout=5.0) + for p in supervisor._processes: + assert not p.is_alive() + print("everything was cleaned up!") + + +@pytest.mark.asyncio +async def test_failed_startup(monkeypatch): + """ + A) One of the vLLM servers crashes during startup. + B) DPSupervisor detects this, and cleans up resources. + """ + args = _make_args() + + vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)] + + async with _run_supervisor(args, monkeypatch) as (supervisor, _task): + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + + for port in vllm_server_ports: + await _poll_until_api_server_running(port) + + await _kill_server(port) + + await asyncio.wait_for(_task, timeout=5.0) + for p in supervisor._processes: + assert not p.is_alive() + + +@pytest.mark.asyncio +async def test_becomes_unhealthy(monkeypatch): + """ + A) Supervisor /health returns 503 while children are unhealthy. + B) /health returns 200 once every child reports healthy. + C) Child process becomes unhealthy. + D) Detected and shutdown. + """ + args = _make_args() + + vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)] + + async with _run_supervisor(args, monkeypatch) as (supervisor, _task): + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + + for port in vllm_server_ports: + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + await _poll_until_api_server_running(port) + + await _set_healthy(vllm_server_ports[0]) + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + print("/health is 503 --- expected!") + + for port in vllm_server_ports: + await _set_healthy(port) + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(200) + assert supervisor.is_ready + print("/health is 200 --- expected!") + + await _set_unhealthy(port) + + await asyncio.wait_for(_task, timeout=5.0) + for p in supervisor._processes: + assert not p.is_alive() + print("everything was cleaned up!") + + +@pytest.mark.asyncio +async def test_dp_server_fails(monkeypatch): + """ + A) Supervisor /health returns 503 while children are unhealthy. + B) /health returns 200 once every child reports healthy. + C) Child process fails. + D) Detected and shutdown. + """ + args = _make_args() + + vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)] + + async with _run_supervisor(args, monkeypatch) as (supervisor, _task): + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + + for port in vllm_server_ports: + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + await _poll_until_api_server_running(port) + + await _set_healthy(vllm_server_ports[0]) + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(503) + assert not supervisor.is_ready + print("/health is 503 --- expected!") + + for port in vllm_server_ports: + await _set_healthy(port) + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(200) + assert supervisor.is_ready + print("/health is 200 --- expected!") + + dp_mock_server_process = supervisor._processes[0] + os.kill(dp_mock_server_process.pid, signal.SIGKILL) + await asyncio.sleep(1.0) + assert not dp_mock_server_process.is_alive() + + await asyncio.wait_for(_task, timeout=5.0) + for p in supervisor._processes: + assert not p.is_alive() + print("everything was cleaned up!") + + +@pytest.mark.asyncio +async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch): + """ + Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight + request drain). The supervisor is configured with shutdown_timeout=10, + so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The + children exit naturally within that window, so no force-kill should occur + and the measured wall-clock time must be >= 10s. + """ + _DRAIN_SECONDS = 10.0 + _SHUTDOWN_TIMEOUT = 10.0 + + args = _make_args(shutdown_timeout=_SHUTDOWN_TIMEOUT) + vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)] + + async with _run_supervisor( + args, monkeypatch, launch_fn=launch_mock_vllm_with_drain + ) as (supervisor, _task): + for port in vllm_server_ports: + await _poll_until_api_server_running(port) + + for port in vllm_server_ports: + await _set_healthy(port) + await asyncio.sleep(1.0) + assert await _poll_supervisor_health(200) + assert supervisor.is_ready + + start_t = time.perf_counter() + os.kill(os.getpid(), signal.SIGTERM) + + print(f"DRAINING FOR {_DRAIN_SECONDS}") + await asyncio.wait_for(_task, timeout=_DRAIN_SECONDS + CHILD_EXIT_GRACE_S + 5.0) + elapsed = time.perf_counter() - start_t + + assert elapsed >= _DRAIN_SECONDS, ( + f"Supervisor exited after only {elapsed:.1f}s; " + f"expected >= {_DRAIN_SECONDS}s for request draining" + ) + + for p in supervisor._processes: + assert not p.is_alive() + print(f"Supervisor waited {elapsed:.1f}s for children to drain — expected!") diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index aa82b70eb9..e93d8d03ac 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -476,6 +476,7 @@ class EngineArgs: data_parallel_rpc_port: int | None = None data_parallel_hybrid_lb: bool = False data_parallel_external_lb: bool = False + data_parallel_multi_port_external_lb: bool = False data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter @@ -1043,6 +1044,15 @@ class EngineArgs: "-dpe", **parallel_kwargs["data_parallel_external_lb"], ) + parallel_group.add_argument( + "--data-parallel-multi-port-external-lb", + "-dpm", + action="store_true", + default=False, + help="Run a node-local supervisor that launches one external-LB API " + "server per local data parallel rank and exposes aggregated health on " + "a supervisor port.", + ) parallel_group.add_argument( "--enable-expert-parallel", "-ep", diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index 6d6cfa6cd7..5972bd48aa 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -12,6 +12,9 @@ import vllm.envs as envs from vllm.entrypoints.cli.types import CLISubcommand from vllm.entrypoints.openai.api_server import run_server, setup_server from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args +from vllm.entrypoints.openai.dp_supervisor import ( + run_dp_supervisor, +) from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG from vllm.logger import init_logger from vllm.usage.usage_lib import UsageContext @@ -66,31 +69,41 @@ class ServeSubcommand(CLISubcommand): args.api_server_count = 0 # Detect LB mode for defaulting api_server_count. + # Multi-port: --data-parallel-multi-port-external-lb # External LB: --data-parallel-external-lb or --data-parallel-rank # Hybrid LB: --data-parallel-hybrid-lb or --data-parallel-start-rank is_external_lb = ( args.data_parallel_external_lb or args.data_parallel_rank is not None ) - is_hybrid_lb = ( - args.data_parallel_hybrid_lb or args.data_parallel_start_rank is not None - ) - if is_external_lb and is_hybrid_lb: + # If --data_parallel_multi_port_external_lb and --data_parallel_hybrid_lb + # are unset, default to hybrid if --data-parallel-start-rank is set + is_hybrid_lb = is_multi_port = False + if ( + not args.data_parallel_hybrid_lb + and not args.data_parallel_multi_port_external_lb + ): + is_hybrid_lb = args.data_parallel_start_rank is not None + else: + is_hybrid_lb = args.data_parallel_hybrid_lb + is_multi_port = args.data_parallel_multi_port_external_lb + + if sum([is_multi_port, is_external_lb, is_hybrid_lb]) > 1: raise ValueError( - "Cannot use both external and hybrid data parallel load " - "balancing modes. External LB is enabled via " - "--data-parallel-external-lb or --data-parallel-rank. " - "Hybrid LB is enabled via --data-parallel-hybrid-lb or " - "--data-parallel-start-rank. Use one mode or the other." + "Cannot use more than one data parallel load balancing mode. " + "Choose one of: --data-parallel-multi-port-external-lb, " + "--data-parallel-external-lb (or --data-parallel-rank), " + "--data-parallel-hybrid-lb (or --data-parallel-start-rank)." ) # Default api_server_count if not explicitly set. - # - Rust frontend: Use 1 (not applicable as it's multithreaded) - # - External LB: Leave as 1 (external LB handles distribution) + # - Multi-port: 1 (supervisor spawns one server per local DP rank) + # - Rust frontend: 1 (not applicable as it's multithreaded) + # - External LB: 1 (external LB handles distribution) # - Hybrid LB: Use local DP size (internal LB for local ranks only) # - Internal LB: Use full DP size if args.api_server_count is None: - if is_external_lb or envs.VLLM_RUST_FRONTEND_PATH: + if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH: args.api_server_count = 1 elif is_hybrid_lb: args.api_server_count = args.data_parallel_size_local or 1 @@ -109,7 +122,8 @@ class ServeSubcommand(CLISubcommand): ) elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1: logger.warning( - "Ignoring --api-server-count=%d when using rust front-end process" + "Ignoring --api-server-count=%d when using rust front-end process", + args.api_server_count, ) args.api_server_count = 1 @@ -122,7 +136,9 @@ class ServeSubcommand(CLISubcommand): ) args.api_server_count = 1 - if args.api_server_count < 1: + if is_multi_port: + run_dp_supervisor(args) + elif args.api_server_count < 1: run_headless(args) elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH: run_multi_api_server(args) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index b9d8721643..fc315a4565 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -665,8 +665,7 @@ async def build_and_serve_renderer( async def run_server(args, **uvicorn_kwargs) -> None: """Run a single-worker API server.""" - # Add process-specific prefix to stdout and stderr. - decorate_logs("APIServer") + decorate_logs("APIServer", skip_if_decorated=True) # Interrupt initialization if SIGTERM arrives before uvicorn installs its # own signal handlers. Once uvicorn is running it replaces this. diff --git a/vllm/entrypoints/openai/cli_args.py b/vllm/entrypoints/openai/cli_args.py index 3ebdec3c67..d130e83422 100644 --- a/vllm/entrypoints/openai/cli_args.py +++ b/vllm/entrypoints/openai/cli_args.py @@ -229,6 +229,17 @@ class FrontendArgs(BaseFrontendArgs): """Host name.""" port: int = 8000 """Port number.""" + data_parallel_supervisor_port: int = 9256 + """HTTP port for aggregated health endpoints in multi-port external LB + mode.""" + dp_supervisor_probe_interval_s: float = 5.0 + """Seconds between aggregated health probes in multi-port external LB mode.""" + dp_supervisor_probe_timeout_s: float = 5.0 + """Seconds to wait between retries when a child health probe fails with a + connection error in multi-port external LB mode.""" + dp_supervisor_probe_failure_threshold: int = 3 + """Number of consecutive connection-error retries before a child health + probe is declared failed in multi-port external LB mode.""" uds: str | None = None """Unix domain socket path. If set, host and port arguments are ignored.""" uvicorn_log_level: Literal[ @@ -387,6 +398,13 @@ def validate_parsed_serve_args(args: argparse.Namespace): if args.enable_log_outputs and not args.enable_log_requests: raise TypeError("Error: --enable-log-outputs requires --enable-log-requests") + if args.data_parallel_multi_port_external_lb: + from vllm.entrypoints.openai.dp_supervisor import ( + validate_multi_port_external_lb_args, + ) + + validate_multi_port_external_lb_args(args) + def create_parser_for_docs() -> FlexibleArgumentParser: parser_for_docs = FlexibleArgumentParser( diff --git a/vllm/entrypoints/openai/dp_supervisor.py b/vllm/entrypoints/openai/dp_supervisor.py new file mode 100644 index 0000000000..2dff91fa79 --- /dev/null +++ b/vllm/entrypoints/openai/dp_supervisor.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import copy +import multiprocessing +import os +import signal +import time +from functools import partial +from http import HTTPStatus +from multiprocessing.process import BaseProcess + +import aiohttp +import psutil +import uvicorn +import uvloop +from fastapi import FastAPI, Response + +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils.system_utils import ( + decorate_logs, + kill_process_tree, + set_process_title, + update_environment_variables, +) + +logger = init_logger(__name__) + +CHILD_EXIT_GRACE_S = 5.0 + + +def infer_multi_port_external_lb_start_rank(args: argparse.Namespace) -> int: + start_rank = getattr(args, "data_parallel_start_rank", None) + if start_rank is not None: + return start_rank + + node_rank = getattr(args, "node_rank", 0) or 0 + local_size = getattr(args, "data_parallel_size_local", 0) or 0 + return node_rank * local_size + + +def validate_multi_port_external_lb_args(args: argparse.Namespace) -> None: + if getattr(args, "grpc", False): + raise ValueError( + "Error: --data-parallel-multi-port-external-lb does not support --grpc" + ) + if args.uds is not None: + raise ValueError( + "Error: --data-parallel-multi-port-external-lb does not support --uds" + ) + if any((args.ssl_keyfile, args.ssl_certfile, args.ssl_ca_certs)): + raise ValueError( + "Error: --data-parallel-multi-port-external-lb does not support HTTPS yet" + ) + if args.api_server_count not in (None, 1): + raise ValueError( + "Error: --data-parallel-multi-port-external-lb currently requires " + "--api-server-count=1" + ) + if args.data_parallel_rank is not None: + raise ValueError( + "Error: --data-parallel-multi-port-external-lb manages child " + "--data-parallel-rank values internally" + ) + if args.data_parallel_external_lb or args.data_parallel_hybrid_lb: + raise ValueError( + "Error: --data-parallel-multi-port-external-lb cannot be combined with " + "--data-parallel-external-lb or --data-parallel-hybrid-lb" + ) + if args.data_parallel_size < 2: + raise ValueError( + "Error: --data-parallel-multi-port-external-lb requires " + "--data-parallel-size > 1" + ) + + local_size = args.data_parallel_size_local + if local_size is None or local_size < 2: + raise ValueError( + "Error: --data-parallel-multi-port-external-lb requires " + "--data-parallel-size-local >= 2" + ) + if local_size > args.data_parallel_size: + raise ValueError( + "Error: --data-parallel-size-local cannot exceed --data-parallel-size" + ) + if args.data_parallel_size % local_size != 0: + raise ValueError( + "Error: --data-parallel-size must be divisible by " + "--data-parallel-size-local" + ) + + start_rank = infer_multi_port_external_lb_start_rank(args) + if start_rank + local_size > args.data_parallel_size: + raise ValueError( + "Error: multi-port supervised ranks would exceed --data-parallel-size" + ) + + supervisor_port = args.data_parallel_supervisor_port + child_port_min = args.port + child_port_max = args.port + local_size - 1 + if child_port_min <= supervisor_port <= child_port_max: + raise ValueError( + f"Error: --data-parallel-supervisor-port {supervisor_port} " + f"overlaps with child rank ports {child_port_min}-{child_port_max}" + ) + + +def _build_vllm_dp_server_args( + args: argparse.Namespace, local_rank: int +) -> argparse.Namespace: + child_args = copy.copy(args) + child_args.port = args.port + local_rank + child_args.data_parallel_rank = ( + infer_multi_port_external_lb_start_rank(args) + local_rank + ) + child_args.data_parallel_start_rank = None + child_args.data_parallel_size_local = 1 + child_args.data_parallel_external_lb = True + child_args.data_parallel_hybrid_lb = False + child_args.data_parallel_multi_port_external_lb = False + child_args.data_parallel_supervisor_port = None + child_args.api_server_count = 1 + return child_args + + +def _build_vllm_dp_server_env( + args: argparse.Namespace, local_rank: int +) -> dict[str, str]: + # set visible devices for the child process + devices_per_rank = args.tensor_parallel_size * args.pipeline_parallel_size + start = local_rank * devices_per_rank + stop = start + devices_per_rank + device_env = current_platform.device_control_env_var + visible_devices = ",".join( + str(current_platform.device_id_to_physical_device_id(idx)) + for idx in range(start, stop) + ) + return {device_env: visible_devices} + + +def _child_base_url(args: argparse.Namespace, port: int) -> str: + host = args.host or "127.0.0.1" + if host == "0.0.0.0": + host = "127.0.0.1" + elif host == "::": + host = "::1" + return f"http://{host}:{port}" + + +def _join_processes_with_timeout(processes: list[BaseProcess], timeout: float) -> None: + deadline = time.monotonic() + timeout + for process in processes: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + process.join(timeout=remaining) + + +async def _probe_endpoint( + session: aiohttp.ClientSession, + args: argparse.Namespace, + port: int, + path: str, + conn_err_failure_threshold: int = 3, + conn_err_retry_delay: float = 5.0, +) -> bool: + """ + Probe /health endpoint for 200 status. + + If there is a connection error, retry every N seconds. + """ + for iteration in range(conn_err_failure_threshold): + try: + async with session.get(_child_base_url(args, port) + path) as response: + # vLLM returns 503 on EngineDeadError, so we should return + # immediately if vLLM responds with a non-200 status code. + return response.status == HTTPStatus.OK + except (aiohttp.ClientError, asyncio.TimeoutError) as e: + # Allow retry of connection errors. + logger.debug( + "Probe attempt %d/%d failed on port %d: %r", + iteration + 1, + conn_err_failure_threshold, + port, + e, + ) + + if iteration < conn_err_failure_threshold - 1: + await asyncio.sleep(conn_err_retry_delay) + + return False + + +def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI: + app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None) + app.state.supervisor = supervisor + + def _status_response(ok: bool) -> Response: + return Response( + status_code=(HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE) + ) + + @app.get("/health", include_in_schema=False) + async def health() -> Response: + return _status_response(app.state.supervisor.is_ready) + + @app.get("/ready", include_in_schema=False) + @app.get("/readyz", include_in_schema=False) + async def ready() -> Response: + return _status_response(app.state.supervisor.is_ready) + + return app + + +def _run_vllm_dp_server( + child_args: argparse.Namespace, env_updates: dict[str, str] +) -> None: + """ + Entrypoint function for the vLLM DP Server. + """ + from vllm.entrypoints.openai.api_server import run_server + + # Create a fresh process group for the vLLM DP Server, + # so that CTRL-C is propagated cleanly. + os.setpgrp() + + name = f"APIServer_DP{child_args.data_parallel_rank}" + update_environment_variables(env_updates) + set_process_title(name) + decorate_logs(name) + uvloop.run(run_server(child_args)) + + +class DPSupervisor: + def __init__(self, args: argparse.Namespace): + validate_multi_port_external_lb_args(args) + self.args = args + self.supervisor_port = args.data_parallel_supervisor_port + self.child_ports = [ + args.port + local_rank + for local_rank in range(args.data_parallel_size_local) + ] + self._is_ready = False + self._processes: list[BaseProcess] = [] + self._shutdown_event = asyncio.Event() + self._shutdown_signal = signal.SIGTERM + + @property + def is_ready(self) -> bool: + return self._is_ready and not self._shutdown_event.is_set() + + async def run(self) -> None: + loop = asyncio.get_running_loop() + + # K8s sends SIGTERM for shutdown - begin graceful termination. + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, partial(self._handle_signal, sig)) + + # Launch DPSupervisor Server. + app = _build_dp_supervisor_app(self) + decorate_logs("DPSupervisor") + host = self.args.host or "0.0.0.0" + config = uvicorn.Config( + app, + host=host, + port=self.supervisor_port, + log_level=self.args.uvicorn_log_level, + ) + supervisor_server = uvicorn.Server(config) + supervisor_server_task = asyncio.create_task( + supervisor_server.serve(), + name="dp-supervisor", + ) + supervisor_server_task.add_done_callback( + lambda _task: self._shutdown_event.set() + ) + + # Ensure DPSupervisor task starts on the event loop. + while not supervisor_server.started: + if supervisor_server_task.done(): + supervisor_server_task.result() + raise RuntimeError("DPSupervisor exited before startup.") + await asyncio.sleep(0.05) + logger.info("Started DPSupervisor on %s:%d", host, self.supervisor_port) + + # Launch and Monitor vLLM Server Processes. + try: + self._start_children() + await self._monitor_children() + finally: + self._is_ready = False + await self._shutdown_children() + + # Shutdown the DP Supervisor server. + supervisor_server.should_exit = True + await supervisor_server_task + + def _handle_signal(self, signum: int) -> None: + """ + Signal handler that is added to the event loop. + + This catches the SIGTERM from K8s and begins graceful shutdown, + by setting the _shutdown_event(), which is watched by the main + coroutine monitoring the vLLM DP Servers. + """ + + if self._shutdown_event.is_set(): + return + + self._shutdown_signal = signal.Signals(signum) + logger.info( + "DPSupervisor received %s, shutting down.", + self._shutdown_signal.name, + ) + + self._shutdown_event.set() + self._is_ready = False + + def _start_children(self) -> None: + """ + Launch vLLM DP Servers on separate GPUs. + """ + logger.info("Launching vLLM DP Servers") + context = multiprocessing.get_context("spawn") + for local_rank in range(self.args.data_parallel_size_local): + child_args = _build_vllm_dp_server_args(self.args, local_rank) + child_env = _build_vllm_dp_server_env(self.args, local_rank) + process = context.Process( + target=_run_vllm_dp_server, + name=f"APIServer_DPRank_{child_args.data_parallel_rank}", + args=(child_args, child_env), + ) + process.start() + self._processes.append(process) + + async def _probe_all_children(self) -> None: + """ + Background coroutine: probes all child endpoints on each interval. + + Exits when any server becomes unhealthy after being ready, signalling + _monitor_children to initiate shutdown. + """ + timeout = aiohttp.ClientTimeout(total=self.args.dp_supervisor_probe_timeout_s) + async with aiohttp.ClientSession(timeout=timeout) as session: + while not self._shutdown_event.is_set(): + threshold = ( + self.args.dp_supervisor_probe_failure_threshold + if self._is_ready + else 1 + ) + results = await asyncio.gather( + *( + _probe_endpoint( + session, + self.args, + port, + "/health", + conn_err_failure_threshold=threshold, + conn_err_retry_delay=self.args.dp_supervisor_probe_interval_s, + ) + for port in self.child_ports + ), + return_exceptions=True, + ) + all_healthy = all(r is True for r in results) + + if all_healthy: + # If all healthy, we are ready to receive requests. + # This conditional avoids a potential race condition + # where shutdown is set, THEN the probe returns true. + if not self._shutdown_event.is_set(): + self._is_ready = True + elif self._is_ready: + # Once ready, any failure in the probe means vLLM is dead. + num_unhealthy = sum(1 for r in results if r is not True) + logger.info( + "DPSupervisor probe found %s unhealthy DP Servers.", + num_unhealthy, + ) + self._is_ready = False + self._shutdown_event.set() + return + + with contextlib.suppress(asyncio.TimeoutError): + logger.debug( + "Waiting for %s seconds before next probe", + self.args.dp_supervisor_probe_interval_s, + ) + await asyncio.wait_for( + self._shutdown_event.wait(), + timeout=self.args.dp_supervisor_probe_interval_s, + ) + + async def _monitor_children(self) -> None: + """ + Main coroutine task that monitors the children vLLM servers. + + Before the vLLM servers are /ready: + - if the pid is dead, we will shut down + - if the probe fails, we try again after dp_supervisor_probe_interval_s + + After the vLLM servers are /ready: + - if the pid is dead, we will shut down + - if the probe fails, we will shut down + """ + probe_task = asyncio.create_task( + self._probe_all_children(), name="dp-health-probe" + ) + + try: + while not self._shutdown_event.is_set(): + # 1. Check for dead processes + n_failed = len([p for p in self._processes if not p.is_alive()]) + if n_failed > 0: + logger.info("DPSupervisor found %s exited DP Servers.", n_failed) + break + + # 2. Check if the probe background task crashed or failed. + if probe_task.done(): + # Extract exception if it crashed, or log failure + exc = probe_task.exception() if not probe_task.cancelled() else None + logger.info("DPSupervisor probe task stopped. Exception: %s", exc) + break + + # Sleep for probe_interval seconds or until a shutdown. + with contextlib.suppress(asyncio.TimeoutError): + logger.debug( + "Waiting for %s seconds before next monitor", + self.args.dp_supervisor_probe_interval_s, + ) + await asyncio.wait_for( + self._shutdown_event.wait(), + timeout=self.args.dp_supervisor_probe_interval_s, + ) + + finally: + # Cleanup probe task if needed. + if not probe_task.done(): + probe_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await probe_task + + async def _shutdown_children(self) -> None: + """Terminate the vLLM DP servers.""" + timeout = self.args.shutdown_timeout + CHILD_EXIT_GRACE_S + + try: + logger.info( + "DPSupervisor forwarding %s to DP Servers.", + self._shutdown_signal.name, + ) + for process in self._processes: + pid = process.pid + if not process.is_alive() or pid is None: + continue + with contextlib.suppress(ProcessLookupError, OSError): + os.kill(pid, self._shutdown_signal) + + try: + await asyncio.to_thread( + _join_processes_with_timeout, self._processes, timeout + ) + except asyncio.CancelledError: + logger.warning("Shutdown await cancelled") + raise + finally: + for process in self._processes: + pid = process.pid + if not process.is_alive() or pid is None: + continue + logger.warning( + "DP server %s did not exit within %.1fs; force killing.", + process.name, + timeout, + ) + with contextlib.suppress( + ProcessLookupError, + OSError, + psutil.NoSuchProcess, + psutil.AccessDenied, + ): + kill_process_tree(pid) + + +def run_dp_supervisor(args: argparse.Namespace) -> None: + uvloop.run(DPSupervisor(args).run()) diff --git a/vllm/utils/system_utils.py b/vllm/utils/system_utils.py index db904c29e0..7f56f972a4 100644 --- a/vllm/utils/system_utils.py +++ b/vllm/utils/system_utils.py @@ -234,12 +234,17 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None: file.write = write_with_prefix # type: ignore[method-assign] -def decorate_logs(process_name: str | None = None) -> None: +def decorate_logs( + process_name: str | None = None, *, skip_if_decorated: bool = False +) -> None: """Decorate stdout/stderr with process name and PID prefix.""" # Respect VLLM_CONFIGURE_LOGGING environment variable if not envs.VLLM_CONFIGURE_LOGGING: return + if skip_if_decorated and hasattr(sys.stdout, "_original_write"): + return + if process_name is None: process_name = get_mp_context().current_process().name