[Frontend] DP Supervisor (#40841)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Robert Shaw <robertgshaw2@gmail.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: robertgshaw2-redhat <robertgshaw2@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
robertgshaw2-redhat
Robert Shaw
Nick Hill
parent
60af5c16ee
commit
0ddd7dd656
@@ -0,0 +1,610 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Tests for DPSupervisor: unit tests and lifecycle integration tests.
|
||||
|
||||
Lifecycle integration tests replace child vLLM servers with lightweight
|
||||
aiohttp "fake" servers controlled by the test, so the suite runs without GPUs.
|
||||
_start_children is monkeypatched to install FakeProcess objects (with
|
||||
controllable liveness/timing) alongside those fake HTTP servers.
|
||||
|
||||
Port allocation (kept far from default vLLM ports to avoid conflicts):
|
||||
Supervisor : 19256
|
||||
Children : 18000, 18001
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Response
|
||||
|
||||
import vllm.entrypoints.openai.dp_supervisor as dp_sup
|
||||
from vllm.entrypoints.openai.dp_supervisor import (
|
||||
CHILD_EXIT_GRACE_S,
|
||||
DPSupervisor,
|
||||
_build_vllm_dp_server_args,
|
||||
infer_multi_port_external_lb_start_rank,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SUPERVISOR_PORT = 19256
|
||||
_CHILD_PORT_BASE = 18000
|
||||
_N_CHILDREN = 2
|
||||
_PROBE_INTERVAL = 1.0
|
||||
_POLL_INTERVAL = 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Args factories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_unit_args(**overrides) -> argparse.Namespace:
|
||||
"""Minimal args for unit tests (no real network activity)."""
|
||||
base = {
|
||||
"host": None,
|
||||
"port": 8000,
|
||||
"data_parallel_multi_port_external_lb": True,
|
||||
"data_parallel_supervisor_port": 9256,
|
||||
"dp_supervisor_probe_interval_s": 5.0,
|
||||
"dp_supervisor_probe_timeout_s": 5.0,
|
||||
"dp_supervisor_probe_failure_threshold": 3,
|
||||
"data_parallel_size": 8,
|
||||
"data_parallel_size_local": 4,
|
||||
"data_parallel_start_rank": None,
|
||||
"data_parallel_rank": None,
|
||||
"data_parallel_external_lb": False,
|
||||
"data_parallel_hybrid_lb": False,
|
||||
"api_server_count": None,
|
||||
"headless": False,
|
||||
"grpc": False,
|
||||
"uds": None,
|
||||
"ssl_keyfile": None,
|
||||
"ssl_certfile": None,
|
||||
"ssl_ca_certs": None,
|
||||
"node_rank": 1,
|
||||
"tensor_parallel_size": 1,
|
||||
"pipeline_parallel_size": 1,
|
||||
"uvicorn_log_level": "info",
|
||||
"shutdown_timeout": 5.0,
|
||||
}
|
||||
base.update(overrides)
|
||||
return argparse.Namespace(**base)
|
||||
|
||||
|
||||
def _make_args(**overrides) -> argparse.Namespace:
|
||||
"""Args for lifecycle integration tests (real loopback servers)."""
|
||||
base: dict = dict(
|
||||
host="127.0.0.1",
|
||||
port=_CHILD_PORT_BASE,
|
||||
data_parallel_multi_port_external_lb=True,
|
||||
data_parallel_supervisor_port=_SUPERVISOR_PORT,
|
||||
dp_supervisor_probe_interval_s=_PROBE_INTERVAL,
|
||||
dp_supervisor_probe_timeout_s=1.0,
|
||||
dp_supervisor_probe_failure_threshold=3,
|
||||
data_parallel_size=_N_CHILDREN,
|
||||
data_parallel_size_local=_N_CHILDREN,
|
||||
data_parallel_start_rank=0,
|
||||
data_parallel_rank=None,
|
||||
data_parallel_external_lb=False,
|
||||
data_parallel_hybrid_lb=False,
|
||||
api_server_count=None,
|
||||
headless=False,
|
||||
grpc=False,
|
||||
uds=None,
|
||||
ssl_keyfile=None,
|
||||
ssl_certfile=None,
|
||||
ssl_ca_certs=None,
|
||||
node_rank=0,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
uvicorn_log_level="warning",
|
||||
shutdown_timeout=0.0,
|
||||
)
|
||||
base.update(overrides)
|
||||
return argparse.Namespace(**base)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_infer_multi_port_external_lb_start_rank_uses_node_rank():
|
||||
args = _make_unit_args()
|
||||
assert infer_multi_port_external_lb_start_rank(args) == 4
|
||||
|
||||
|
||||
def test_build_multi_port_external_lb_child_args_sets_external_rank_server():
|
||||
args = _make_unit_args(data_parallel_start_rank=8, api_server_count=None)
|
||||
child_args = _build_vllm_dp_server_args(args, local_rank=2)
|
||||
|
||||
assert child_args.port == 8002
|
||||
assert child_args.data_parallel_rank == 10
|
||||
assert child_args.data_parallel_size_local == 1
|
||||
assert child_args.data_parallel_external_lb is True
|
||||
assert child_args.data_parallel_hybrid_lb is False
|
||||
assert child_args.data_parallel_multi_port_external_lb is False
|
||||
assert child_args.api_server_count == 1
|
||||
|
||||
|
||||
def test_aggregates_health():
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
supervisor._is_ready = True
|
||||
assert supervisor.is_ready is True
|
||||
|
||||
|
||||
def test_handles_shutdown_event():
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
supervisor._is_ready = True
|
||||
supervisor._shutdown_event.set()
|
||||
assert supervisor.is_ready is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_child_exit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
supervisor._processes = [
|
||||
SimpleNamespace(
|
||||
name="APIServer_DPRank_4", exitcode=None, is_alive=lambda: True
|
||||
),
|
||||
SimpleNamespace(name="APIServer_DPRank_5", exitcode=17, is_alive=lambda: False),
|
||||
]
|
||||
|
||||
async def fake_probe(*_args, **_kwargs) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
|
||||
|
||||
await supervisor._monitor_children()
|
||||
assert supervisor._is_ready is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_probe_failure(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
supervisor = DPSupervisor(_make_unit_args(dp_supervisor_probe_interval_s=0.0))
|
||||
supervisor.child_ports = [8000]
|
||||
probe_results = iter([True, False])
|
||||
|
||||
async def fake_probe(*_args, **_kwargs) -> bool:
|
||||
return next(probe_results)
|
||||
|
||||
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
|
||||
|
||||
await supervisor._monitor_children()
|
||||
assert supervisor._is_ready is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_if_supervisor_server_error_on_startup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
class FakeLoop:
|
||||
def add_signal_handler(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def remove_signal_handler(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
class FakeServer:
|
||||
def __init__(self, _config):
|
||||
self.started = False
|
||||
self.should_exit = False
|
||||
|
||||
async def serve(self):
|
||||
raise ValueError("supervisor boom")
|
||||
|
||||
async def fake_shutdown_children(self):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop())
|
||||
monkeypatch.setattr(dp_sup.uvicorn, "Server", FakeServer)
|
||||
monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children)
|
||||
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
|
||||
with pytest.raises(ValueError, match="supervisor boom"):
|
||||
await supervisor.run()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle integration tests – MockVLLMServer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MockVLLMServer:
|
||||
"""
|
||||
Minimal FastAPI server that mimics one vLLM replica.
|
||||
GET /health returns 200 when healthy, 503 otherwise.
|
||||
Health state is toggled by the test via set_healthy().
|
||||
"""
|
||||
|
||||
def __init__(self, port: int, drain_seconds: float = 0.0) -> None:
|
||||
self.port = port
|
||||
self._healthy = False
|
||||
self._drain_seconds = drain_seconds
|
||||
self._server: uvicorn.Server | None = None
|
||||
self._serve_task: asyncio.Task | None = None
|
||||
|
||||
async def start(self) -> None:
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> Response:
|
||||
print(f"MockServer {self.port}: /health: {self._healthy}")
|
||||
return Response(status_code=200 if self._healthy else 503)
|
||||
|
||||
@app.get("/set_healthy")
|
||||
async def set_healthy() -> Response:
|
||||
print(f"MockServer {self.port}: /set_healthy")
|
||||
self._healthy = True
|
||||
return Response(status_code=200)
|
||||
|
||||
@app.get("/set_unhealthy")
|
||||
async def set_unhealthy() -> Response:
|
||||
print(f"MockServer {self.port}: /set_unhealthy")
|
||||
self._healthy = False
|
||||
return Response(status_code=200)
|
||||
|
||||
@app.get("/kill")
|
||||
async def kill() -> Response:
|
||||
print(f"MockServer {self.port}: /kill")
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host="127.0.0.1",
|
||||
port=self.port,
|
||||
log_level="warning",
|
||||
lifespan="off",
|
||||
)
|
||||
self._server = uvicorn.Server(config)
|
||||
|
||||
# Configure request draining if needed.
|
||||
# Uvicorn's capture_signals() installs signal.signal(SIGTERM, self.handle_exit),
|
||||
# which sets should_exit=True immediately. Override handle_exit on the instance
|
||||
# so capture_signals() picks up our version that drains first.
|
||||
if self._drain_seconds > 0:
|
||||
self._shutdown_event = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
async def _drain_and_stop() -> None:
|
||||
await self._shutdown_event.wait()
|
||||
print(f"MockServer {self.port}: draining for {self._drain_seconds}s.")
|
||||
await asyncio.sleep(self._drain_seconds)
|
||||
print("Setting should_exit")
|
||||
if self._server is not None:
|
||||
self._server.should_exit = True
|
||||
|
||||
self._drain_task = asyncio.create_task(_drain_and_stop())
|
||||
|
||||
def _custom_handle_exit(sig: int, frame: object) -> None:
|
||||
print("Got SIGTERM, setting shutdown.")
|
||||
if not self._shutdown_event.is_set():
|
||||
loop.call_soon_threadsafe(self._shutdown_event.set)
|
||||
|
||||
self._server.handle_exit = _custom_handle_exit
|
||||
|
||||
self._serve_task = asyncio.create_task(self._server.serve())
|
||||
while not self._server.started:
|
||||
await asyncio.sleep(0.01)
|
||||
print(f"Mock DP Server on port {self.port} started")
|
||||
|
||||
await self._serve_task
|
||||
|
||||
|
||||
def launch_mock_vllm(child_args: argparse.Namespace, env_updates: dict[str, str]):
|
||||
logger.info("Launching mock vLLM on port %s", child_args.port)
|
||||
mock_vllm = MockVLLMServer(port=child_args.port)
|
||||
asyncio.run(mock_vllm.start())
|
||||
|
||||
|
||||
def launch_mock_vllm_with_drain(
|
||||
child_args: argparse.Namespace, env_updates: dict[str, str]
|
||||
):
|
||||
logger.info("Launching mock vLLM with 15s drain on port %s", child_args.port)
|
||||
mock_vllm = MockVLLMServer(port=child_args.port, drain_seconds=10.0)
|
||||
asyncio.run(mock_vllm.start())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle test helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _poll_supervisor_health(expected_status: int) -> bool:
|
||||
"""
|
||||
Poll GET /health on the supervisor until expected_status is seen.
|
||||
A connection error is treated as 503-equivalent when expected_status != 200.
|
||||
"""
|
||||
url = f"http://127.0.0.1:{_SUPERVISOR_PORT}/health"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status != expected_status:
|
||||
print(f"expected: {expected_status=}, got: {resp.status=}")
|
||||
return False
|
||||
return True
|
||||
except aiohttp.ClientError:
|
||||
if expected_status != -1:
|
||||
print(f"expected: {expected_status=}, got: aiohttp.ClientError")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def _poll_until_api_server_running(port: int, retries: int = 10) -> None:
|
||||
url = f"http://127.0.0.1:{port}/health"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for _ in range(retries):
|
||||
try:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status != 200:
|
||||
return
|
||||
await asyncio.sleep(1.0)
|
||||
except aiohttp.ClientError:
|
||||
print("Test detected not started yet, sleeping for 1s")
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
|
||||
async def _set_healthy(port: int) -> None:
|
||||
url = f"http://127.0.0.1:{port}/set_healthy"
|
||||
async with aiohttp.ClientSession() as session, session.get(url) as resp:
|
||||
assert resp.status == 200
|
||||
|
||||
|
||||
async def _set_unhealthy(port: int) -> None:
|
||||
url = f"http://127.0.0.1:{port}/set_unhealthy"
|
||||
async with aiohttp.ClientSession() as session, session.get(url) as resp:
|
||||
assert resp.status == 200
|
||||
|
||||
|
||||
async def _kill_server(port: int) -> None:
|
||||
url = f"http://127.0.0.1:{port}/kill"
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session, session.get(url) as resp:
|
||||
assert resp.status != 200
|
||||
except Exception as e:
|
||||
assert isinstance(e, aiohttp.ClientConnectorError)
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _run_supervisor(
|
||||
args: argparse.Namespace,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
launch_fn=None,
|
||||
):
|
||||
if launch_fn is None:
|
||||
launch_fn = launch_mock_vllm
|
||||
monkeypatch.setattr(dp_sup, "_run_vllm_dp_server", launch_fn)
|
||||
supervisor = DPSupervisor(args)
|
||||
task = asyncio.create_task(supervisor.run())
|
||||
await asyncio.sleep(1.0)
|
||||
try:
|
||||
yield supervisor, task
|
||||
finally:
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle integration tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_lifecycle(monkeypatch):
|
||||
"""
|
||||
A) Supervisor /health returns 503 while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) SIGTERM and shutdown
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _set_healthy(vllm_server_ports[0])
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
print("/health is 503 --- expected!")
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print("everything was cleaned up!")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_startup(monkeypatch):
|
||||
"""
|
||||
A) One of the vLLM servers crashes during startup.
|
||||
B) DPSupervisor detects this, and cleans up resources.
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _kill_server(port)
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_becomes_unhealthy(monkeypatch):
|
||||
"""
|
||||
A) Supervisor /health returns 503 while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) Child process becomes unhealthy.
|
||||
D) Detected and shutdown.
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _set_healthy(vllm_server_ports[0])
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
print("/health is 503 --- expected!")
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
await _set_unhealthy(port)
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print("everything was cleaned up!")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_server_fails(monkeypatch):
|
||||
"""
|
||||
A) Supervisor /health returns 503 while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) Child process fails.
|
||||
D) Detected and shutdown.
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _set_healthy(vllm_server_ports[0])
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(503)
|
||||
assert not supervisor.is_ready
|
||||
print("/health is 503 --- expected!")
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
dp_mock_server_process = supervisor._processes[0]
|
||||
os.kill(dp_mock_server_process.pid, signal.SIGKILL)
|
||||
await asyncio.sleep(1.0)
|
||||
assert not dp_mock_server_process.is_alive()
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print("everything was cleaned up!")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
|
||||
request drain). The supervisor is configured with shutdown_timeout=10,
|
||||
so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The
|
||||
children exit naturally within that window, so no force-kill should occur
|
||||
and the measured wall-clock time must be >= 10s.
|
||||
"""
|
||||
_DRAIN_SECONDS = 10.0
|
||||
_SHUTDOWN_TIMEOUT = 10.0
|
||||
|
||||
args = _make_args(shutdown_timeout=_SHUTDOWN_TIMEOUT)
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(
|
||||
args, monkeypatch, launch_fn=launch_mock_vllm_with_drain
|
||||
) as (supervisor, _task):
|
||||
for port in vllm_server_ports:
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
|
||||
start_t = time.perf_counter()
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
print(f"DRAINING FOR {_DRAIN_SECONDS}")
|
||||
await asyncio.wait_for(_task, timeout=_DRAIN_SECONDS + CHILD_EXIT_GRACE_S + 5.0)
|
||||
elapsed = time.perf_counter() - start_t
|
||||
|
||||
assert elapsed >= _DRAIN_SECONDS, (
|
||||
f"Supervisor exited after only {elapsed:.1f}s; "
|
||||
f"expected >= {_DRAIN_SECONDS}s for request draining"
|
||||
)
|
||||
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print(f"Supervisor waited {elapsed:.1f}s for children to drain — expected!")
|
||||
@@ -476,6 +476,7 @@ class EngineArgs:
|
||||
data_parallel_rpc_port: int | None = None
|
||||
data_parallel_hybrid_lb: bool = False
|
||||
data_parallel_external_lb: bool = False
|
||||
data_parallel_multi_port_external_lb: bool = False
|
||||
data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend
|
||||
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
|
||||
enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter
|
||||
@@ -1043,6 +1044,15 @@ class EngineArgs:
|
||||
"-dpe",
|
||||
**parallel_kwargs["data_parallel_external_lb"],
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--data-parallel-multi-port-external-lb",
|
||||
"-dpm",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Run a node-local supervisor that launches one external-LB API "
|
||||
"server per local data parallel rank and exposes aggregated health on "
|
||||
"a supervisor port.",
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--enable-expert-parallel",
|
||||
"-ep",
|
||||
|
||||
@@ -12,6 +12,9 @@ import vllm.envs as envs
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.openai.api_server import run_server, setup_server
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||
from vllm.entrypoints.openai.dp_supervisor import (
|
||||
run_dp_supervisor,
|
||||
)
|
||||
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
@@ -66,31 +69,41 @@ class ServeSubcommand(CLISubcommand):
|
||||
args.api_server_count = 0
|
||||
|
||||
# Detect LB mode for defaulting api_server_count.
|
||||
# Multi-port: --data-parallel-multi-port-external-lb
|
||||
# External LB: --data-parallel-external-lb or --data-parallel-rank
|
||||
# Hybrid LB: --data-parallel-hybrid-lb or --data-parallel-start-rank
|
||||
is_external_lb = (
|
||||
args.data_parallel_external_lb or args.data_parallel_rank is not None
|
||||
)
|
||||
is_hybrid_lb = (
|
||||
args.data_parallel_hybrid_lb or args.data_parallel_start_rank is not None
|
||||
)
|
||||
|
||||
if is_external_lb and is_hybrid_lb:
|
||||
# If --data_parallel_multi_port_external_lb and --data_parallel_hybrid_lb
|
||||
# are unset, default to hybrid if --data-parallel-start-rank is set
|
||||
is_hybrid_lb = is_multi_port = False
|
||||
if (
|
||||
not args.data_parallel_hybrid_lb
|
||||
and not args.data_parallel_multi_port_external_lb
|
||||
):
|
||||
is_hybrid_lb = args.data_parallel_start_rank is not None
|
||||
else:
|
||||
is_hybrid_lb = args.data_parallel_hybrid_lb
|
||||
is_multi_port = args.data_parallel_multi_port_external_lb
|
||||
|
||||
if sum([is_multi_port, is_external_lb, is_hybrid_lb]) > 1:
|
||||
raise ValueError(
|
||||
"Cannot use both external and hybrid data parallel load "
|
||||
"balancing modes. External LB is enabled via "
|
||||
"--data-parallel-external-lb or --data-parallel-rank. "
|
||||
"Hybrid LB is enabled via --data-parallel-hybrid-lb or "
|
||||
"--data-parallel-start-rank. Use one mode or the other."
|
||||
"Cannot use more than one data parallel load balancing mode. "
|
||||
"Choose one of: --data-parallel-multi-port-external-lb, "
|
||||
"--data-parallel-external-lb (or --data-parallel-rank), "
|
||||
"--data-parallel-hybrid-lb (or --data-parallel-start-rank)."
|
||||
)
|
||||
|
||||
# Default api_server_count if not explicitly set.
|
||||
# - Rust frontend: Use 1 (not applicable as it's multithreaded)
|
||||
# - External LB: Leave as 1 (external LB handles distribution)
|
||||
# - Multi-port: 1 (supervisor spawns one server per local DP rank)
|
||||
# - Rust frontend: 1 (not applicable as it's multithreaded)
|
||||
# - External LB: 1 (external LB handles distribution)
|
||||
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
|
||||
# - Internal LB: Use full DP size
|
||||
if args.api_server_count is None:
|
||||
if is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
|
||||
if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
|
||||
args.api_server_count = 1
|
||||
elif is_hybrid_lb:
|
||||
args.api_server_count = args.data_parallel_size_local or 1
|
||||
@@ -109,7 +122,8 @@ class ServeSubcommand(CLISubcommand):
|
||||
)
|
||||
elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
|
||||
logger.warning(
|
||||
"Ignoring --api-server-count=%d when using rust front-end process"
|
||||
"Ignoring --api-server-count=%d when using rust front-end process",
|
||||
args.api_server_count,
|
||||
)
|
||||
args.api_server_count = 1
|
||||
|
||||
@@ -122,7 +136,9 @@ class ServeSubcommand(CLISubcommand):
|
||||
)
|
||||
args.api_server_count = 1
|
||||
|
||||
if args.api_server_count < 1:
|
||||
if is_multi_port:
|
||||
run_dp_supervisor(args)
|
||||
elif args.api_server_count < 1:
|
||||
run_headless(args)
|
||||
elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
|
||||
run_multi_api_server(args)
|
||||
|
||||
@@ -665,8 +665,7 @@ async def build_and_serve_renderer(
|
||||
async def run_server(args, **uvicorn_kwargs) -> None:
|
||||
"""Run a single-worker API server."""
|
||||
|
||||
# Add process-specific prefix to stdout and stderr.
|
||||
decorate_logs("APIServer")
|
||||
decorate_logs("APIServer", skip_if_decorated=True)
|
||||
|
||||
# Interrupt initialization if SIGTERM arrives before uvicorn installs its
|
||||
# own signal handlers. Once uvicorn is running it replaces this.
|
||||
|
||||
@@ -229,6 +229,17 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
"""Host name."""
|
||||
port: int = 8000
|
||||
"""Port number."""
|
||||
data_parallel_supervisor_port: int = 9256
|
||||
"""HTTP port for aggregated health endpoints in multi-port external LB
|
||||
mode."""
|
||||
dp_supervisor_probe_interval_s: float = 5.0
|
||||
"""Seconds between aggregated health probes in multi-port external LB mode."""
|
||||
dp_supervisor_probe_timeout_s: float = 5.0
|
||||
"""Seconds to wait between retries when a child health probe fails with a
|
||||
connection error in multi-port external LB mode."""
|
||||
dp_supervisor_probe_failure_threshold: int = 3
|
||||
"""Number of consecutive connection-error retries before a child health
|
||||
probe is declared failed in multi-port external LB mode."""
|
||||
uds: str | None = None
|
||||
"""Unix domain socket path. If set, host and port arguments are ignored."""
|
||||
uvicorn_log_level: Literal[
|
||||
@@ -387,6 +398,13 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
if args.enable_log_outputs and not args.enable_log_requests:
|
||||
raise TypeError("Error: --enable-log-outputs requires --enable-log-requests")
|
||||
|
||||
if args.data_parallel_multi_port_external_lb:
|
||||
from vllm.entrypoints.openai.dp_supervisor import (
|
||||
validate_multi_port_external_lb_args,
|
||||
)
|
||||
|
||||
validate_multi_port_external_lb_args(args)
|
||||
|
||||
|
||||
def create_parser_for_docs() -> FlexibleArgumentParser:
|
||||
parser_for_docs = FlexibleArgumentParser(
|
||||
|
||||
@@ -0,0 +1,493 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from functools import partial
|
||||
from http import HTTPStatus
|
||||
from multiprocessing.process import BaseProcess
|
||||
|
||||
import aiohttp
|
||||
import psutil
|
||||
import uvicorn
|
||||
import uvloop
|
||||
from fastapi import FastAPI, Response
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import (
|
||||
decorate_logs,
|
||||
kill_process_tree,
|
||||
set_process_title,
|
||||
update_environment_variables,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
CHILD_EXIT_GRACE_S = 5.0
|
||||
|
||||
|
||||
def infer_multi_port_external_lb_start_rank(args: argparse.Namespace) -> int:
|
||||
start_rank = getattr(args, "data_parallel_start_rank", None)
|
||||
if start_rank is not None:
|
||||
return start_rank
|
||||
|
||||
node_rank = getattr(args, "node_rank", 0) or 0
|
||||
local_size = getattr(args, "data_parallel_size_local", 0) or 0
|
||||
return node_rank * local_size
|
||||
|
||||
|
||||
def validate_multi_port_external_lb_args(args: argparse.Namespace) -> None:
|
||||
if getattr(args, "grpc", False):
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb does not support --grpc"
|
||||
)
|
||||
if args.uds is not None:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb does not support --uds"
|
||||
)
|
||||
if any((args.ssl_keyfile, args.ssl_certfile, args.ssl_ca_certs)):
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb does not support HTTPS yet"
|
||||
)
|
||||
if args.api_server_count not in (None, 1):
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb currently requires "
|
||||
"--api-server-count=1"
|
||||
)
|
||||
if args.data_parallel_rank is not None:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb manages child "
|
||||
"--data-parallel-rank values internally"
|
||||
)
|
||||
if args.data_parallel_external_lb or args.data_parallel_hybrid_lb:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb cannot be combined with "
|
||||
"--data-parallel-external-lb or --data-parallel-hybrid-lb"
|
||||
)
|
||||
if args.data_parallel_size < 2:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb requires "
|
||||
"--data-parallel-size > 1"
|
||||
)
|
||||
|
||||
local_size = args.data_parallel_size_local
|
||||
if local_size is None or local_size < 2:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-multi-port-external-lb requires "
|
||||
"--data-parallel-size-local >= 2"
|
||||
)
|
||||
if local_size > args.data_parallel_size:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-size-local cannot exceed --data-parallel-size"
|
||||
)
|
||||
if args.data_parallel_size % local_size != 0:
|
||||
raise ValueError(
|
||||
"Error: --data-parallel-size must be divisible by "
|
||||
"--data-parallel-size-local"
|
||||
)
|
||||
|
||||
start_rank = infer_multi_port_external_lb_start_rank(args)
|
||||
if start_rank + local_size > args.data_parallel_size:
|
||||
raise ValueError(
|
||||
"Error: multi-port supervised ranks would exceed --data-parallel-size"
|
||||
)
|
||||
|
||||
supervisor_port = args.data_parallel_supervisor_port
|
||||
child_port_min = args.port
|
||||
child_port_max = args.port + local_size - 1
|
||||
if child_port_min <= supervisor_port <= child_port_max:
|
||||
raise ValueError(
|
||||
f"Error: --data-parallel-supervisor-port {supervisor_port} "
|
||||
f"overlaps with child rank ports {child_port_min}-{child_port_max}"
|
||||
)
|
||||
|
||||
|
||||
def _build_vllm_dp_server_args(
|
||||
args: argparse.Namespace, local_rank: int
|
||||
) -> argparse.Namespace:
|
||||
child_args = copy.copy(args)
|
||||
child_args.port = args.port + local_rank
|
||||
child_args.data_parallel_rank = (
|
||||
infer_multi_port_external_lb_start_rank(args) + local_rank
|
||||
)
|
||||
child_args.data_parallel_start_rank = None
|
||||
child_args.data_parallel_size_local = 1
|
||||
child_args.data_parallel_external_lb = True
|
||||
child_args.data_parallel_hybrid_lb = False
|
||||
child_args.data_parallel_multi_port_external_lb = False
|
||||
child_args.data_parallel_supervisor_port = None
|
||||
child_args.api_server_count = 1
|
||||
return child_args
|
||||
|
||||
|
||||
def _build_vllm_dp_server_env(
|
||||
args: argparse.Namespace, local_rank: int
|
||||
) -> dict[str, str]:
|
||||
# set visible devices for the child process
|
||||
devices_per_rank = args.tensor_parallel_size * args.pipeline_parallel_size
|
||||
start = local_rank * devices_per_rank
|
||||
stop = start + devices_per_rank
|
||||
device_env = current_platform.device_control_env_var
|
||||
visible_devices = ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(idx))
|
||||
for idx in range(start, stop)
|
||||
)
|
||||
return {device_env: visible_devices}
|
||||
|
||||
|
||||
def _child_base_url(args: argparse.Namespace, port: int) -> str:
|
||||
host = args.host or "127.0.0.1"
|
||||
if host == "0.0.0.0":
|
||||
host = "127.0.0.1"
|
||||
elif host == "::":
|
||||
host = "::1"
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
|
||||
def _join_processes_with_timeout(processes: list[BaseProcess], timeout: float) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
for process in processes:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
process.join(timeout=remaining)
|
||||
|
||||
|
||||
async def _probe_endpoint(
|
||||
session: aiohttp.ClientSession,
|
||||
args: argparse.Namespace,
|
||||
port: int,
|
||||
path: str,
|
||||
conn_err_failure_threshold: int = 3,
|
||||
conn_err_retry_delay: float = 5.0,
|
||||
) -> bool:
|
||||
"""
|
||||
Probe /health endpoint for 200 status.
|
||||
|
||||
If there is a connection error, retry every N seconds.
|
||||
"""
|
||||
for iteration in range(conn_err_failure_threshold):
|
||||
try:
|
||||
async with session.get(_child_base_url(args, port) + path) as response:
|
||||
# vLLM returns 503 on EngineDeadError, so we should return
|
||||
# immediately if vLLM responds with a non-200 status code.
|
||||
return response.status == HTTPStatus.OK
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError) as e:
|
||||
# Allow retry of connection errors.
|
||||
logger.debug(
|
||||
"Probe attempt %d/%d failed on port %d: %r",
|
||||
iteration + 1,
|
||||
conn_err_failure_threshold,
|
||||
port,
|
||||
e,
|
||||
)
|
||||
|
||||
if iteration < conn_err_failure_threshold - 1:
|
||||
await asyncio.sleep(conn_err_retry_delay)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _build_dp_supervisor_app(supervisor: DPSupervisor) -> FastAPI:
|
||||
app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
|
||||
app.state.supervisor = supervisor
|
||||
|
||||
def _status_response(ok: bool) -> Response:
|
||||
return Response(
|
||||
status_code=(HTTPStatus.OK if ok else HTTPStatus.SERVICE_UNAVAILABLE)
|
||||
)
|
||||
|
||||
@app.get("/health", include_in_schema=False)
|
||||
async def health() -> Response:
|
||||
return _status_response(app.state.supervisor.is_ready)
|
||||
|
||||
@app.get("/ready", include_in_schema=False)
|
||||
@app.get("/readyz", include_in_schema=False)
|
||||
async def ready() -> Response:
|
||||
return _status_response(app.state.supervisor.is_ready)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def _run_vllm_dp_server(
|
||||
child_args: argparse.Namespace, env_updates: dict[str, str]
|
||||
) -> None:
|
||||
"""
|
||||
Entrypoint function for the vLLM DP Server.
|
||||
"""
|
||||
from vllm.entrypoints.openai.api_server import run_server
|
||||
|
||||
# Create a fresh process group for the vLLM DP Server,
|
||||
# so that CTRL-C is propagated cleanly.
|
||||
os.setpgrp()
|
||||
|
||||
name = f"APIServer_DP{child_args.data_parallel_rank}"
|
||||
update_environment_variables(env_updates)
|
||||
set_process_title(name)
|
||||
decorate_logs(name)
|
||||
uvloop.run(run_server(child_args))
|
||||
|
||||
|
||||
class DPSupervisor:
|
||||
def __init__(self, args: argparse.Namespace):
|
||||
validate_multi_port_external_lb_args(args)
|
||||
self.args = args
|
||||
self.supervisor_port = args.data_parallel_supervisor_port
|
||||
self.child_ports = [
|
||||
args.port + local_rank
|
||||
for local_rank in range(args.data_parallel_size_local)
|
||||
]
|
||||
self._is_ready = False
|
||||
self._processes: list[BaseProcess] = []
|
||||
self._shutdown_event = asyncio.Event()
|
||||
self._shutdown_signal = signal.SIGTERM
|
||||
|
||||
@property
|
||||
def is_ready(self) -> bool:
|
||||
return self._is_ready and not self._shutdown_event.is_set()
|
||||
|
||||
async def run(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
# K8s sends SIGTERM for shutdown - begin graceful termination.
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, partial(self._handle_signal, sig))
|
||||
|
||||
# Launch DPSupervisor Server.
|
||||
app = _build_dp_supervisor_app(self)
|
||||
decorate_logs("DPSupervisor")
|
||||
host = self.args.host or "0.0.0.0"
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host=host,
|
||||
port=self.supervisor_port,
|
||||
log_level=self.args.uvicorn_log_level,
|
||||
)
|
||||
supervisor_server = uvicorn.Server(config)
|
||||
supervisor_server_task = asyncio.create_task(
|
||||
supervisor_server.serve(),
|
||||
name="dp-supervisor",
|
||||
)
|
||||
supervisor_server_task.add_done_callback(
|
||||
lambda _task: self._shutdown_event.set()
|
||||
)
|
||||
|
||||
# Ensure DPSupervisor task starts on the event loop.
|
||||
while not supervisor_server.started:
|
||||
if supervisor_server_task.done():
|
||||
supervisor_server_task.result()
|
||||
raise RuntimeError("DPSupervisor exited before startup.")
|
||||
await asyncio.sleep(0.05)
|
||||
logger.info("Started DPSupervisor on %s:%d", host, self.supervisor_port)
|
||||
|
||||
# Launch and Monitor vLLM Server Processes.
|
||||
try:
|
||||
self._start_children()
|
||||
await self._monitor_children()
|
||||
finally:
|
||||
self._is_ready = False
|
||||
await self._shutdown_children()
|
||||
|
||||
# Shutdown the DP Supervisor server.
|
||||
supervisor_server.should_exit = True
|
||||
await supervisor_server_task
|
||||
|
||||
def _handle_signal(self, signum: int) -> None:
|
||||
"""
|
||||
Signal handler that is added to the event loop.
|
||||
|
||||
This catches the SIGTERM from K8s and begins graceful shutdown,
|
||||
by setting the _shutdown_event(), which is watched by the main
|
||||
coroutine monitoring the vLLM DP Servers.
|
||||
"""
|
||||
|
||||
if self._shutdown_event.is_set():
|
||||
return
|
||||
|
||||
self._shutdown_signal = signal.Signals(signum)
|
||||
logger.info(
|
||||
"DPSupervisor received %s, shutting down.",
|
||||
self._shutdown_signal.name,
|
||||
)
|
||||
|
||||
self._shutdown_event.set()
|
||||
self._is_ready = False
|
||||
|
||||
def _start_children(self) -> None:
|
||||
"""
|
||||
Launch vLLM DP Servers on separate GPUs.
|
||||
"""
|
||||
logger.info("Launching vLLM DP Servers")
|
||||
context = multiprocessing.get_context("spawn")
|
||||
for local_rank in range(self.args.data_parallel_size_local):
|
||||
child_args = _build_vllm_dp_server_args(self.args, local_rank)
|
||||
child_env = _build_vllm_dp_server_env(self.args, local_rank)
|
||||
process = context.Process(
|
||||
target=_run_vllm_dp_server,
|
||||
name=f"APIServer_DPRank_{child_args.data_parallel_rank}",
|
||||
args=(child_args, child_env),
|
||||
)
|
||||
process.start()
|
||||
self._processes.append(process)
|
||||
|
||||
async def _probe_all_children(self) -> None:
|
||||
"""
|
||||
Background coroutine: probes all child endpoints on each interval.
|
||||
|
||||
Exits when any server becomes unhealthy after being ready, signalling
|
||||
_monitor_children to initiate shutdown.
|
||||
"""
|
||||
timeout = aiohttp.ClientTimeout(total=self.args.dp_supervisor_probe_timeout_s)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
while not self._shutdown_event.is_set():
|
||||
threshold = (
|
||||
self.args.dp_supervisor_probe_failure_threshold
|
||||
if self._is_ready
|
||||
else 1
|
||||
)
|
||||
results = await asyncio.gather(
|
||||
*(
|
||||
_probe_endpoint(
|
||||
session,
|
||||
self.args,
|
||||
port,
|
||||
"/health",
|
||||
conn_err_failure_threshold=threshold,
|
||||
conn_err_retry_delay=self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
for port in self.child_ports
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
all_healthy = all(r is True for r in results)
|
||||
|
||||
if all_healthy:
|
||||
# If all healthy, we are ready to receive requests.
|
||||
# This conditional avoids a potential race condition
|
||||
# where shutdown is set, THEN the probe returns true.
|
||||
if not self._shutdown_event.is_set():
|
||||
self._is_ready = True
|
||||
elif self._is_ready:
|
||||
# Once ready, any failure in the probe means vLLM is dead.
|
||||
num_unhealthy = sum(1 for r in results if r is not True)
|
||||
logger.info(
|
||||
"DPSupervisor probe found %s unhealthy DP Servers.",
|
||||
num_unhealthy,
|
||||
)
|
||||
self._is_ready = False
|
||||
self._shutdown_event.set()
|
||||
return
|
||||
|
||||
with contextlib.suppress(asyncio.TimeoutError):
|
||||
logger.debug(
|
||||
"Waiting for %s seconds before next probe",
|
||||
self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
await asyncio.wait_for(
|
||||
self._shutdown_event.wait(),
|
||||
timeout=self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
|
||||
async def _monitor_children(self) -> None:
|
||||
"""
|
||||
Main coroutine task that monitors the children vLLM servers.
|
||||
|
||||
Before the vLLM servers are /ready:
|
||||
- if the pid is dead, we will shut down
|
||||
- if the probe fails, we try again after dp_supervisor_probe_interval_s
|
||||
|
||||
After the vLLM servers are /ready:
|
||||
- if the pid is dead, we will shut down
|
||||
- if the probe fails, we will shut down
|
||||
"""
|
||||
probe_task = asyncio.create_task(
|
||||
self._probe_all_children(), name="dp-health-probe"
|
||||
)
|
||||
|
||||
try:
|
||||
while not self._shutdown_event.is_set():
|
||||
# 1. Check for dead processes
|
||||
n_failed = len([p for p in self._processes if not p.is_alive()])
|
||||
if n_failed > 0:
|
||||
logger.info("DPSupervisor found %s exited DP Servers.", n_failed)
|
||||
break
|
||||
|
||||
# 2. Check if the probe background task crashed or failed.
|
||||
if probe_task.done():
|
||||
# Extract exception if it crashed, or log failure
|
||||
exc = probe_task.exception() if not probe_task.cancelled() else None
|
||||
logger.info("DPSupervisor probe task stopped. Exception: %s", exc)
|
||||
break
|
||||
|
||||
# Sleep for probe_interval seconds or until a shutdown.
|
||||
with contextlib.suppress(asyncio.TimeoutError):
|
||||
logger.debug(
|
||||
"Waiting for %s seconds before next monitor",
|
||||
self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
await asyncio.wait_for(
|
||||
self._shutdown_event.wait(),
|
||||
timeout=self.args.dp_supervisor_probe_interval_s,
|
||||
)
|
||||
|
||||
finally:
|
||||
# Cleanup probe task if needed.
|
||||
if not probe_task.done():
|
||||
probe_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await probe_task
|
||||
|
||||
async def _shutdown_children(self) -> None:
|
||||
"""Terminate the vLLM DP servers."""
|
||||
timeout = self.args.shutdown_timeout + CHILD_EXIT_GRACE_S
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
"DPSupervisor forwarding %s to DP Servers.",
|
||||
self._shutdown_signal.name,
|
||||
)
|
||||
for process in self._processes:
|
||||
pid = process.pid
|
||||
if not process.is_alive() or pid is None:
|
||||
continue
|
||||
with contextlib.suppress(ProcessLookupError, OSError):
|
||||
os.kill(pid, self._shutdown_signal)
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
_join_processes_with_timeout, self._processes, timeout
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
logger.warning("Shutdown await cancelled")
|
||||
raise
|
||||
finally:
|
||||
for process in self._processes:
|
||||
pid = process.pid
|
||||
if not process.is_alive() or pid is None:
|
||||
continue
|
||||
logger.warning(
|
||||
"DP server %s did not exit within %.1fs; force killing.",
|
||||
process.name,
|
||||
timeout,
|
||||
)
|
||||
with contextlib.suppress(
|
||||
ProcessLookupError,
|
||||
OSError,
|
||||
psutil.NoSuchProcess,
|
||||
psutil.AccessDenied,
|
||||
):
|
||||
kill_process_tree(pid)
|
||||
|
||||
|
||||
def run_dp_supervisor(args: argparse.Namespace) -> None:
|
||||
uvloop.run(DPSupervisor(args).run())
|
||||
@@ -234,12 +234,17 @@ def _add_prefix(file: TextIO, worker_name: str, pid: int) -> None:
|
||||
file.write = write_with_prefix # type: ignore[method-assign]
|
||||
|
||||
|
||||
def decorate_logs(process_name: str | None = None) -> None:
|
||||
def decorate_logs(
|
||||
process_name: str | None = None, *, skip_if_decorated: bool = False
|
||||
) -> None:
|
||||
"""Decorate stdout/stderr with process name and PID prefix."""
|
||||
# Respect VLLM_CONFIGURE_LOGGING environment variable
|
||||
if not envs.VLLM_CONFIGURE_LOGGING:
|
||||
return
|
||||
|
||||
if skip_if_decorated and hasattr(sys.stdout, "_original_write"):
|
||||
return
|
||||
|
||||
if process_name is None:
|
||||
process_name = get_mp_context().current_process().name
|
||||
|
||||
|
||||
Reference in New Issue
Block a user