[Frontend] Move cli_args.py and dp_supervisor.py out openai folder (#53659)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-08-25 07:02:32 +00:00
committed by GitHub
parent c74dd2d208
commit 735daf8ef6
14 changed files with 42 additions and 33 deletions
-404
View File
@@ -1,404 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.utils.argparse_utils import FlexibleArgumentParser
from ...utils import VLLM_PATH
LORA_MODULE = {
"name": "module2",
"path": "/path/to/module2",
"base_model_name": "llama",
}
CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
assert CHATML_JINJA_PATH.exists()
def _build_vllm_parsers():
vllm_parser = FlexibleArgumentParser()
subparsers = vllm_parser.add_subparsers()
serve_parser = subparsers.add_parser("serve")
make_arg_parser(serve_parser)
return {"vllm": vllm_parser, "vllm serve": serve_parser}
@pytest.fixture
def vllm_parser():
return _build_vllm_parsers()["vllm"]
@pytest.fixture
def serve_parser():
return _build_vllm_parsers()["vllm serve"]
### Test config parsing
def test_config_arg_parsing(serve_parser, cli_config_file):
args = serve_parser.parse_args([])
assert args.port == 8000
args = serve_parser.parse_args(["--config", cli_config_file])
assert args.port == 12312
args = serve_parser.parse_args(
[
"--config",
cli_config_file,
"--port",
"9000",
]
)
assert args.port == 9000
args = serve_parser.parse_args(
[
"--port",
"9000",
"--config",
cli_config_file,
]
)
assert args.port == 9000
### Tests for LoRA module parsing
def test_valid_key_value_format(serve_parser):
# Test old format: name=path
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
]
)
expected = [LoRAModulePath(name="module1", path="/path/to/module1")]
assert args.lora_modules == expected
def test_valid_json_format(serve_parser):
# Test valid JSON format input
args = serve_parser.parse_args(
[
"--lora-modules",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama")
]
assert args.lora_modules == expected
def test_invalid_json_format(serve_parser):
# Test invalid JSON format input, missing closing brace
with pytest.raises(SystemExit):
serve_parser.parse_args(
["--lora-modules", '{"name": "module3", "path": "/path/to/module3"']
)
def test_invalid_type_error(serve_parser):
# Test type error when values are not JSON or key=value
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
"invalid_format", # This is not JSON or key=value format
]
)
def test_invalid_json_field(serve_parser):
# Test valid JSON format but missing required fields
with pytest.raises(SystemExit):
serve_parser.parse_args(
[
"--lora-modules",
'{"name": "module4"}', # Missing required 'path' field
]
)
def test_empty_values(serve_parser):
# Test when no LoRA modules are provided
args = serve_parser.parse_args(["--lora-modules", ""])
assert args.lora_modules == []
def test_multiple_valid_inputs(serve_parser):
# Test multiple valid inputs (both old and JSON format)
args = serve_parser.parse_args(
[
"--lora-modules",
"module1=/path/to/module1",
json.dumps(LORA_MODULE),
]
)
expected = [
LoRAModulePath(name="module1", path="/path/to/module1"),
LoRAModulePath(
name="module2", path="/path/to/module2", base_model_name="llama"
),
]
assert args.lora_modules == expected
### Tests for serve argument validation that run prior to loading
def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
"""Ensure validation fails if tool choice is enabled with no call parser"""
# If we enable-auto-tool-choice, explode with no tool-call-parser
args = serve_parser.parse_args(args=["--enable-auto-tool-choice"])
with pytest.raises(TypeError):
validate_parsed_serve_args(args)
def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
"""Ensure validation passes with tool choice enabled with a call parser"""
args = serve_parser.parse_args(
args=[
"--enable-auto-tool-choice",
"--tool-call-parser",
"mistral",
]
)
validate_parsed_serve_args(args)
def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser):
"""Ensure validation fails if reasoning is enabled with auto tool choice"""
args = serve_parser.parse_args(
args=[
"--enable-auto-tool-choice",
"--reasoning-parser",
"deepseek_r1",
]
)
with pytest.raises(TypeError):
validate_parsed_serve_args(args)
def test_passes_with_reasoning_parser(serve_parser):
"""Ensure validation passes if reasoning is enabled
with a reasoning parser"""
args = serve_parser.parse_args(
args=[
"--reasoning-parser",
"deepseek_r1",
]
)
validate_parsed_serve_args(args)
def test_chat_template_validation_for_happy_paths(serve_parser):
"""Ensure validation passes if the chat template exists"""
args = serve_parser.parse_args(
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
)
validate_parsed_serve_args(args)
def test_chat_template_validation_for_sad_paths(serve_parser):
"""Ensure validation fails if the chat template doesn't exist"""
args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"])
with pytest.raises(ValueError):
validate_parsed_serve_args(args)
def test_per_request_metrics_requires_log_stats(serve_parser):
args = serve_parser.parse_args(
args=["--enable-per-request-metrics", "--disable-log-stats"]
)
with pytest.raises(ValueError):
validate_parsed_serve_args(args)
def _build_launch_render_parser():
"""Mirror `vllm launch render`.
`vllm/entrypoints/cli/main.py` parses subcommands with ``dest="subparser"``,
and `LaunchSubcommandBase.add_cli_args` builds the component parser with
`make_arg_parser`, so a launch component carries serve args under
``args.subparser == "launch"``.
"""
vllm_parser = FlexibleArgumentParser()
subparsers = vllm_parser.add_subparsers(required=False, dest="subparser")
launch_parser = subparsers.add_parser("launch")
launch_subparsers = launch_parser.add_subparsers(
required=True, dest="launch_component"
)
render_parser = launch_subparsers.add_parser("render")
make_arg_parser(render_parser)
return vllm_parser
@pytest.fixture
def launch_render_parser():
return _build_launch_render_parser()
def test_launch_render_validates_serve_args(launch_render_parser):
"""`vllm launch render` reuses the serve parser, so it gets the serve checks"""
args = launch_render_parser.parse_args(
args=["launch", "render", "--enable-auto-tool-choice"]
)
assert args.subparser == "launch"
with pytest.raises(TypeError):
validate_parsed_serve_args(args)
def test_subcommand_without_serve_args_skips_validation():
"""A subcommand that does not use the serve parser is still skipped"""
vllm_parser = FlexibleArgumentParser()
subparsers = vllm_parser.add_subparsers(required=False, dest="subparser")
subparsers.add_parser("chat")
args = vllm_parser.parse_args(args=["chat"])
validate_parsed_serve_args(args)
@pytest.mark.parametrize(
"cli_args, expected",
[
([], 0),
(["--sse-keep-alive-interval", "0"], 0),
(["--sse-keep-alive-interval", "15"], 15),
],
)
def test_sse_keep_alive_interval_valid(serve_parser, cli_args, expected):
"""The default is 0 and zero or positive integers parse exactly."""
args = serve_parser.parse_args(args=cli_args)
assert args.sse_keep_alive_interval == expected
validate_parsed_serve_args(args)
@pytest.mark.parametrize("value", ["-1", "-5"])
def test_sse_keep_alive_interval_negative(serve_parser, value):
"""Negative intervals are rejected by validation."""
args = serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
with pytest.raises(ValueError):
validate_parsed_serve_args(args)
@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "0.5", "abc"])
def test_sse_keep_alive_interval_non_integer(serve_parser, value):
"""Non-integer values fail argparse's int parsing with SystemExit."""
with pytest.raises(SystemExit):
serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
@pytest.mark.parametrize(
"cli_args, expected_middleware",
[
(
["--middleware", "middleware1", "--middleware", "middleware2"],
["middleware1", "middleware2"],
),
([], []),
],
)
def test_middleware(serve_parser, cli_args, expected_middleware):
"""Ensure multiple middleware args are parsed properly"""
args = serve_parser.parse_args(args=cli_args)
assert args.middleware == expected_middleware
def test_default_chat_template_kwargs_parsing(serve_parser):
"""Ensure default_chat_template_kwargs JSON is parsed correctly"""
args = serve_parser.parse_args(
args=["--default-chat-template-kwargs", '{"enable_thinking": false}']
)
assert args.default_chat_template_kwargs == {"enable_thinking": False}
def test_default_chat_template_kwargs_complex(serve_parser):
"""Ensure complex default_chat_template_kwargs JSON is parsed correctly"""
kwargs_json = '{"enable_thinking": false, "custom_param": "value", "num": 42}'
args = serve_parser.parse_args(args=["--default-chat-template-kwargs", kwargs_json])
assert args.default_chat_template_kwargs == {
"enable_thinking": False,
"custom_param": "value",
"num": 42,
}
def test_default_chat_template_kwargs_default_none(serve_parser):
"""Ensure default_chat_template_kwargs defaults to None"""
args = serve_parser.parse_args(args=[])
assert args.default_chat_template_kwargs is None
def test_default_chat_template_kwargs_invalid_json(serve_parser):
"""Ensure invalid JSON raises an error"""
with pytest.raises(SystemExit):
serve_parser.parse_args(
args=["--default-chat-template-kwargs", "not valid json"]
)
@pytest.mark.parametrize(
"args, raises",
[
(["user/model"], None),
(["user/model", "--served-model-name", "model"], None),
(["--served-model-name", "model", "user/model"], ValueError),
(["--served-model-name", "model", "--config", "config.yaml"], None),
(["--served-model-name", "model", "--config", "config.yaml"], ValueError),
],
ids=[
"model_tag_only",
"model_tag_with_served_model_name",
"served_model_name_before_model_tag",
"served_model_name_with_model_in_config",
"served_model_name_with_no_model_in_config",
],
)
def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
"""Ensure that users don't misuse --served-model-name and end up with the default
model tag instead of the one they intended to serve."""
# Call the serve subparser
args.insert(0, "serve")
# Create a dummy config file if the test case includes it
if "config.yaml" in args:
# Create a dummy config file if the test case includes it
config_path = tmp_path / "config.yaml"
config_path.write_text("model: user/model" if raises is None else "port: 8000")
args[args.index("config.yaml")] = config_path.as_posix()
# Do the parsing and check for expected exceptions or values
if raises is None:
parsed_args = vllm_parser.parse_args(args=args)
expected = "user/model"
assert parsed_args.model_tag == expected or parsed_args.model == expected
else:
with pytest.raises(raises):
vllm_parser.parse_args(args=args)
### Tests for LoRA target modules parsing
def test_lora_target_modules_single(serve_parser):
"""Test parsing single lora-target-modules argument"""
args = serve_parser.parse_args(
args=["--enable-lora", "--lora-target-modules", "o_proj"]
)
assert args.lora_target_modules == ["o_proj"]
def test_lora_target_modules_multiple(serve_parser):
"""Test parsing multiple lora-target-modules arguments"""
args = serve_parser.parse_args(
args=[
"--enable-lora",
"--lora-target-modules",
"o_proj",
"qkv_proj",
"down_proj",
]
)
assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
def test_lora_target_modules_default_none(serve_parser):
"""Test that lora-target-modules defaults to None"""
args = serve_parser.parse_args(args=[])
assert args.lora_target_modules is None
@@ -1,927 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for DPSupervisor: unit tests and lifecycle integration tests.
Lifecycle integration tests replace child vLLM servers with lightweight
aiohttp "fake" servers controlled by the test, so the suite runs without GPUs.
_start_children is monkeypatched to install FakeProcess objects (with
controllable liveness/timing) alongside those fake HTTP servers.
Port allocation (kept far from default vLLM ports to avoid conflicts):
Supervisor : 19256
Children : 18000, 18001
"""
from __future__ import annotations
import argparse
import asyncio
import contextlib
import os
import signal
import subprocess
import tempfile
import time
from pathlib import Path
from types import SimpleNamespace
import aiohttp
import pytest
import uvicorn
from fastapi import FastAPI, Response
import vllm.entrypoints.openai.dp_supervisor as dp_sup
from vllm.entrypoints.openai.dp_supervisor import (
CHILD_EXIT_GRACE_S,
DPSupervisor,
_build_vllm_dp_server_args,
infer_multi_port_external_lb_start_rank,
validate_multi_port_external_lb_args,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_SUPERVISOR_PORT = 19256
_CHILD_PORT_BASE = 18000
_N_CHILDREN = 2
_PROBE_INTERVAL = 1.0
_POLL_INTERVAL = 1.0
# ---------------------------------------------------------------------------
# Args factories
# ---------------------------------------------------------------------------
def _make_unit_args(**overrides) -> argparse.Namespace:
"""Minimal args for unit tests (no real network activity)."""
base = {
"host": None,
"port": 8000,
"data_parallel_multi_port_external_lb": True,
"data_parallel_supervisor_port": 9256,
"dp_supervisor_probe_interval_s": 5.0,
"dp_supervisor_probe_timeout_s": 5.0,
"dp_supervisor_probe_failure_threshold": 3,
"data_parallel_size": 8,
"data_parallel_size_local": 4,
"data_parallel_start_rank": None,
"data_parallel_rank": None,
"data_parallel_external_lb": False,
"data_parallel_hybrid_lb": False,
"api_server_count": None,
"headless": False,
"grpc": False,
"uds": None,
"ssl_keyfile": None,
"ssl_certfile": None,
"ssl_ca_certs": None,
"ssl_cert_reqs": 0,
"ssl_ciphers": None,
"node_rank": 1,
"tensor_parallel_size": 1,
"pipeline_parallel_size": 1,
"uvicorn_log_level": "info",
"shutdown_timeout": 5.0,
"disable_uvicorn_access_log": False,
"disable_access_log_for_endpoints": None,
"log_config_file": None,
}
base.update(overrides)
return argparse.Namespace(**base)
def _make_args(**overrides) -> argparse.Namespace:
"""Args for lifecycle integration tests (real loopback servers)."""
base: dict = dict(
host="127.0.0.1",
port=_CHILD_PORT_BASE,
data_parallel_multi_port_external_lb=True,
data_parallel_supervisor_port=_SUPERVISOR_PORT,
dp_supervisor_probe_interval_s=_PROBE_INTERVAL,
dp_supervisor_probe_timeout_s=1.0,
dp_supervisor_probe_failure_threshold=3,
data_parallel_size=_N_CHILDREN,
data_parallel_size_local=_N_CHILDREN,
data_parallel_start_rank=0,
data_parallel_rank=None,
data_parallel_external_lb=False,
data_parallel_hybrid_lb=False,
api_server_count=None,
headless=False,
grpc=False,
uds=None,
ssl_keyfile=None,
ssl_certfile=None,
ssl_ca_certs=None,
ssl_cert_reqs=0,
ssl_ciphers=None,
node_rank=0,
tensor_parallel_size=1,
pipeline_parallel_size=1,
uvicorn_log_level="warning",
shutdown_timeout=0.0,
disable_uvicorn_access_log=False,
disable_access_log_for_endpoints=None,
log_config_file=None,
)
base.update(overrides)
return argparse.Namespace(**base)
def _generate_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]:
"""Generate a self-signed certificate for HTTPS lifecycle tests."""
cert_file = cert_dir / "cert.pem"
key_file = cert_dir / "key.pem"
subprocess.run(
[
"openssl",
"req",
"-x509",
"-newkey",
"rsa:2048",
"-keyout",
str(key_file),
"-out",
str(cert_file),
"-days",
"1",
"-nodes",
"-subj",
"/CN=localhost",
],
check=True,
capture_output=True,
)
return cert_file, key_file
# ---------------------------------------------------------------------------
# Unit tests
# ---------------------------------------------------------------------------
def test_infer_multi_port_external_lb_start_rank_uses_node_rank():
args = _make_unit_args()
assert infer_multi_port_external_lb_start_rank(args) == 4
def test_build_multi_port_external_lb_child_args_sets_external_rank_server():
args = _make_unit_args(data_parallel_start_rank=8, api_server_count=None)
child_args = _build_vllm_dp_server_args(args, local_rank=2)
assert child_args.port == 8002
assert child_args.data_parallel_rank == 10
assert child_args.data_parallel_size_local == 1
assert child_args.data_parallel_external_lb is True
assert child_args.data_parallel_hybrid_lb is False
assert child_args.data_parallel_multi_port_external_lb is False
assert child_args.api_server_count == 1
def test_run_vllm_dp_server_uses_python_server_by_default(monkeypatch):
calls: list[str] = []
monkeypatch.setattr(dp_sup.os, "setpgrp", lambda: None)
monkeypatch.setattr(dp_sup, "set_process_title", lambda *_args: None)
monkeypatch.setattr(dp_sup, "decorate_logs", lambda *_args: None)
monkeypatch.setattr(dp_sup.envs, "VLLM_RUST_FRONTEND_PATH", None, raising=False)
monkeypatch.setattr(
dp_sup, "_run_python_vllm_dp_server", lambda _args: calls.append("python")
)
monkeypatch.setattr(
dp_sup, "_run_rust_vllm_dp_server", lambda _args: calls.append("rust")
)
dp_sup._run_vllm_dp_server(_make_unit_args(data_parallel_rank=4))
assert calls == ["python"]
def test_run_vllm_dp_server_uses_rust_frontend_when_enabled(monkeypatch):
calls: list[str] = []
monkeypatch.setattr(dp_sup.os, "setpgrp", lambda: None)
monkeypatch.setattr(dp_sup, "set_process_title", lambda *_args: None)
monkeypatch.setattr(dp_sup, "decorate_logs", lambda *_args: None)
monkeypatch.setattr(dp_sup.envs, "VLLM_USE_RUST_FRONTEND", True, raising=False)
monkeypatch.setattr(
dp_sup.envs,
"VLLM_RUST_FRONTEND_PATH",
"/tmp/vllm-rs",
raising=False,
)
monkeypatch.setattr(
dp_sup, "_run_python_vllm_dp_server", lambda _args: calls.append("python")
)
monkeypatch.setattr(
dp_sup, "_run_rust_vllm_dp_server", lambda _args: calls.append("rust")
)
dp_sup._run_vllm_dp_server(_make_unit_args(data_parallel_rank=4))
assert calls == ["rust"]
def test_validate_multi_port_external_lb_args_allows_ssl():
args = _make_unit_args(
ssl_keyfile="/tmp/server.key",
ssl_certfile="/tmp/server.crt",
ssl_ca_certs="/tmp/ca.crt",
)
validate_multi_port_external_lb_args(args)
def test_aggregates_health():
supervisor = DPSupervisor(_make_unit_args())
supervisor._is_ready = True
assert supervisor.is_ready is True
def test_handles_shutdown_event():
supervisor = DPSupervisor(_make_unit_args())
supervisor._is_ready = True
supervisor._shutdown_event.set()
assert supervisor.is_ready is False
@pytest.mark.asyncio
async def test_shutdown_children_uses_engine_process_timeout(
monkeypatch: pytest.MonkeyPatch,
):
supervisor = DPSupervisor(_make_unit_args(shutdown_timeout=0.0))
supervisor._processes = [
SimpleNamespace(name="APIServer_DPRank_4", pid=None, is_alive=lambda: False)
]
calls = []
timeout_calls = []
def get_process_timeout(request_timeout, manager_timeout):
timeout_calls.append((request_timeout, manager_timeout))
return 15.0
monkeypatch.setattr(
dp_sup,
"get_engine_process_shutdown_timeout",
get_process_timeout,
)
monkeypatch.setattr(
dp_sup,
"_join_processes_with_timeout",
lambda processes, timeout: calls.append((processes, timeout)),
)
await supervisor._shutdown_children()
assert timeout_calls == [(0.0, 0.0)]
assert calls == [
(supervisor._processes, 15.0 + CHILD_EXIT_GRACE_S),
]
@pytest.mark.asyncio
async def test_handles_child_exit(
monkeypatch: pytest.MonkeyPatch,
):
supervisor = DPSupervisor(_make_unit_args())
supervisor._processes = [
SimpleNamespace(
name="APIServer_DPRank_4", exitcode=None, is_alive=lambda: True
),
SimpleNamespace(name="APIServer_DPRank_5", exitcode=17, is_alive=lambda: False),
]
async def fake_probe(*_args, **_kwargs) -> bool:
return True
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
await supervisor._monitor_children()
assert supervisor._is_ready is False
@pytest.mark.asyncio
async def test_handles_probe_failure(
monkeypatch: pytest.MonkeyPatch,
):
supervisor = DPSupervisor(_make_unit_args(dp_supervisor_probe_interval_s=0.0))
supervisor.child_ports = [8000]
probe_results = iter([True, False])
async def fake_probe(*_args, **_kwargs) -> bool:
return next(probe_results)
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
await supervisor._monitor_children()
assert supervisor._is_ready is False
@pytest.mark.asyncio
async def test_shutdown_if_supervisor_server_error_on_startup(
monkeypatch: pytest.MonkeyPatch,
):
class FakeLoop:
def add_signal_handler(self, *_args, **_kwargs):
pass
def remove_signal_handler(self, *_args, **_kwargs):
pass
class FakeServer:
def __init__(self, _config):
self.started = False
self.should_exit = False
async def serve(self):
raise ValueError("supervisor boom")
async def fake_shutdown_children(self):
return None
def fake_start_children(self):
return None
async def fake_monitor_children(self):
# Mark ready so the supervisor server is started, then block until
# shutdown (triggered when the failing server task exits).
self._is_ready = True
await self._shutdown_event.wait()
monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop())
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children)
monkeypatch.setattr(DPSupervisor, "_start_children", fake_start_children)
monkeypatch.setattr(DPSupervisor, "_monitor_children", fake_monitor_children)
supervisor = DPSupervisor(_make_unit_args())
with pytest.raises(ValueError, match="supervisor boom"):
await supervisor.run()
# ---------------------------------------------------------------------------
# Access-log filtering unit tests
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_start_server_applies_access_log_filter(monkeypatch):
"""get_uvicorn_log_config result is forwarded to uvicorn.Config."""
sentinel_config = {"version": 1, "loggers": {}}
captured_config: list[uvicorn.Config] = []
monkeypatch.setattr(dp_sup, "get_uvicorn_log_config", lambda _args: sentinel_config)
class FakeServer:
def __init__(self, config):
captured_config.append(config)
self.started = True
self.should_exit = False
async def serve(self):
pass
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
args = _make_unit_args(
host="127.0.0.1",
disable_access_log_for_endpoints="/health,/readyz",
)
supervisor = DPSupervisor(args)
await supervisor._start_server()
assert len(captured_config) == 1
config = captured_config[0]
assert config.log_config is sentinel_config
assert config.access_log is True
@pytest.mark.asyncio
async def test_start_server_disables_access_log(monkeypatch):
"""disable_uvicorn_access_log=True sets access_log=False."""
captured_config: list[uvicorn.Config] = []
monkeypatch.setattr(dp_sup, "get_uvicorn_log_config", lambda _args: None)
class FakeServer:
def __init__(self, config):
captured_config.append(config)
self.started = True
self.should_exit = False
async def serve(self):
pass
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
args = _make_unit_args(
host="127.0.0.1",
disable_uvicorn_access_log=True,
)
supervisor = DPSupervisor(args)
await supervisor._start_server()
assert len(captured_config) == 1
assert captured_config[0].access_log is False
@pytest.mark.asyncio
async def test_start_server_no_log_config_when_no_filter(monkeypatch):
"""Without filtering options, default uvicorn log_config is used."""
captured_config: list[uvicorn.Config] = []
monkeypatch.setattr(dp_sup, "get_uvicorn_log_config", lambda _args: None)
class FakeServer:
def __init__(self, config):
captured_config.append(config)
self.started = True
self.should_exit = False
async def serve(self):
pass
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
args = _make_unit_args(host="127.0.0.1")
supervisor = DPSupervisor(args)
await supervisor._start_server()
assert len(captured_config) == 1
config = captured_config[0]
assert config.log_config == uvicorn.Config(app=None).log_config
assert config.access_log is True
# ---------------------------------------------------------------------------
# Lifecycle integration tests – MockVLLMServer
# ---------------------------------------------------------------------------
class MockVLLMServer:
"""
Minimal FastAPI server that mimics one vLLM replica.
GET /health returns 200 when healthy, 503 otherwise.
Health state is toggled by the test via set_healthy().
"""
def __init__(
self,
port: int,
drain_seconds: float = 0.0,
ssl_keyfile: str | None = None,
ssl_certfile: str | None = None,
) -> None:
self.port = port
self._healthy = False
self._drain_seconds = drain_seconds
self._ssl_keyfile = ssl_keyfile
self._ssl_certfile = ssl_certfile
self._server: uvicorn.Server | None = None
self._serve_task: asyncio.Task | None = None
async def start(self) -> None:
app = FastAPI()
@app.get("/health")
async def health() -> Response:
print(f"MockServer {self.port}: /health: {self._healthy}")
return Response(status_code=200 if self._healthy else 503)
@app.get("/set_healthy")
async def set_healthy() -> Response:
print(f"MockServer {self.port}: /set_healthy")
self._healthy = True
return Response(status_code=200)
@app.get("/set_unhealthy")
async def set_unhealthy() -> Response:
print(f"MockServer {self.port}: /set_unhealthy")
self._healthy = False
return Response(status_code=200)
@app.get("/kill")
async def kill() -> Response:
print(f"MockServer {self.port}: /kill")
os.kill(os.getpid(), signal.SIGKILL)
config = uvicorn.Config(
app,
host="127.0.0.1",
port=self.port,
log_level="warning",
lifespan="off",
ssl_keyfile=self._ssl_keyfile,
ssl_certfile=self._ssl_certfile,
)
self._server = uvicorn.Server(config)
# Configure request draining if needed.
# Uvicorn's capture_signals() installs signal.signal(SIGTERM, self.handle_exit),
# which sets should_exit=True immediately. Override handle_exit on the instance
# so capture_signals() picks up our version that drains first.
if self._drain_seconds > 0:
self._shutdown_event = asyncio.Event()
loop = asyncio.get_running_loop()
async def _drain_and_stop() -> None:
await self._shutdown_event.wait()
print(f"MockServer {self.port}: draining for {self._drain_seconds}s.")
await asyncio.sleep(self._drain_seconds)
print("Setting should_exit")
if self._server is not None:
self._server.should_exit = True
self._drain_task = asyncio.create_task(_drain_and_stop())
def _custom_handle_exit(sig: int, frame: object) -> None:
print("Got SIGTERM, setting shutdown.")
if not self._shutdown_event.is_set():
loop.call_soon_threadsafe(self._shutdown_event.set)
self._server.handle_exit = _custom_handle_exit
self._serve_task = asyncio.create_task(self._server.serve())
while not self._server.started:
await asyncio.sleep(0.01)
print(f"Mock DP Server on port {self.port} started")
await self._serve_task
def launch_mock_vllm(child_args: argparse.Namespace):
logger.info("Launching mock vLLM on port %s", child_args.port)
mock_vllm = MockVLLMServer(
port=child_args.port,
ssl_keyfile=child_args.ssl_keyfile,
ssl_certfile=child_args.ssl_certfile,
)
asyncio.run(mock_vllm.start())
def launch_mock_vllm_with_drain(
child_args: argparse.Namespace,
):
logger.info("Launching mock vLLM with 15s drain on port %s", child_args.port)
mock_vllm = MockVLLMServer(
port=child_args.port,
drain_seconds=10.0,
ssl_keyfile=child_args.ssl_keyfile,
ssl_certfile=child_args.ssl_certfile,
)
asyncio.run(mock_vllm.start())
# ---------------------------------------------------------------------------
# Lifecycle test helpers
# ---------------------------------------------------------------------------
async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -> bool:
"""
GET /health on the supervisor once and check for expected_status.
Pass expected_status=-1 to assert the supervisor is not listening yet
(a connection error is expected). The supervisor only starts its HTTP
server once every child is ready, so /health is refused until then.
"""
scheme = "https" if use_ssl else "http"
url = f"{scheme}://127.0.0.1:{_SUPERVISOR_PORT}/health"
async with aiohttp.ClientSession() as session:
try:
async with session.get(url, ssl=False if use_ssl else None) as resp:
if resp.status != expected_status:
print(f"expected: {expected_status=}, got: {resp.status=}")
return False
return True
except aiohttp.ClientError:
if expected_status != -1:
print(f"expected: {expected_status=}, got: aiohttp.ClientError")
return False
return True
async def _await_supervisor_health(
expected_status: int, use_ssl: bool = False, retries: int = 20
) -> bool:
"""Retry _poll_supervisor_health, tolerating supervisor server startup."""
for _ in range(retries):
if await _poll_supervisor_health(expected_status, use_ssl=use_ssl):
return True
await asyncio.sleep(0.5)
return False
async def _poll_until_api_server_running(
port: int, timeout_s: float = 30.0, use_ssl: bool = False
) -> None:
"""Return once the child accepts a request; it reports 503 until healthy."""
scheme = "https" if use_ssl else "http"
url = f"{scheme}://127.0.0.1:{port}/health"
deadline = time.monotonic() + timeout_s
last_exc: Exception | None = None
async with aiohttp.ClientSession() as session:
while (remaining_s := deadline - time.monotonic()) > 0:
try:
request_timeout = aiohttp.ClientTimeout(total=min(2.0, remaining_s))
async with session.get(
url,
ssl=False if use_ssl else None,
timeout=request_timeout,
):
return
except (aiohttp.ClientError, asyncio.TimeoutError) as exc:
last_exc = exc
print("Test detected not started yet, sleeping for 1s")
remaining_s = deadline - time.monotonic()
if remaining_s > 0:
await asyncio.sleep(min(1.0, remaining_s))
raise TimeoutError(
f"API server on port {port} did not start within {timeout_s}s"
) from last_exc
async def _set_healthy(port: int, use_ssl: bool = False) -> None:
scheme = "https" if use_ssl else "http"
url = f"{scheme}://127.0.0.1:{port}/set_healthy"
async with (
aiohttp.ClientSession() as session,
session.get(url, ssl=False if use_ssl else None) as resp,
):
assert resp.status == 200
async def _set_unhealthy(port: int, use_ssl: bool = False) -> None:
scheme = "https" if use_ssl else "http"
url = f"{scheme}://127.0.0.1:{port}/set_unhealthy"
async with (
aiohttp.ClientSession() as session,
session.get(url, ssl=False if use_ssl else None) as resp,
):
assert resp.status == 200
async def _kill_server(port: int, use_ssl: bool = False) -> None:
scheme = "https" if use_ssl else "http"
url = f"{scheme}://127.0.0.1:{port}/kill"
try:
async with (
aiohttp.ClientSession() as session,
session.get(url, ssl=False if use_ssl else None) as resp,
):
assert resp.status != 200
except aiohttp.ClientConnectionError:
return
@contextlib.asynccontextmanager
async def _run_supervisor(
args: argparse.Namespace,
monkeypatch: pytest.MonkeyPatch,
launch_fn=None,
):
if launch_fn is None:
launch_fn = launch_mock_vllm
monkeypatch.setattr(dp_sup, "_run_vllm_dp_server", launch_fn)
supervisor = DPSupervisor(args)
task = asyncio.create_task(supervisor.run())
await asyncio.sleep(1.0)
try:
yield supervisor, task
finally:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
# ---------------------------------------------------------------------------
# Lifecycle integration tests
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_basic_lifecycle(monkeypatch):
"""
A) Supervisor is not listening while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) SIGTERM and shutdown
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
for port in vllm_server_ports:
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
await _poll_until_api_server_running(port)
await _set_healthy(vllm_server_ports[0])
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
print("supervisor not listening --- expected!")
for port in vllm_server_ports:
await _set_healthy(port)
assert await _await_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
os.kill(os.getpid(), signal.SIGTERM)
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
print("everything was cleaned up!")
@pytest.mark.asyncio
async def test_basic_lifecycle_with_ssl(monkeypatch):
with tempfile.TemporaryDirectory() as cert_dir:
cert_file, key_file = _generate_self_signed_cert(Path(cert_dir))
args = _make_args(
ssl_keyfile=str(key_file),
ssl_certfile=str(cert_file),
)
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(-1, use_ssl=True)
assert not supervisor.is_ready
for port in vllm_server_ports:
assert await _poll_supervisor_health(-1, use_ssl=True)
assert not supervisor.is_ready
await _poll_until_api_server_running(port, use_ssl=True)
for port in vllm_server_ports:
await _set_healthy(port, use_ssl=True)
assert await _await_supervisor_health(200, use_ssl=True)
assert supervisor.is_ready
@pytest.mark.asyncio
async def test_failed_startup(monkeypatch):
"""
A) One of the vLLM servers crashes during startup.
B) DPSupervisor detects this, and cleans up resources.
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
for port in vllm_server_ports:
await _poll_until_api_server_running(port)
await _kill_server(port)
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
@pytest.mark.asyncio
async def test_becomes_unhealthy(monkeypatch):
"""
A) Supervisor is not listening while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) Child process becomes unhealthy.
D) Detected and shutdown.
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
for port in vllm_server_ports:
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
await _poll_until_api_server_running(port)
await _set_healthy(vllm_server_ports[0])
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
print("supervisor not listening --- expected!")
for port in vllm_server_ports:
await _set_healthy(port)
assert await _await_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
await _set_unhealthy(port)
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
print("everything was cleaned up!")
@pytest.mark.asyncio
async def test_dp_server_fails(monkeypatch):
"""
A) Supervisor is not listening while children are unhealthy.
B) /health returns 200 once every child reports healthy.
C) Child process fails.
D) Detected and shutdown.
"""
args = _make_args()
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
for port in vllm_server_ports:
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
await _poll_until_api_server_running(port)
await _set_healthy(vllm_server_ports[0])
await asyncio.sleep(1.0)
assert await _poll_supervisor_health(-1)
assert not supervisor.is_ready
print("supervisor not listening --- expected!")
for port in vllm_server_ports:
await _set_healthy(port)
assert await _await_supervisor_health(200)
assert supervisor.is_ready
print("/health is 200 --- expected!")
dp_mock_server_process = supervisor._processes[0]
os.kill(dp_mock_server_process.pid, signal.SIGKILL)
await asyncio.sleep(1.0)
assert not dp_mock_server_process.is_alive()
await asyncio.wait_for(_task, timeout=5.0)
for p in supervisor._processes:
assert not p.is_alive()
print("everything was cleaned up!")
@pytest.mark.asyncio
async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch):
"""
Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
request drain). The supervisor is configured with shutdown_timeout=10,
so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The
children exit naturally within that window, so no force-kill should occur
and the measured wall-clock time must be >= 10s.
"""
_DRAIN_SECONDS = 10.0
_SHUTDOWN_TIMEOUT = 10.0
args = _make_args(shutdown_timeout=_SHUTDOWN_TIMEOUT)
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
async with _run_supervisor(
args, monkeypatch, launch_fn=launch_mock_vllm_with_drain
) as (supervisor, _task):
for port in vllm_server_ports:
await _poll_until_api_server_running(port)
for port in vllm_server_ports:
await _set_healthy(port)
assert await _await_supervisor_health(200)
assert supervisor.is_ready
start_t = time.perf_counter()
os.kill(os.getpid(), signal.SIGTERM)
print(f"DRAINING FOR {_DRAIN_SECONDS}")
await asyncio.wait_for(_task, timeout=_DRAIN_SECONDS + CHILD_EXIT_GRACE_S + 5.0)
elapsed = time.perf_counter() - start_t
assert elapsed >= _DRAIN_SECONDS, (
f"Supervisor exited after only {elapsed:.1f}s; "
f"expected >= {_DRAIN_SECONDS}s for request draining"
)
for p in supervisor._processes:
assert not p.is_alive()
print(f"Supervisor waited {elapsed:.1f}s for children to drain — expected!")