[Frontend] Move cli_args.py and dp_supervisor.py out openai folder (#53659)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,404 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from ...utils import VLLM_PATH
|
||||
|
||||
LORA_MODULE = {
|
||||
"name": "module2",
|
||||
"path": "/path/to/module2",
|
||||
"base_model_name": "llama",
|
||||
}
|
||||
CHATML_JINJA_PATH = VLLM_PATH / "examples/template_chatml.jinja"
|
||||
assert CHATML_JINJA_PATH.exists()
|
||||
|
||||
|
||||
def _build_vllm_parsers():
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
subparsers = vllm_parser.add_subparsers()
|
||||
serve_parser = subparsers.add_parser("serve")
|
||||
make_arg_parser(serve_parser)
|
||||
return {"vllm": vllm_parser, "vllm serve": serve_parser}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vllm_parser():
|
||||
return _build_vllm_parsers()["vllm"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def serve_parser():
|
||||
return _build_vllm_parsers()["vllm serve"]
|
||||
|
||||
|
||||
### Test config parsing
|
||||
def test_config_arg_parsing(serve_parser, cli_config_file):
|
||||
args = serve_parser.parse_args([])
|
||||
assert args.port == 8000
|
||||
args = serve_parser.parse_args(["--config", cli_config_file])
|
||||
assert args.port == 12312
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--config",
|
||||
cli_config_file,
|
||||
"--port",
|
||||
"9000",
|
||||
]
|
||||
)
|
||||
assert args.port == 9000
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--port",
|
||||
"9000",
|
||||
"--config",
|
||||
cli_config_file,
|
||||
]
|
||||
)
|
||||
assert args.port == 9000
|
||||
|
||||
|
||||
### Tests for LoRA module parsing
|
||||
def test_valid_key_value_format(serve_parser):
|
||||
# Test old format: name=path
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"module1=/path/to/module1",
|
||||
]
|
||||
)
|
||||
expected = [LoRAModulePath(name="module1", path="/path/to/module1")]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
def test_valid_json_format(serve_parser):
|
||||
# Test valid JSON format input
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
json.dumps(LORA_MODULE),
|
||||
]
|
||||
)
|
||||
expected = [
|
||||
LoRAModulePath(name="module2", path="/path/to/module2", base_model_name="llama")
|
||||
]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
def test_invalid_json_format(serve_parser):
|
||||
# Test invalid JSON format input, missing closing brace
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
["--lora-modules", '{"name": "module3", "path": "/path/to/module3"']
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_type_error(serve_parser):
|
||||
# Test type error when values are not JSON or key=value
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"invalid_format", # This is not JSON or key=value format
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_json_field(serve_parser):
|
||||
# Test valid JSON format but missing required fields
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
'{"name": "module4"}', # Missing required 'path' field
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_empty_values(serve_parser):
|
||||
# Test when no LoRA modules are provided
|
||||
args = serve_parser.parse_args(["--lora-modules", ""])
|
||||
assert args.lora_modules == []
|
||||
|
||||
|
||||
def test_multiple_valid_inputs(serve_parser):
|
||||
# Test multiple valid inputs (both old and JSON format)
|
||||
args = serve_parser.parse_args(
|
||||
[
|
||||
"--lora-modules",
|
||||
"module1=/path/to/module1",
|
||||
json.dumps(LORA_MODULE),
|
||||
]
|
||||
)
|
||||
expected = [
|
||||
LoRAModulePath(name="module1", path="/path/to/module1"),
|
||||
LoRAModulePath(
|
||||
name="module2", path="/path/to/module2", base_model_name="llama"
|
||||
),
|
||||
]
|
||||
assert args.lora_modules == expected
|
||||
|
||||
|
||||
### Tests for serve argument validation that run prior to loading
|
||||
def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
|
||||
"""Ensure validation fails if tool choice is enabled with no call parser"""
|
||||
# If we enable-auto-tool-choice, explode with no tool-call-parser
|
||||
args = serve_parser.parse_args(args=["--enable-auto-tool-choice"])
|
||||
with pytest.raises(TypeError):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser):
|
||||
"""Ensure validation passes with tool choice enabled with a call parser"""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"mistral",
|
||||
]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser):
|
||||
"""Ensure validation fails if reasoning is enabled with auto tool choice"""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-auto-tool-choice",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
]
|
||||
)
|
||||
with pytest.raises(TypeError):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_passes_with_reasoning_parser(serve_parser):
|
||||
"""Ensure validation passes if reasoning is enabled
|
||||
with a reasoning parser"""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_chat_template_validation_for_happy_paths(serve_parser):
|
||||
"""Ensure validation passes if the chat template exists"""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()]
|
||||
)
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_chat_template_validation_for_sad_paths(serve_parser):
|
||||
"""Ensure validation fails if the chat template doesn't exist"""
|
||||
args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"])
|
||||
with pytest.raises(ValueError):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_per_request_metrics_requires_log_stats(serve_parser):
|
||||
args = serve_parser.parse_args(
|
||||
args=["--enable-per-request-metrics", "--disable-log-stats"]
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def _build_launch_render_parser():
|
||||
"""Mirror `vllm launch render`.
|
||||
|
||||
`vllm/entrypoints/cli/main.py` parses subcommands with ``dest="subparser"``,
|
||||
and `LaunchSubcommandBase.add_cli_args` builds the component parser with
|
||||
`make_arg_parser`, so a launch component carries serve args under
|
||||
``args.subparser == "launch"``.
|
||||
"""
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
subparsers = vllm_parser.add_subparsers(required=False, dest="subparser")
|
||||
launch_parser = subparsers.add_parser("launch")
|
||||
launch_subparsers = launch_parser.add_subparsers(
|
||||
required=True, dest="launch_component"
|
||||
)
|
||||
render_parser = launch_subparsers.add_parser("render")
|
||||
make_arg_parser(render_parser)
|
||||
return vllm_parser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def launch_render_parser():
|
||||
return _build_launch_render_parser()
|
||||
|
||||
|
||||
def test_launch_render_validates_serve_args(launch_render_parser):
|
||||
"""`vllm launch render` reuses the serve parser, so it gets the serve checks"""
|
||||
args = launch_render_parser.parse_args(
|
||||
args=["launch", "render", "--enable-auto-tool-choice"]
|
||||
)
|
||||
assert args.subparser == "launch"
|
||||
with pytest.raises(TypeError):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
def test_subcommand_without_serve_args_skips_validation():
|
||||
"""A subcommand that does not use the serve parser is still skipped"""
|
||||
vllm_parser = FlexibleArgumentParser()
|
||||
subparsers = vllm_parser.add_subparsers(required=False, dest="subparser")
|
||||
subparsers.add_parser("chat")
|
||||
args = vllm_parser.parse_args(args=["chat"])
|
||||
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cli_args, expected",
|
||||
[
|
||||
([], 0),
|
||||
(["--sse-keep-alive-interval", "0"], 0),
|
||||
(["--sse-keep-alive-interval", "15"], 15),
|
||||
],
|
||||
)
|
||||
def test_sse_keep_alive_interval_valid(serve_parser, cli_args, expected):
|
||||
"""The default is 0 and zero or positive integers parse exactly."""
|
||||
args = serve_parser.parse_args(args=cli_args)
|
||||
assert args.sse_keep_alive_interval == expected
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["-1", "-5"])
|
||||
def test_sse_keep_alive_interval_negative(serve_parser, value):
|
||||
"""Negative intervals are rejected by validation."""
|
||||
args = serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
|
||||
with pytest.raises(ValueError):
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["nan", "inf", "-inf", "0.5", "abc"])
|
||||
def test_sse_keep_alive_interval_non_integer(serve_parser, value):
|
||||
"""Non-integer values fail argparse's int parsing with SystemExit."""
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(args=["--sse-keep-alive-interval", value])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cli_args, expected_middleware",
|
||||
[
|
||||
(
|
||||
["--middleware", "middleware1", "--middleware", "middleware2"],
|
||||
["middleware1", "middleware2"],
|
||||
),
|
||||
([], []),
|
||||
],
|
||||
)
|
||||
def test_middleware(serve_parser, cli_args, expected_middleware):
|
||||
"""Ensure multiple middleware args are parsed properly"""
|
||||
args = serve_parser.parse_args(args=cli_args)
|
||||
assert args.middleware == expected_middleware
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_parsing(serve_parser):
|
||||
"""Ensure default_chat_template_kwargs JSON is parsed correctly"""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--default-chat-template-kwargs", '{"enable_thinking": false}']
|
||||
)
|
||||
assert args.default_chat_template_kwargs == {"enable_thinking": False}
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_complex(serve_parser):
|
||||
"""Ensure complex default_chat_template_kwargs JSON is parsed correctly"""
|
||||
kwargs_json = '{"enable_thinking": false, "custom_param": "value", "num": 42}'
|
||||
args = serve_parser.parse_args(args=["--default-chat-template-kwargs", kwargs_json])
|
||||
assert args.default_chat_template_kwargs == {
|
||||
"enable_thinking": False,
|
||||
"custom_param": "value",
|
||||
"num": 42,
|
||||
}
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_default_none(serve_parser):
|
||||
"""Ensure default_chat_template_kwargs defaults to None"""
|
||||
args = serve_parser.parse_args(args=[])
|
||||
assert args.default_chat_template_kwargs is None
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
||||
"""Ensure invalid JSON raises an error"""
|
||||
with pytest.raises(SystemExit):
|
||||
serve_parser.parse_args(
|
||||
args=["--default-chat-template-kwargs", "not valid json"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args, raises",
|
||||
[
|
||||
(["user/model"], None),
|
||||
(["user/model", "--served-model-name", "model"], None),
|
||||
(["--served-model-name", "model", "user/model"], ValueError),
|
||||
(["--served-model-name", "model", "--config", "config.yaml"], None),
|
||||
(["--served-model-name", "model", "--config", "config.yaml"], ValueError),
|
||||
],
|
||||
ids=[
|
||||
"model_tag_only",
|
||||
"model_tag_with_served_model_name",
|
||||
"served_model_name_before_model_tag",
|
||||
"served_model_name_with_model_in_config",
|
||||
"served_model_name_with_no_model_in_config",
|
||||
],
|
||||
)
|
||||
def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
|
||||
"""Ensure that users don't misuse --served-model-name and end up with the default
|
||||
model tag instead of the one they intended to serve."""
|
||||
# Call the serve subparser
|
||||
args.insert(0, "serve")
|
||||
# Create a dummy config file if the test case includes it
|
||||
if "config.yaml" in args:
|
||||
# Create a dummy config file if the test case includes it
|
||||
config_path = tmp_path / "config.yaml"
|
||||
config_path.write_text("model: user/model" if raises is None else "port: 8000")
|
||||
args[args.index("config.yaml")] = config_path.as_posix()
|
||||
# Do the parsing and check for expected exceptions or values
|
||||
if raises is None:
|
||||
parsed_args = vllm_parser.parse_args(args=args)
|
||||
expected = "user/model"
|
||||
assert parsed_args.model_tag == expected or parsed_args.model == expected
|
||||
else:
|
||||
with pytest.raises(raises):
|
||||
vllm_parser.parse_args(args=args)
|
||||
|
||||
|
||||
### Tests for LoRA target modules parsing
|
||||
def test_lora_target_modules_single(serve_parser):
|
||||
"""Test parsing single lora-target-modules argument"""
|
||||
args = serve_parser.parse_args(
|
||||
args=["--enable-lora", "--lora-target-modules", "o_proj"]
|
||||
)
|
||||
assert args.lora_target_modules == ["o_proj"]
|
||||
|
||||
|
||||
def test_lora_target_modules_multiple(serve_parser):
|
||||
"""Test parsing multiple lora-target-modules arguments"""
|
||||
args = serve_parser.parse_args(
|
||||
args=[
|
||||
"--enable-lora",
|
||||
"--lora-target-modules",
|
||||
"o_proj",
|
||||
"qkv_proj",
|
||||
"down_proj",
|
||||
]
|
||||
)
|
||||
assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
|
||||
|
||||
|
||||
def test_lora_target_modules_default_none(serve_parser):
|
||||
"""Test that lora-target-modules defaults to None"""
|
||||
args = serve_parser.parse_args(args=[])
|
||||
assert args.lora_target_modules is None
|
||||
@@ -1,927 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Tests for DPSupervisor: unit tests and lifecycle integration tests.
|
||||
|
||||
Lifecycle integration tests replace child vLLM servers with lightweight
|
||||
aiohttp "fake" servers controlled by the test, so the suite runs without GPUs.
|
||||
_start_children is monkeypatched to install FakeProcess objects (with
|
||||
controllable liveness/timing) alongside those fake HTTP servers.
|
||||
|
||||
Port allocation (kept far from default vLLM ports to avoid conflicts):
|
||||
Supervisor : 19256
|
||||
Children : 18000, 18001
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Response
|
||||
|
||||
import vllm.entrypoints.openai.dp_supervisor as dp_sup
|
||||
from vllm.entrypoints.openai.dp_supervisor import (
|
||||
CHILD_EXIT_GRACE_S,
|
||||
DPSupervisor,
|
||||
_build_vllm_dp_server_args,
|
||||
infer_multi_port_external_lb_start_rank,
|
||||
validate_multi_port_external_lb_args,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SUPERVISOR_PORT = 19256
|
||||
_CHILD_PORT_BASE = 18000
|
||||
_N_CHILDREN = 2
|
||||
_PROBE_INTERVAL = 1.0
|
||||
_POLL_INTERVAL = 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Args factories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_unit_args(**overrides) -> argparse.Namespace:
|
||||
"""Minimal args for unit tests (no real network activity)."""
|
||||
base = {
|
||||
"host": None,
|
||||
"port": 8000,
|
||||
"data_parallel_multi_port_external_lb": True,
|
||||
"data_parallel_supervisor_port": 9256,
|
||||
"dp_supervisor_probe_interval_s": 5.0,
|
||||
"dp_supervisor_probe_timeout_s": 5.0,
|
||||
"dp_supervisor_probe_failure_threshold": 3,
|
||||
"data_parallel_size": 8,
|
||||
"data_parallel_size_local": 4,
|
||||
"data_parallel_start_rank": None,
|
||||
"data_parallel_rank": None,
|
||||
"data_parallel_external_lb": False,
|
||||
"data_parallel_hybrid_lb": False,
|
||||
"api_server_count": None,
|
||||
"headless": False,
|
||||
"grpc": False,
|
||||
"uds": None,
|
||||
"ssl_keyfile": None,
|
||||
"ssl_certfile": None,
|
||||
"ssl_ca_certs": None,
|
||||
"ssl_cert_reqs": 0,
|
||||
"ssl_ciphers": None,
|
||||
"node_rank": 1,
|
||||
"tensor_parallel_size": 1,
|
||||
"pipeline_parallel_size": 1,
|
||||
"uvicorn_log_level": "info",
|
||||
"shutdown_timeout": 5.0,
|
||||
"disable_uvicorn_access_log": False,
|
||||
"disable_access_log_for_endpoints": None,
|
||||
"log_config_file": None,
|
||||
}
|
||||
base.update(overrides)
|
||||
return argparse.Namespace(**base)
|
||||
|
||||
|
||||
def _make_args(**overrides) -> argparse.Namespace:
|
||||
"""Args for lifecycle integration tests (real loopback servers)."""
|
||||
base: dict = dict(
|
||||
host="127.0.0.1",
|
||||
port=_CHILD_PORT_BASE,
|
||||
data_parallel_multi_port_external_lb=True,
|
||||
data_parallel_supervisor_port=_SUPERVISOR_PORT,
|
||||
dp_supervisor_probe_interval_s=_PROBE_INTERVAL,
|
||||
dp_supervisor_probe_timeout_s=1.0,
|
||||
dp_supervisor_probe_failure_threshold=3,
|
||||
data_parallel_size=_N_CHILDREN,
|
||||
data_parallel_size_local=_N_CHILDREN,
|
||||
data_parallel_start_rank=0,
|
||||
data_parallel_rank=None,
|
||||
data_parallel_external_lb=False,
|
||||
data_parallel_hybrid_lb=False,
|
||||
api_server_count=None,
|
||||
headless=False,
|
||||
grpc=False,
|
||||
uds=None,
|
||||
ssl_keyfile=None,
|
||||
ssl_certfile=None,
|
||||
ssl_ca_certs=None,
|
||||
ssl_cert_reqs=0,
|
||||
ssl_ciphers=None,
|
||||
node_rank=0,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
uvicorn_log_level="warning",
|
||||
shutdown_timeout=0.0,
|
||||
disable_uvicorn_access_log=False,
|
||||
disable_access_log_for_endpoints=None,
|
||||
log_config_file=None,
|
||||
)
|
||||
base.update(overrides)
|
||||
return argparse.Namespace(**base)
|
||||
|
||||
|
||||
def _generate_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]:
|
||||
"""Generate a self-signed certificate for HTTPS lifecycle tests."""
|
||||
cert_file = cert_dir / "cert.pem"
|
||||
key_file = cert_dir / "key.pem"
|
||||
subprocess.run(
|
||||
[
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-keyout",
|
||||
str(key_file),
|
||||
"-out",
|
||||
str(cert_file),
|
||||
"-days",
|
||||
"1",
|
||||
"-nodes",
|
||||
"-subj",
|
||||
"/CN=localhost",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
return cert_file, key_file
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_infer_multi_port_external_lb_start_rank_uses_node_rank():
|
||||
args = _make_unit_args()
|
||||
assert infer_multi_port_external_lb_start_rank(args) == 4
|
||||
|
||||
|
||||
def test_build_multi_port_external_lb_child_args_sets_external_rank_server():
|
||||
args = _make_unit_args(data_parallel_start_rank=8, api_server_count=None)
|
||||
child_args = _build_vllm_dp_server_args(args, local_rank=2)
|
||||
|
||||
assert child_args.port == 8002
|
||||
assert child_args.data_parallel_rank == 10
|
||||
assert child_args.data_parallel_size_local == 1
|
||||
assert child_args.data_parallel_external_lb is True
|
||||
assert child_args.data_parallel_hybrid_lb is False
|
||||
assert child_args.data_parallel_multi_port_external_lb is False
|
||||
assert child_args.api_server_count == 1
|
||||
|
||||
|
||||
def test_run_vllm_dp_server_uses_python_server_by_default(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(dp_sup.os, "setpgrp", lambda: None)
|
||||
monkeypatch.setattr(dp_sup, "set_process_title", lambda *_args: None)
|
||||
monkeypatch.setattr(dp_sup, "decorate_logs", lambda *_args: None)
|
||||
monkeypatch.setattr(dp_sup.envs, "VLLM_RUST_FRONTEND_PATH", None, raising=False)
|
||||
monkeypatch.setattr(
|
||||
dp_sup, "_run_python_vllm_dp_server", lambda _args: calls.append("python")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dp_sup, "_run_rust_vllm_dp_server", lambda _args: calls.append("rust")
|
||||
)
|
||||
|
||||
dp_sup._run_vllm_dp_server(_make_unit_args(data_parallel_rank=4))
|
||||
|
||||
assert calls == ["python"]
|
||||
|
||||
|
||||
def test_run_vllm_dp_server_uses_rust_frontend_when_enabled(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(dp_sup.os, "setpgrp", lambda: None)
|
||||
monkeypatch.setattr(dp_sup, "set_process_title", lambda *_args: None)
|
||||
monkeypatch.setattr(dp_sup, "decorate_logs", lambda *_args: None)
|
||||
monkeypatch.setattr(dp_sup.envs, "VLLM_USE_RUST_FRONTEND", True, raising=False)
|
||||
monkeypatch.setattr(
|
||||
dp_sup.envs,
|
||||
"VLLM_RUST_FRONTEND_PATH",
|
||||
"/tmp/vllm-rs",
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dp_sup, "_run_python_vllm_dp_server", lambda _args: calls.append("python")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dp_sup, "_run_rust_vllm_dp_server", lambda _args: calls.append("rust")
|
||||
)
|
||||
|
||||
dp_sup._run_vllm_dp_server(_make_unit_args(data_parallel_rank=4))
|
||||
|
||||
assert calls == ["rust"]
|
||||
|
||||
|
||||
def test_validate_multi_port_external_lb_args_allows_ssl():
|
||||
args = _make_unit_args(
|
||||
ssl_keyfile="/tmp/server.key",
|
||||
ssl_certfile="/tmp/server.crt",
|
||||
ssl_ca_certs="/tmp/ca.crt",
|
||||
)
|
||||
validate_multi_port_external_lb_args(args)
|
||||
|
||||
|
||||
def test_aggregates_health():
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
supervisor._is_ready = True
|
||||
assert supervisor.is_ready is True
|
||||
|
||||
|
||||
def test_handles_shutdown_event():
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
supervisor._is_ready = True
|
||||
supervisor._shutdown_event.set()
|
||||
assert supervisor.is_ready is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_children_uses_engine_process_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
supervisor = DPSupervisor(_make_unit_args(shutdown_timeout=0.0))
|
||||
supervisor._processes = [
|
||||
SimpleNamespace(name="APIServer_DPRank_4", pid=None, is_alive=lambda: False)
|
||||
]
|
||||
calls = []
|
||||
timeout_calls = []
|
||||
|
||||
def get_process_timeout(request_timeout, manager_timeout):
|
||||
timeout_calls.append((request_timeout, manager_timeout))
|
||||
return 15.0
|
||||
|
||||
monkeypatch.setattr(
|
||||
dp_sup,
|
||||
"get_engine_process_shutdown_timeout",
|
||||
get_process_timeout,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dp_sup,
|
||||
"_join_processes_with_timeout",
|
||||
lambda processes, timeout: calls.append((processes, timeout)),
|
||||
)
|
||||
|
||||
await supervisor._shutdown_children()
|
||||
|
||||
assert timeout_calls == [(0.0, 0.0)]
|
||||
assert calls == [
|
||||
(supervisor._processes, 15.0 + CHILD_EXIT_GRACE_S),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_child_exit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
supervisor._processes = [
|
||||
SimpleNamespace(
|
||||
name="APIServer_DPRank_4", exitcode=None, is_alive=lambda: True
|
||||
),
|
||||
SimpleNamespace(name="APIServer_DPRank_5", exitcode=17, is_alive=lambda: False),
|
||||
]
|
||||
|
||||
async def fake_probe(*_args, **_kwargs) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
|
||||
|
||||
await supervisor._monitor_children()
|
||||
assert supervisor._is_ready is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handles_probe_failure(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
supervisor = DPSupervisor(_make_unit_args(dp_supervisor_probe_interval_s=0.0))
|
||||
supervisor.child_ports = [8000]
|
||||
probe_results = iter([True, False])
|
||||
|
||||
async def fake_probe(*_args, **_kwargs) -> bool:
|
||||
return next(probe_results)
|
||||
|
||||
monkeypatch.setattr(dp_sup, "_probe_endpoint", fake_probe)
|
||||
|
||||
await supervisor._monitor_children()
|
||||
assert supervisor._is_ready is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_if_supervisor_server_error_on_startup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
class FakeLoop:
|
||||
def add_signal_handler(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def remove_signal_handler(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
class FakeServer:
|
||||
def __init__(self, _config):
|
||||
self.started = False
|
||||
self.should_exit = False
|
||||
|
||||
async def serve(self):
|
||||
raise ValueError("supervisor boom")
|
||||
|
||||
async def fake_shutdown_children(self):
|
||||
return None
|
||||
|
||||
def fake_start_children(self):
|
||||
return None
|
||||
|
||||
async def fake_monitor_children(self):
|
||||
# Mark ready so the supervisor server is started, then block until
|
||||
# shutdown (triggered when the failing server task exits).
|
||||
self._is_ready = True
|
||||
await self._shutdown_event.wait()
|
||||
|
||||
monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop())
|
||||
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
|
||||
monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children)
|
||||
monkeypatch.setattr(DPSupervisor, "_start_children", fake_start_children)
|
||||
monkeypatch.setattr(DPSupervisor, "_monitor_children", fake_monitor_children)
|
||||
|
||||
supervisor = DPSupervisor(_make_unit_args())
|
||||
|
||||
with pytest.raises(ValueError, match="supervisor boom"):
|
||||
await supervisor.run()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Access-log filtering unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_server_applies_access_log_filter(monkeypatch):
|
||||
"""get_uvicorn_log_config result is forwarded to uvicorn.Config."""
|
||||
sentinel_config = {"version": 1, "loggers": {}}
|
||||
captured_config: list[uvicorn.Config] = []
|
||||
|
||||
monkeypatch.setattr(dp_sup, "get_uvicorn_log_config", lambda _args: sentinel_config)
|
||||
|
||||
class FakeServer:
|
||||
def __init__(self, config):
|
||||
captured_config.append(config)
|
||||
self.started = True
|
||||
self.should_exit = False
|
||||
|
||||
async def serve(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
|
||||
|
||||
args = _make_unit_args(
|
||||
host="127.0.0.1",
|
||||
disable_access_log_for_endpoints="/health,/readyz",
|
||||
)
|
||||
supervisor = DPSupervisor(args)
|
||||
await supervisor._start_server()
|
||||
|
||||
assert len(captured_config) == 1
|
||||
config = captured_config[0]
|
||||
assert config.log_config is sentinel_config
|
||||
assert config.access_log is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_server_disables_access_log(monkeypatch):
|
||||
"""disable_uvicorn_access_log=True sets access_log=False."""
|
||||
captured_config: list[uvicorn.Config] = []
|
||||
|
||||
monkeypatch.setattr(dp_sup, "get_uvicorn_log_config", lambda _args: None)
|
||||
|
||||
class FakeServer:
|
||||
def __init__(self, config):
|
||||
captured_config.append(config)
|
||||
self.started = True
|
||||
self.should_exit = False
|
||||
|
||||
async def serve(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
|
||||
|
||||
args = _make_unit_args(
|
||||
host="127.0.0.1",
|
||||
disable_uvicorn_access_log=True,
|
||||
)
|
||||
supervisor = DPSupervisor(args)
|
||||
await supervisor._start_server()
|
||||
|
||||
assert len(captured_config) == 1
|
||||
assert captured_config[0].access_log is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_server_no_log_config_when_no_filter(monkeypatch):
|
||||
"""Without filtering options, default uvicorn log_config is used."""
|
||||
captured_config: list[uvicorn.Config] = []
|
||||
|
||||
monkeypatch.setattr(dp_sup, "get_uvicorn_log_config", lambda _args: None)
|
||||
|
||||
class FakeServer:
|
||||
def __init__(self, config):
|
||||
captured_config.append(config)
|
||||
self.started = True
|
||||
self.should_exit = False
|
||||
|
||||
async def serve(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
|
||||
|
||||
args = _make_unit_args(host="127.0.0.1")
|
||||
supervisor = DPSupervisor(args)
|
||||
await supervisor._start_server()
|
||||
|
||||
assert len(captured_config) == 1
|
||||
config = captured_config[0]
|
||||
assert config.log_config == uvicorn.Config(app=None).log_config
|
||||
assert config.access_log is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle integration tests – MockVLLMServer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MockVLLMServer:
|
||||
"""
|
||||
Minimal FastAPI server that mimics one vLLM replica.
|
||||
GET /health returns 200 when healthy, 503 otherwise.
|
||||
Health state is toggled by the test via set_healthy().
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
port: int,
|
||||
drain_seconds: float = 0.0,
|
||||
ssl_keyfile: str | None = None,
|
||||
ssl_certfile: str | None = None,
|
||||
) -> None:
|
||||
self.port = port
|
||||
self._healthy = False
|
||||
self._drain_seconds = drain_seconds
|
||||
self._ssl_keyfile = ssl_keyfile
|
||||
self._ssl_certfile = ssl_certfile
|
||||
self._server: uvicorn.Server | None = None
|
||||
self._serve_task: asyncio.Task | None = None
|
||||
|
||||
async def start(self) -> None:
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> Response:
|
||||
print(f"MockServer {self.port}: /health: {self._healthy}")
|
||||
return Response(status_code=200 if self._healthy else 503)
|
||||
|
||||
@app.get("/set_healthy")
|
||||
async def set_healthy() -> Response:
|
||||
print(f"MockServer {self.port}: /set_healthy")
|
||||
self._healthy = True
|
||||
return Response(status_code=200)
|
||||
|
||||
@app.get("/set_unhealthy")
|
||||
async def set_unhealthy() -> Response:
|
||||
print(f"MockServer {self.port}: /set_unhealthy")
|
||||
self._healthy = False
|
||||
return Response(status_code=200)
|
||||
|
||||
@app.get("/kill")
|
||||
async def kill() -> Response:
|
||||
print(f"MockServer {self.port}: /kill")
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host="127.0.0.1",
|
||||
port=self.port,
|
||||
log_level="warning",
|
||||
lifespan="off",
|
||||
ssl_keyfile=self._ssl_keyfile,
|
||||
ssl_certfile=self._ssl_certfile,
|
||||
)
|
||||
self._server = uvicorn.Server(config)
|
||||
|
||||
# Configure request draining if needed.
|
||||
# Uvicorn's capture_signals() installs signal.signal(SIGTERM, self.handle_exit),
|
||||
# which sets should_exit=True immediately. Override handle_exit on the instance
|
||||
# so capture_signals() picks up our version that drains first.
|
||||
if self._drain_seconds > 0:
|
||||
self._shutdown_event = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
async def _drain_and_stop() -> None:
|
||||
await self._shutdown_event.wait()
|
||||
print(f"MockServer {self.port}: draining for {self._drain_seconds}s.")
|
||||
await asyncio.sleep(self._drain_seconds)
|
||||
print("Setting should_exit")
|
||||
if self._server is not None:
|
||||
self._server.should_exit = True
|
||||
|
||||
self._drain_task = asyncio.create_task(_drain_and_stop())
|
||||
|
||||
def _custom_handle_exit(sig: int, frame: object) -> None:
|
||||
print("Got SIGTERM, setting shutdown.")
|
||||
if not self._shutdown_event.is_set():
|
||||
loop.call_soon_threadsafe(self._shutdown_event.set)
|
||||
|
||||
self._server.handle_exit = _custom_handle_exit
|
||||
|
||||
self._serve_task = asyncio.create_task(self._server.serve())
|
||||
while not self._server.started:
|
||||
await asyncio.sleep(0.01)
|
||||
print(f"Mock DP Server on port {self.port} started")
|
||||
|
||||
await self._serve_task
|
||||
|
||||
|
||||
def launch_mock_vllm(child_args: argparse.Namespace):
|
||||
logger.info("Launching mock vLLM on port %s", child_args.port)
|
||||
mock_vllm = MockVLLMServer(
|
||||
port=child_args.port,
|
||||
ssl_keyfile=child_args.ssl_keyfile,
|
||||
ssl_certfile=child_args.ssl_certfile,
|
||||
)
|
||||
asyncio.run(mock_vllm.start())
|
||||
|
||||
|
||||
def launch_mock_vllm_with_drain(
|
||||
child_args: argparse.Namespace,
|
||||
):
|
||||
logger.info("Launching mock vLLM with 15s drain on port %s", child_args.port)
|
||||
mock_vllm = MockVLLMServer(
|
||||
port=child_args.port,
|
||||
drain_seconds=10.0,
|
||||
ssl_keyfile=child_args.ssl_keyfile,
|
||||
ssl_certfile=child_args.ssl_certfile,
|
||||
)
|
||||
asyncio.run(mock_vllm.start())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle test helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -> bool:
|
||||
"""
|
||||
GET /health on the supervisor once and check for expected_status.
|
||||
|
||||
Pass expected_status=-1 to assert the supervisor is not listening yet
|
||||
(a connection error is expected). The supervisor only starts its HTTP
|
||||
server once every child is ready, so /health is refused until then.
|
||||
"""
|
||||
scheme = "https" if use_ssl else "http"
|
||||
url = f"{scheme}://127.0.0.1:{_SUPERVISOR_PORT}/health"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
async with session.get(url, ssl=False if use_ssl else None) as resp:
|
||||
if resp.status != expected_status:
|
||||
print(f"expected: {expected_status=}, got: {resp.status=}")
|
||||
return False
|
||||
return True
|
||||
except aiohttp.ClientError:
|
||||
if expected_status != -1:
|
||||
print(f"expected: {expected_status=}, got: aiohttp.ClientError")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def _await_supervisor_health(
|
||||
expected_status: int, use_ssl: bool = False, retries: int = 20
|
||||
) -> bool:
|
||||
"""Retry _poll_supervisor_health, tolerating supervisor server startup."""
|
||||
for _ in range(retries):
|
||||
if await _poll_supervisor_health(expected_status, use_ssl=use_ssl):
|
||||
return True
|
||||
await asyncio.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
async def _poll_until_api_server_running(
|
||||
port: int, timeout_s: float = 30.0, use_ssl: bool = False
|
||||
) -> None:
|
||||
"""Return once the child accepts a request; it reports 503 until healthy."""
|
||||
scheme = "https" if use_ssl else "http"
|
||||
url = f"{scheme}://127.0.0.1:{port}/health"
|
||||
deadline = time.monotonic() + timeout_s
|
||||
last_exc: Exception | None = None
|
||||
async with aiohttp.ClientSession() as session:
|
||||
while (remaining_s := deadline - time.monotonic()) > 0:
|
||||
try:
|
||||
request_timeout = aiohttp.ClientTimeout(total=min(2.0, remaining_s))
|
||||
async with session.get(
|
||||
url,
|
||||
ssl=False if use_ssl else None,
|
||||
timeout=request_timeout,
|
||||
):
|
||||
return
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError) as exc:
|
||||
last_exc = exc
|
||||
print("Test detected not started yet, sleeping for 1s")
|
||||
remaining_s = deadline - time.monotonic()
|
||||
if remaining_s > 0:
|
||||
await asyncio.sleep(min(1.0, remaining_s))
|
||||
raise TimeoutError(
|
||||
f"API server on port {port} did not start within {timeout_s}s"
|
||||
) from last_exc
|
||||
|
||||
|
||||
async def _set_healthy(port: int, use_ssl: bool = False) -> None:
|
||||
scheme = "https" if use_ssl else "http"
|
||||
url = f"{scheme}://127.0.0.1:{port}/set_healthy"
|
||||
async with (
|
||||
aiohttp.ClientSession() as session,
|
||||
session.get(url, ssl=False if use_ssl else None) as resp,
|
||||
):
|
||||
assert resp.status == 200
|
||||
|
||||
|
||||
async def _set_unhealthy(port: int, use_ssl: bool = False) -> None:
|
||||
scheme = "https" if use_ssl else "http"
|
||||
url = f"{scheme}://127.0.0.1:{port}/set_unhealthy"
|
||||
async with (
|
||||
aiohttp.ClientSession() as session,
|
||||
session.get(url, ssl=False if use_ssl else None) as resp,
|
||||
):
|
||||
assert resp.status == 200
|
||||
|
||||
|
||||
async def _kill_server(port: int, use_ssl: bool = False) -> None:
|
||||
scheme = "https" if use_ssl else "http"
|
||||
url = f"{scheme}://127.0.0.1:{port}/kill"
|
||||
try:
|
||||
async with (
|
||||
aiohttp.ClientSession() as session,
|
||||
session.get(url, ssl=False if use_ssl else None) as resp,
|
||||
):
|
||||
assert resp.status != 200
|
||||
except aiohttp.ClientConnectionError:
|
||||
return
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _run_supervisor(
|
||||
args: argparse.Namespace,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
launch_fn=None,
|
||||
):
|
||||
if launch_fn is None:
|
||||
launch_fn = launch_mock_vllm
|
||||
monkeypatch.setattr(dp_sup, "_run_vllm_dp_server", launch_fn)
|
||||
supervisor = DPSupervisor(args)
|
||||
task = asyncio.create_task(supervisor.run())
|
||||
await asyncio.sleep(1.0)
|
||||
try:
|
||||
yield supervisor, task
|
||||
finally:
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle integration tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_lifecycle(monkeypatch):
|
||||
"""
|
||||
A) Supervisor is not listening while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) SIGTERM and shutdown
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _set_healthy(vllm_server_ports[0])
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
print("supervisor not listening --- expected!")
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
assert await _await_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print("everything was cleaned up!")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_lifecycle_with_ssl(monkeypatch):
|
||||
with tempfile.TemporaryDirectory() as cert_dir:
|
||||
cert_file, key_file = _generate_self_signed_cert(Path(cert_dir))
|
||||
args = _make_args(
|
||||
ssl_keyfile=str(key_file),
|
||||
ssl_certfile=str(cert_file),
|
||||
)
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(-1, use_ssl=True)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
assert await _poll_supervisor_health(-1, use_ssl=True)
|
||||
assert not supervisor.is_ready
|
||||
await _poll_until_api_server_running(port, use_ssl=True)
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port, use_ssl=True)
|
||||
|
||||
assert await _await_supervisor_health(200, use_ssl=True)
|
||||
assert supervisor.is_ready
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_startup(monkeypatch):
|
||||
"""
|
||||
A) One of the vLLM servers crashes during startup.
|
||||
B) DPSupervisor detects this, and cleans up resources.
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _kill_server(port)
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_becomes_unhealthy(monkeypatch):
|
||||
"""
|
||||
A) Supervisor is not listening while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) Child process becomes unhealthy.
|
||||
D) Detected and shutdown.
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _set_healthy(vllm_server_ports[0])
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
print("supervisor not listening --- expected!")
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
assert await _await_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
await _set_unhealthy(port)
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print("everything was cleaned up!")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dp_server_fails(monkeypatch):
|
||||
"""
|
||||
A) Supervisor is not listening while children are unhealthy.
|
||||
B) /health returns 200 once every child reports healthy.
|
||||
C) Child process fails.
|
||||
D) Detected and shutdown.
|
||||
"""
|
||||
args = _make_args()
|
||||
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
|
||||
for port in vllm_server_ports:
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
await _set_healthy(vllm_server_ports[0])
|
||||
await asyncio.sleep(1.0)
|
||||
assert await _poll_supervisor_health(-1)
|
||||
assert not supervisor.is_ready
|
||||
print("supervisor not listening --- expected!")
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
assert await _await_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
print("/health is 200 --- expected!")
|
||||
|
||||
dp_mock_server_process = supervisor._processes[0]
|
||||
os.kill(dp_mock_server_process.pid, signal.SIGKILL)
|
||||
await asyncio.sleep(1.0)
|
||||
assert not dp_mock_server_process.is_alive()
|
||||
|
||||
await asyncio.wait_for(_task, timeout=5.0)
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print("everything was cleaned up!")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight
|
||||
request drain). The supervisor is configured with shutdown_timeout=10,
|
||||
so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The
|
||||
children exit naturally within that window, so no force-kill should occur
|
||||
and the measured wall-clock time must be >= 10s.
|
||||
"""
|
||||
_DRAIN_SECONDS = 10.0
|
||||
_SHUTDOWN_TIMEOUT = 10.0
|
||||
|
||||
args = _make_args(shutdown_timeout=_SHUTDOWN_TIMEOUT)
|
||||
vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
|
||||
|
||||
async with _run_supervisor(
|
||||
args, monkeypatch, launch_fn=launch_mock_vllm_with_drain
|
||||
) as (supervisor, _task):
|
||||
for port in vllm_server_ports:
|
||||
await _poll_until_api_server_running(port)
|
||||
|
||||
for port in vllm_server_ports:
|
||||
await _set_healthy(port)
|
||||
assert await _await_supervisor_health(200)
|
||||
assert supervisor.is_ready
|
||||
|
||||
start_t = time.perf_counter()
|
||||
os.kill(os.getpid(), signal.SIGTERM)
|
||||
|
||||
print(f"DRAINING FOR {_DRAIN_SECONDS}")
|
||||
await asyncio.wait_for(_task, timeout=_DRAIN_SECONDS + CHILD_EXIT_GRACE_S + 5.0)
|
||||
elapsed = time.perf_counter() - start_t
|
||||
|
||||
assert elapsed >= _DRAIN_SECONDS, (
|
||||
f"Supervisor exited after only {elapsed:.1f}s; "
|
||||
f"expected >= {_DRAIN_SECONDS}s for request draining"
|
||||
)
|
||||
|
||||
for p in supervisor._processes:
|
||||
assert not p.is_alive()
|
||||
print(f"Supervisor waited {elapsed:.1f}s for children to drain — expected!")
|
||||
Reference in New Issue
Block a user