[Frontend] Move api_server.py out openai folder (#52131)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Spawn worker targets kept free of heavy imports.
|
||||
|
||||
``multiprocessing`` with the ``spawn`` start method re-imports the module that
|
||||
defines a process target in the child. Housing these stubs in a stdlib-only
|
||||
module keeps child startup fast and deterministic, instead of paying a multi-
|
||||
second ``import vllm`` before the child can run.
|
||||
"""
|
||||
|
||||
|
||||
def exit_before_report_worker(listen_address, sock, args, client_config=None):
|
||||
"""Exit immediately without touching ``actual_address_pipe``."""
|
||||
return
|
||||
@@ -0,0 +1,403 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import multiprocessing
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
|
||||
exit_before_report_worker,
|
||||
)
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
from vllm.v1.utils import (
|
||||
APIServerProcessManager,
|
||||
get_engine_client_zmq_addr,
|
||||
wait_for_completion_or_failure,
|
||||
)
|
||||
|
||||
# Global variables to control worker behavior
|
||||
WORKER_RUNTIME_SECONDS = 0.5
|
||||
|
||||
|
||||
# Mock implementation of run_api_server_worker
|
||||
def mock_run_api_server_worker(listen_address, sock, args, client_config=None):
|
||||
"""Mock run_api_server_worker that runs for a specific time."""
|
||||
print(f"Mock worker started with client_config: {client_config}")
|
||||
time.sleep(WORKER_RUNTIME_SECONDS)
|
||||
print("Mock worker completed successfully")
|
||||
|
||||
|
||||
# Module-level stub for the gather_actual_addresses test. Must be
|
||||
# importable by `multiprocessing.spawn` (no closures, no nesting).
|
||||
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
|
||||
"""Bind ROUTER/PULL with a kernel-assigned port, report the actual
|
||||
endpoints back via the pipe, then exit."""
|
||||
ctx = zmq.Context()
|
||||
try:
|
||||
in_sock = make_zmq_socket(
|
||||
ctx, client_config["input_address"], zmq.ROUTER, bind=True
|
||||
)
|
||||
out_sock = make_zmq_socket(
|
||||
ctx, client_config["output_address"], zmq.PULL, bind=True
|
||||
)
|
||||
try:
|
||||
pipe = client_config["actual_address_pipe"]
|
||||
try:
|
||||
pipe.send(
|
||||
{
|
||||
"input_address": in_sock.getsockopt(zmq.LAST_ENDPOINT).decode(),
|
||||
"output_address": out_sock.getsockopt(
|
||||
zmq.LAST_ENDPOINT
|
||||
).decode(),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
pipe.close()
|
||||
finally:
|
||||
in_sock.close(linger=0)
|
||||
out_sock.close(linger=0)
|
||||
finally:
|
||||
ctx.term()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_server_args():
|
||||
"""Fixture to provide arguments for APIServerProcessManager."""
|
||||
sock = socket.socket()
|
||||
return {
|
||||
"target_server_fn": mock_run_api_server_worker,
|
||||
"listen_address": "localhost:8000",
|
||||
"sock": sock,
|
||||
"args": "test_args", # Simple string to avoid pickling issues
|
||||
"num_servers": 3,
|
||||
"input_addresses": [
|
||||
"tcp://127.0.0.1:5001",
|
||||
"tcp://127.0.0.1:5002",
|
||||
"tcp://127.0.0.1:5003",
|
||||
],
|
||||
"output_addresses": [
|
||||
"tcp://127.0.0.1:6001",
|
||||
"tcp://127.0.0.1:6002",
|
||||
"tcp://127.0.0.1:6003",
|
||||
],
|
||||
"stats_update_address": "tcp://127.0.0.1:7000",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_stats_update", [True, False])
|
||||
def test_api_server_process_manager_init(api_server_args, with_stats_update):
|
||||
"""Test initializing the APIServerProcessManager."""
|
||||
# Set the worker runtime to ensure tests complete in reasonable time
|
||||
global WORKER_RUNTIME_SECONDS
|
||||
WORKER_RUNTIME_SECONDS = 0.5
|
||||
|
||||
# Copy the args to avoid mutating them
|
||||
args = api_server_args.copy()
|
||||
|
||||
if not with_stats_update:
|
||||
args.pop("stats_update_address")
|
||||
manager = APIServerProcessManager(**args)
|
||||
|
||||
try:
|
||||
# Verify the manager was initialized correctly
|
||||
assert len(manager.processes) == 3
|
||||
|
||||
# Verify all processes are running
|
||||
for proc in manager.processes:
|
||||
assert proc.is_alive()
|
||||
|
||||
print("Waiting for processes to run...")
|
||||
time.sleep(WORKER_RUNTIME_SECONDS / 2)
|
||||
|
||||
# They should still be alive at this point
|
||||
for proc in manager.processes:
|
||||
assert proc.is_alive()
|
||||
|
||||
finally:
|
||||
# Always clean up the processes
|
||||
print("Cleaning up processes...")
|
||||
manager.shutdown()
|
||||
|
||||
# Give processes time to terminate
|
||||
time.sleep(0.2)
|
||||
|
||||
# Verify all processes were terminated
|
||||
for proc in manager.processes:
|
||||
assert not proc.is_alive()
|
||||
|
||||
|
||||
@patch("vllm.v1.utils.run_api_server_worker_proc", mock_run_api_server_worker)
|
||||
def test_wait_for_completion_or_failure(api_server_args):
|
||||
"""Test that wait_for_completion_or_failure works with failures."""
|
||||
global WORKER_RUNTIME_SECONDS
|
||||
WORKER_RUNTIME_SECONDS = 1.0
|
||||
|
||||
# Create the manager
|
||||
manager = APIServerProcessManager(**api_server_args)
|
||||
|
||||
try:
|
||||
assert len(manager.processes) == 3
|
||||
|
||||
# Create a result capture for the thread
|
||||
result: dict[str, Exception | None] = {"exception": None}
|
||||
|
||||
def run_with_exception_capture():
|
||||
try:
|
||||
wait_for_completion_or_failure(api_server_manager=manager)
|
||||
except Exception as e:
|
||||
result["exception"] = e
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
# Start a thread to run wait_for_completion_or_failure
|
||||
wait_thread = threading.Thread(target=run_with_exception_capture, daemon=True)
|
||||
wait_thread.start()
|
||||
|
||||
# Let all processes run for a short time
|
||||
time.sleep(0.2)
|
||||
|
||||
# All processes should still be running
|
||||
assert all(proc.is_alive() for proc in manager.processes)
|
||||
|
||||
# Now simulate a process failure
|
||||
print("Simulating process failure...")
|
||||
manager.processes[0].terminate()
|
||||
|
||||
# Wait for the wait_for_completion_or_failure
|
||||
# to detect and handle the failure
|
||||
# This should trigger it to terminate all other processes
|
||||
wait_thread.join(timeout=1.0)
|
||||
|
||||
# The wait thread should have exited
|
||||
assert not wait_thread.is_alive()
|
||||
|
||||
# Verify that an exception was raised with appropriate error message
|
||||
assert result["exception"] is not None
|
||||
assert "died with exit code" in str(result["exception"])
|
||||
|
||||
# All processes should now be terminated
|
||||
for i, proc in enumerate(manager.processes):
|
||||
assert not proc.is_alive(), f"Process {i} should not be alive"
|
||||
|
||||
finally:
|
||||
manager.shutdown()
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
@pytest.mark.timeout(30)
|
||||
def test_normal_completion(api_server_args):
|
||||
"""Test that wait_for_completion_or_failure works in normal completion."""
|
||||
global WORKER_RUNTIME_SECONDS
|
||||
WORKER_RUNTIME_SECONDS = 0.1
|
||||
|
||||
# Create the manager
|
||||
manager = APIServerProcessManager(**api_server_args)
|
||||
|
||||
try:
|
||||
# Give processes time to terminate
|
||||
# wait for processes to complete
|
||||
remaining_processes = manager.processes.copy()
|
||||
while remaining_processes:
|
||||
for proc in remaining_processes:
|
||||
if not proc.is_alive():
|
||||
remaining_processes.remove(proc)
|
||||
time.sleep(0.1)
|
||||
|
||||
# Verify all processes have terminated
|
||||
for i, proc in enumerate(manager.processes):
|
||||
assert not proc.is_alive(), f"Process {i} still alive after terminate()"
|
||||
|
||||
# Now call wait_for_completion_or_failure
|
||||
# since all processes have already
|
||||
# terminated, it should return immediately
|
||||
# with no error
|
||||
try:
|
||||
wait_for_completion_or_failure(api_server_manager=manager)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
finally:
|
||||
# Clean up just in case
|
||||
manager.shutdown()
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
@pytest.mark.timeout(30)
|
||||
def test_external_process_monitoring(api_server_args):
|
||||
"""Test that wait_for_completion_or_failure handles additional processes."""
|
||||
global WORKER_RUNTIME_SECONDS
|
||||
prev_worker_runtime = WORKER_RUNTIME_SECONDS
|
||||
WORKER_RUNTIME_SECONDS = 100
|
||||
|
||||
# Create and start the external process
|
||||
# (simulates local_engine_manager or coordinator)
|
||||
spawn_context = multiprocessing.get_context("spawn")
|
||||
external_proc = spawn_context.Process(
|
||||
target=mock_run_api_server_worker, name="MockExternalProcess"
|
||||
)
|
||||
external_proc.start()
|
||||
|
||||
# Create the class to simulate a coordinator
|
||||
class MockCoordinator:
|
||||
def __init__(self, proc):
|
||||
self.proc = proc
|
||||
|
||||
def shutdown(self):
|
||||
if self.proc.is_alive():
|
||||
self.proc.terminate()
|
||||
self.proc.join(timeout=0.5)
|
||||
|
||||
# Create a mock coordinator with the external process
|
||||
mock_coordinator = MockCoordinator(external_proc)
|
||||
|
||||
# Create the API server manager
|
||||
manager = APIServerProcessManager(**api_server_args)
|
||||
|
||||
try:
|
||||
# Verify manager initialization
|
||||
assert len(manager.processes) == 3
|
||||
|
||||
# Create a result capture for the thread
|
||||
result: dict[str, Exception | None] = {"exception": None}
|
||||
|
||||
def run_with_exception_capture():
|
||||
try:
|
||||
wait_for_completion_or_failure(
|
||||
api_server_manager=manager, coordinator=mock_coordinator
|
||||
)
|
||||
except Exception as e:
|
||||
result["exception"] = e
|
||||
finally:
|
||||
manager.shutdown()
|
||||
mock_coordinator.shutdown()
|
||||
|
||||
# Start a thread to run wait_for_completion_or_failure
|
||||
wait_thread = threading.Thread(target=run_with_exception_capture, daemon=True)
|
||||
wait_thread.start()
|
||||
|
||||
# Terminate the external process to trigger a failure
|
||||
time.sleep(0.2)
|
||||
external_proc.terminate()
|
||||
|
||||
# Wait for the thread to detect the failure
|
||||
wait_thread.join(timeout=1.0)
|
||||
|
||||
# The wait thread should have completed
|
||||
assert not wait_thread.is_alive(), (
|
||||
"wait_for_completion_or_failure thread still running"
|
||||
)
|
||||
|
||||
# Verify that an exception was raised with appropriate error message
|
||||
assert result["exception"] is not None, "No exception was raised"
|
||||
error_message = str(result["exception"])
|
||||
assert "died with exit code" in error_message, (
|
||||
f"Unexpected error message: {error_message}"
|
||||
)
|
||||
assert "MockExternalProcess" in error_message, (
|
||||
f"Error doesn't mention external process: {error_message}"
|
||||
)
|
||||
|
||||
# Verify that all API server processes were terminated as a result
|
||||
for i, proc in enumerate(manager.processes):
|
||||
assert not proc.is_alive(), f"API server process {i} was not terminated"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
manager.shutdown()
|
||||
mock_coordinator.shutdown()
|
||||
time.sleep(0.2)
|
||||
WORKER_RUNTIME_SECONDS = prev_worker_runtime
|
||||
|
||||
|
||||
@pytest.mark.timeout(60)
|
||||
def test_gather_actual_addresses_end_to_end():
|
||||
"""Each child binds ROUTER/PULL with a kernel-picked port and reports
|
||||
the bound endpoints back via its per-child pipe; the manager surfaces
|
||||
them via :py:meth:`gather_actual_addresses`."""
|
||||
host = "127.0.0.1"
|
||||
num_servers = 4
|
||||
|
||||
placeholder_inputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
placeholder_outputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
for addr in placeholder_inputs + placeholder_outputs:
|
||||
assert addr == f"tcp://{host}:0", addr
|
||||
|
||||
sock = socket.socket()
|
||||
manager = APIServerProcessManager(
|
||||
listen_address=f"tcp://{host}:0",
|
||||
sock=sock,
|
||||
args="test_args",
|
||||
num_servers=num_servers,
|
||||
input_addresses=placeholder_inputs,
|
||||
output_addresses=placeholder_outputs,
|
||||
target_server_fn=defer_addresses_stub_worker,
|
||||
)
|
||||
|
||||
try:
|
||||
assert len(manager.processes) == num_servers
|
||||
actual_inputs, actual_outputs = manager.gather_actual_addresses(timeout=15.0)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
time.sleep(0.2)
|
||||
sock.close()
|
||||
|
||||
assert len(actual_inputs) == num_servers
|
||||
assert len(actual_outputs) == num_servers
|
||||
|
||||
for addr in actual_inputs + actual_outputs:
|
||||
scheme, parsed_host, port = split_zmq_path(addr)
|
||||
assert scheme == "tcp", addr
|
||||
assert parsed_host == host, addr
|
||||
assert port and int(port) > 0, addr
|
||||
|
||||
all_addrs = actual_inputs + actual_outputs
|
||||
assert len(set(all_addrs)) == len(all_addrs), all_addrs
|
||||
|
||||
|
||||
@pytest.mark.timeout(30)
|
||||
def test_gather_actual_addresses_child_crash_before_report():
|
||||
"""A child that exits before sending its endpoints must surface a
|
||||
clear ``RuntimeError`` rather than hang or return ``None`` slots."""
|
||||
host = "127.0.0.1"
|
||||
num_servers = 2
|
||||
placeholder_inputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
placeholder_outputs = [
|
||||
get_engine_client_zmq_addr(local_only=False, host=host)
|
||||
for _ in range(num_servers)
|
||||
]
|
||||
|
||||
sock = socket.socket()
|
||||
manager = APIServerProcessManager(
|
||||
listen_address=f"tcp://{host}:0",
|
||||
sock=sock,
|
||||
args="test_args",
|
||||
num_servers=num_servers,
|
||||
input_addresses=placeholder_inputs,
|
||||
output_addresses=placeholder_outputs,
|
||||
# exit_before_report_worker exits without touching
|
||||
# ``actual_address_pipe`` — simulates a child that dies before
|
||||
# reporting its bound addresses.
|
||||
target_server_fn=exit_before_report_worker,
|
||||
)
|
||||
try:
|
||||
# Sentinel-first vs pipe-EOF-first both produce "reporting".
|
||||
with pytest.raises(RuntimeError, match="reporting"):
|
||||
manager.gather_actual_addresses(timeout=10.0)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
time.sleep(0.2)
|
||||
sock.close()
|
||||
@@ -0,0 +1,175 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from tests.v1.utils import check_request_balancing
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
DP_SIZE = os.getenv("DP_SIZE", "1")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--api-server-count",
|
||||
"4",
|
||||
"--data_parallel_size",
|
||||
DP_SIZE,
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(default_server_args):
|
||||
with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_single_completion(
|
||||
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
async def make_request():
|
||||
completion = await client.completions.create(
|
||||
model=model_name, prompt="Hello, my name is", max_tokens=10, temperature=1.0
|
||||
)
|
||||
|
||||
assert completion.id is not None
|
||||
assert completion.choices is not None and len(completion.choices) == 1
|
||||
|
||||
choice = completion.choices[0]
|
||||
# The exact number of tokens can vary slightly with temperature=1.0,
|
||||
# so we check for a reasonable minimum length.
|
||||
assert len(choice.text) >= 1
|
||||
# Finish reason might not always be 'length' if the model finishes early
|
||||
# or due to other reasons, especially with high temperature.
|
||||
# So, we'll accept 'length' or 'stop'.
|
||||
assert choice.finish_reason in ("length", "stop")
|
||||
|
||||
# Token counts can also vary, so we check they are positive.
|
||||
assert completion.usage.completion_tokens > 0
|
||||
assert completion.usage.prompt_tokens > 0
|
||||
assert completion.usage.total_tokens > 0
|
||||
return completion
|
||||
|
||||
# Test single request
|
||||
result = await make_request()
|
||||
assert result is not None
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send two bursts of requests
|
||||
num_requests = 100
|
||||
tasks = [make_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
assert len(results) == num_requests
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
tasks = [make_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
assert len(results) == num_requests
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
# Check request balancing via Prometheus metrics if DP_SIZE > 1
|
||||
check_request_balancing(server, int(DP_SIZE))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_completion_streaming(
|
||||
client: openai.AsyncOpenAI, server: RemoteOpenAIServer, model_name: str
|
||||
) -> None:
|
||||
prompt = "What is an LLM?"
|
||||
|
||||
async def make_streaming_request():
|
||||
# Perform a non-streaming request to get the expected full output
|
||||
single_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
single_output = single_completion.choices[0].text
|
||||
|
||||
# Perform the streaming request
|
||||
stream = await client.completions.create(
|
||||
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
|
||||
)
|
||||
chunks: list[str] = []
|
||||
finish_reason_count = 0
|
||||
last_chunk = None
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
last_chunk = chunk # Keep track of the last chunk
|
||||
|
||||
# finish reason should only return in the last block for OpenAI API
|
||||
assert finish_reason_count == 1, "Finish reason should appear exactly once."
|
||||
assert last_chunk is not None, "Stream should have yielded at least one chunk."
|
||||
assert last_chunk.choices[0].finish_reason == "length", (
|
||||
"Finish reason should be 'length'."
|
||||
)
|
||||
# Check that the combined text matches the non-streamed version.
|
||||
assert "".join(chunks) == single_output, (
|
||||
"Streamed output should match non-streamed output."
|
||||
)
|
||||
return True # Indicate success for this request
|
||||
|
||||
# Test single request
|
||||
result = await make_streaming_request()
|
||||
assert result is not None
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send two bursts of requests
|
||||
num_requests = 100
|
||||
tasks = [make_streaming_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == num_requests, (
|
||||
f"Expected {num_requests} results, got {len(results)}"
|
||||
)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
tasks = [make_streaming_request() for _ in range(num_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == num_requests, (
|
||||
f"Expected {num_requests} results, got {len(results)}"
|
||||
)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
# Check request balancing via Prometheus metrics if DP_SIZE > 1
|
||||
check_request_balancing(server, int(DP_SIZE))
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the `vllm launch` CLI subcommand."""
|
||||
|
||||
import argparse
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.cli.launch import (
|
||||
LaunchSubcommand,
|
||||
RenderSubcommand,
|
||||
cmd_init,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def launch_parser():
|
||||
parser = FlexibleArgumentParser(description="test")
|
||||
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
||||
LaunchSubcommand().subparser_init(subparsers)
|
||||
return parser
|
||||
|
||||
|
||||
def test_subcommand_name():
|
||||
assert LaunchSubcommand().name == "launch"
|
||||
|
||||
|
||||
def test_cmd_init_returns_subcommand():
|
||||
result = cmd_init()
|
||||
assert len(result) == 1
|
||||
assert isinstance(result[0], LaunchSubcommand)
|
||||
|
||||
|
||||
# -- Parsing: `vllm launch render` --
|
||||
|
||||
|
||||
def test_parse_launch_render(launch_parser):
|
||||
args = launch_parser.parse_args(["launch", "render", "--model", "test-model"])
|
||||
assert args.launch_component == "render"
|
||||
|
||||
|
||||
def test_parse_launch_requires_component(launch_parser):
|
||||
with pytest.raises(SystemExit):
|
||||
launch_parser.parse_args(["launch", "--model", "test-model"])
|
||||
|
||||
|
||||
def test_parse_launch_invalid_component(launch_parser):
|
||||
with pytest.raises(SystemExit):
|
||||
launch_parser.parse_args(["launch", "unknown", "--model", "test-model"])
|
||||
|
||||
|
||||
# -- Dispatch --
|
||||
|
||||
|
||||
def test_cmd_launch_render_calls_run():
|
||||
args = argparse.Namespace(model_tag=None, model="test-model")
|
||||
with patch("vllm.entrypoints.cli.launch.uvloop.run") as mock_uvloop_run:
|
||||
RenderSubcommand.cmd(args)
|
||||
mock_uvloop_run.assert_called_once()
|
||||
|
||||
|
||||
def test_cmd_launch_model_tag_overrides():
|
||||
args = argparse.Namespace(
|
||||
model_tag="tag-model",
|
||||
model="original-model",
|
||||
launch_command=lambda a: None,
|
||||
)
|
||||
LaunchSubcommand.cmd(args)
|
||||
assert args.model == "tag-model"
|
||||
|
||||
|
||||
def test_cmd_launch_model_tag_none():
|
||||
args = argparse.Namespace(
|
||||
model_tag=None,
|
||||
model="original-model",
|
||||
launch_command=lambda a: None,
|
||||
)
|
||||
LaunchSubcommand.cmd(args)
|
||||
assert args.model == "original-model"
|
||||
|
||||
|
||||
def test_cmd_dispatches():
|
||||
called = {}
|
||||
|
||||
def fake_dispatch(args):
|
||||
called["args"] = args
|
||||
|
||||
args = argparse.Namespace(launch_command=fake_dispatch)
|
||||
LaunchSubcommand.cmd(args)
|
||||
assert "args" in called
|
||||
|
||||
|
||||
# -- Module registration --
|
||||
|
||||
|
||||
def test_subparser_init_returns_parser():
|
||||
parser = FlexibleArgumentParser(description="test")
|
||||
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
||||
result = LaunchSubcommand().subparser_init(subparsers)
|
||||
assert isinstance(result, FlexibleArgumentParser)
|
||||
|
||||
|
||||
def test_launch_registered_in_main():
|
||||
"""Verify that launch module is importable as a CLI module."""
|
||||
import vllm.entrypoints.cli.launch as launch_module
|
||||
|
||||
assert hasattr(launch_module, "cmd_init")
|
||||
subcmds = launch_module.cmd_init()
|
||||
assert any(s.name == "launch" for s in subcmds)
|
||||
@@ -0,0 +1,570 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Integration tests for shutdown behavior, timeout, and signal handling."""
|
||||
|
||||
import asyncio
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
# GPU initialization might take take longer
|
||||
_SERVER_STARTUP_TIMEOUT = 120
|
||||
_PROCESS_EXIT_TIMEOUT = 15
|
||||
_SHUTDOWN_DETECTION_TIMEOUT = 10
|
||||
_CHILD_CLEANUP_TIMEOUT = 10
|
||||
_INFLIGHT_REQUEST_START_TIMEOUT = 5
|
||||
_INFLIGHT_REQUEST_POLL_INTERVAL = 0.1
|
||||
_ABORT_CLIENT_TIMEOUT = 3
|
||||
|
||||
|
||||
def _get_child_pids(parent_pid: int) -> list[int]:
|
||||
try:
|
||||
parent = psutil.Process(parent_pid)
|
||||
return [c.pid for c in parent.children(recursive=True)]
|
||||
except psutil.NoSuchProcess:
|
||||
return []
|
||||
|
||||
|
||||
async def _assert_children_cleaned_up(
|
||||
child_pids: list[int],
|
||||
timeout: float = _CHILD_CLEANUP_TIMEOUT,
|
||||
):
|
||||
"""Wait for child processes to exit and fail if any remain."""
|
||||
if not child_pids:
|
||||
return
|
||||
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
still_alive = []
|
||||
for pid in child_pids:
|
||||
try:
|
||||
p = psutil.Process(pid)
|
||||
if p.is_running() and p.status() != psutil.STATUS_ZOMBIE:
|
||||
still_alive.append(pid)
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
if not still_alive:
|
||||
return
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
pytest.fail(
|
||||
f"Child processes {still_alive} still alive after {timeout}s. "
|
||||
f"Process cleanup may not be working correctly."
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ShutdownState:
|
||||
got_503: bool = False
|
||||
got_500: bool = False
|
||||
requests_after_sigterm: int = 0
|
||||
aborted_requests: int = 0
|
||||
connection_errors: int = 0
|
||||
inflight_requests: int = 0
|
||||
stop_requesting: bool = False
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
async def _concurrent_request_loop(
|
||||
client: openai.AsyncOpenAI,
|
||||
state: ShutdownState,
|
||||
sigterm_sent: asyncio.Event | None = None,
|
||||
concurrency: int = 10,
|
||||
):
|
||||
"""Run multiple concurrent requests to keep the server busy."""
|
||||
|
||||
async def single_request():
|
||||
while not state.stop_requesting:
|
||||
try:
|
||||
state.inflight_requests += 1
|
||||
response = await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Write a story: ",
|
||||
max_tokens=200,
|
||||
)
|
||||
if sigterm_sent is not None and sigterm_sent.is_set():
|
||||
state.requests_after_sigterm += 1
|
||||
# Check if any choice has finish_reason='abort'
|
||||
if any(choice.finish_reason == "abort" for choice in response.choices):
|
||||
state.aborted_requests += 1
|
||||
except openai.APIStatusError as e:
|
||||
if e.status_code == 503:
|
||||
state.got_503 = True
|
||||
elif e.status_code == 500:
|
||||
state.got_500 = True
|
||||
else:
|
||||
state.errors.append(f"API error: {e}")
|
||||
except (openai.APIConnectionError, httpx.RemoteProtocolError):
|
||||
state.connection_errors += 1
|
||||
if sigterm_sent is not None and sigterm_sent.is_set():
|
||||
break
|
||||
except Exception as e:
|
||||
state.errors.append(f"Unexpected error: {e}")
|
||||
break
|
||||
finally:
|
||||
state.inflight_requests -= 1
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
tasks = [asyncio.create_task(single_request()) for _ in range(concurrency)]
|
||||
try:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
finally:
|
||||
for t in tasks:
|
||||
if not t.done():
|
||||
t.cancel()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_on_engine_failure(tmp_path: Path):
|
||||
"""Verify that API returns connection error when server process is killed.
|
||||
|
||||
Starts a vLLM server, kills it to simulate a crash, then verifies that
|
||||
subsequent API calls fail appropriately.
|
||||
"""
|
||||
|
||||
port = get_open_port()
|
||||
|
||||
# Redirect server output to a file rather than a pipe: nothing drains the
|
||||
# pipes while we poll for startup, so a server that writes more than the
|
||||
# pipe buffer holds would block on write and never become ready.
|
||||
server_log = tmp_path / "server.log"
|
||||
with server_log.open("wb") as log_file:
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
# dtype, max-len etc set so that this can run in CI
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.entrypoints.launchers.api_server.entry",
|
||||
"--model",
|
||||
MODEL_NAME,
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
"--port",
|
||||
str(port),
|
||||
"--gpu-memory-utilization",
|
||||
"0.05",
|
||||
"--max-num-seqs",
|
||||
"2",
|
||||
],
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
preexec_fn=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN),
|
||||
)
|
||||
|
||||
# Wait for server startup
|
||||
start_time = time.time()
|
||||
client = openai.AsyncOpenAI(
|
||||
base_url=f"http://localhost:{port}/v1",
|
||||
api_key="dummy",
|
||||
max_retries=0,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
# Poll until server is ready
|
||||
while time.time() - start_time < _SERVER_STARTUP_TIMEOUT:
|
||||
try:
|
||||
await client.completions.create(
|
||||
model=MODEL_NAME, prompt="Hello", max_tokens=1
|
||||
)
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
if proc.poll() is not None:
|
||||
pytest.fail(
|
||||
f"Server died during startup: {proc.returncode}\n"
|
||||
f"{server_log.read_text(errors='replace')}"
|
||||
)
|
||||
else:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=_PROCESS_EXIT_TIMEOUT)
|
||||
pytest.fail(
|
||||
f"Server failed to start in {_SERVER_STARTUP_TIMEOUT} seconds\n"
|
||||
f"{server_log.read_text(errors='replace')}"
|
||||
)
|
||||
|
||||
# Kill server to simulate crash
|
||||
proc.terminate()
|
||||
time.sleep(1)
|
||||
|
||||
# Verify API calls now fail
|
||||
with pytest.raises((openai.APIConnectionError, openai.APIStatusError)):
|
||||
await client.completions.create(
|
||||
model=MODEL_NAME, prompt="This should fail", max_tokens=1
|
||||
)
|
||||
|
||||
return_code = proc.wait(timeout=_PROCESS_EXIT_TIMEOUT)
|
||||
assert return_code is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_timeout_completes_requests():
|
||||
"""Verify wait timeout: new requests rejected, in-flight requests complete."""
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.05",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--shutdown-timeout",
|
||||
"30",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
proc = remote_server.proc
|
||||
child_pids = _get_child_pids(proc.pid)
|
||||
|
||||
state = ShutdownState()
|
||||
sigterm_sent = asyncio.Event()
|
||||
|
||||
request_task = asyncio.create_task(
|
||||
_concurrent_request_loop(client, state, sigterm_sent, concurrency=10)
|
||||
)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
sigterm_sent.set()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
state.stop_requesting = True
|
||||
if not request_task.done():
|
||||
request_task.cancel()
|
||||
await asyncio.gather(request_task, return_exceptions=True)
|
||||
|
||||
# wait timeout should complete in-flight requests
|
||||
assert state.requests_after_sigterm > 0, (
|
||||
f"Wait timeout should complete in-flight requests. "
|
||||
f"503: {state.got_503}, 500: {state.got_500}, "
|
||||
f"conn_errors: {state.connection_errors}, errors: {state.errors}"
|
||||
)
|
||||
# server must stop accepting new requests (503, 500, or connection close)
|
||||
assert state.got_503 or state.got_500 or state.connection_errors > 0, (
|
||||
f"Server should stop accepting requests. "
|
||||
f"completed: {state.requests_after_sigterm}, errors: {state.errors}"
|
||||
)
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0])
|
||||
async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.05",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--shutdown-timeout",
|
||||
"0",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
proc = remote_server.proc
|
||||
child_pids = _get_child_pids(proc.pid)
|
||||
|
||||
if wait_for_engine_idle > 0:
|
||||
client = remote_server.get_async_client()
|
||||
# Send requests to ensure engine is fully initialized
|
||||
for _ in range(2):
|
||||
await client.completions.create(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test request: ",
|
||||
max_tokens=10,
|
||||
)
|
||||
# Wait for engine to become idle
|
||||
await asyncio.sleep(wait_for_engine_idle)
|
||||
|
||||
start_time = time.time()
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
# abort timeout (0) should stop the server promptly.
|
||||
try:
|
||||
proc.wait(timeout=4.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
pytest.fail("Process did not exit after SIGTERM with abort timeout")
|
||||
|
||||
exit_time = time.time() - start_time
|
||||
assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s"
|
||||
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_timeout_with_short_duration():
|
||||
"""Verify server exits cleanly with a short wait timeout."""
|
||||
wait_timeout = 3
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.05",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--shutdown-timeout",
|
||||
str(wait_timeout),
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
proc = remote_server.proc
|
||||
child_pids = _get_child_pids(proc.pid)
|
||||
|
||||
state = ShutdownState()
|
||||
request_task = asyncio.create_task(
|
||||
_concurrent_request_loop(client, state, concurrency=3)
|
||||
)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
start_time = time.time()
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
# server should exit within wait_timeout + buffer
|
||||
max_wait = wait_timeout + 15
|
||||
for _ in range(int(max_wait * 10)):
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
exit_time = time.time() - start_time
|
||||
|
||||
state.stop_requesting = True
|
||||
if not request_task.done():
|
||||
request_task.cancel()
|
||||
await asyncio.gather(request_task, return_exceptions=True)
|
||||
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
pytest.fail(f"Process did not exit within {max_wait}s after SIGTERM")
|
||||
|
||||
assert exit_time < wait_timeout + 10, (
|
||||
f"Took too long to exit ({exit_time:.1f}s), expected <{wait_timeout + 10}s"
|
||||
)
|
||||
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_timeout_fails_inflight_requests():
|
||||
"""Verify abort timeout (0) immediately aborts in-flight requests."""
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.05",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--shutdown-timeout",
|
||||
"0",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
client = remote_server.get_async_client(timeout=_ABORT_CLIENT_TIMEOUT)
|
||||
proc = remote_server.proc
|
||||
child_pids = _get_child_pids(proc.pid)
|
||||
|
||||
state = ShutdownState()
|
||||
sigterm_sent = asyncio.Event()
|
||||
|
||||
request_task = asyncio.create_task(
|
||||
_concurrent_request_loop(client, state, sigterm_sent, concurrency=10)
|
||||
)
|
||||
|
||||
deadline = time.time() + _INFLIGHT_REQUEST_START_TIMEOUT
|
||||
while state.inflight_requests == 0 and time.time() < deadline:
|
||||
await asyncio.sleep(_INFLIGHT_REQUEST_POLL_INTERVAL)
|
||||
assert state.inflight_requests > 0
|
||||
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
sigterm_sent.set()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(request_task, timeout=5)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
state.stop_requesting = True
|
||||
if not request_task.done():
|
||||
request_task.cancel()
|
||||
await asyncio.gather(request_task, return_exceptions=True)
|
||||
|
||||
# With abort timeout (0), requests should be aborted (finish_reason='abort')
|
||||
# or rejected (connection errors or API errors)
|
||||
assert (
|
||||
state.aborted_requests > 0
|
||||
or state.connection_errors > 0
|
||||
or state.got_500
|
||||
or state.got_503
|
||||
), (
|
||||
f"Abort timeout should cause request aborts or failures. "
|
||||
f"aborted: {state.aborted_requests}, "
|
||||
f"503: {state.got_503}, 500: {state.got_500}, "
|
||||
f"conn_errors: {state.connection_errors}, "
|
||||
f"completed: {state.requests_after_sigterm}"
|
||||
)
|
||||
|
||||
# Verify fast shutdown
|
||||
start_time = time.time()
|
||||
for _ in range(100):
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
exit_time = time.time() - start_time
|
||||
assert exit_time < 10, f"Abort timeout shutdown took too long: {exit_time:.1f}s"
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_rejection_during_shutdown():
|
||||
"""Verify new requests are rejected with error during shutdown."""
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.05",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--shutdown-timeout",
|
||||
"30",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
proc = remote_server.proc
|
||||
child_pids = _get_child_pids(proc.pid)
|
||||
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Try to send new requests - they should be rejected
|
||||
rejected_count = 0
|
||||
for _ in range(10):
|
||||
try:
|
||||
await client.completions.create(
|
||||
model=MODEL_NAME, prompt="Hello", max_tokens=10
|
||||
)
|
||||
except (
|
||||
openai.APIStatusError,
|
||||
openai.APIConnectionError,
|
||||
httpx.RemoteProtocolError,
|
||||
):
|
||||
rejected_count += 1
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert rejected_count > 0, (
|
||||
f"Expected requests to be rejected during shutdown, "
|
||||
f"but {rejected_count} were rejected out of 10"
|
||||
)
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_api_server_shutdown():
|
||||
"""Verify shutdown works with multiple API servers."""
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.05",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--shutdown-timeout",
|
||||
"30",
|
||||
"--api-server-count",
|
||||
"2",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, server_args, auto_port=True) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
proc = remote_server.proc
|
||||
child_pids = _get_child_pids(proc.pid)
|
||||
|
||||
assert len(child_pids) >= 2, (
|
||||
f"Expected at least 2 child processes, got {len(child_pids)}"
|
||||
)
|
||||
|
||||
state = ShutdownState()
|
||||
sigterm_sent = asyncio.Event()
|
||||
|
||||
# Start concurrent requests across both API servers
|
||||
request_task = asyncio.create_task(
|
||||
_concurrent_request_loop(client, state, sigterm_sent, concurrency=8)
|
||||
)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send SIGTERM to parent - should propagate to all children
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
sigterm_sent.set()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
state.stop_requesting = True
|
||||
if not request_task.done():
|
||||
request_task.cancel()
|
||||
await asyncio.gather(request_task, return_exceptions=True)
|
||||
|
||||
for _ in range(300): # up to 30 seconds
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
pytest.fail("Process did not exit after SIGTERM")
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from ssl import SSLContext
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.launchers.utils.ssl import SSLCertRefresher
|
||||
|
||||
|
||||
class MockSSLContext(SSLContext):
|
||||
def __init__(self):
|
||||
self.load_cert_chain_count = 0
|
||||
self.load_ca_count = 0
|
||||
|
||||
def load_cert_chain(
|
||||
self,
|
||||
certfile,
|
||||
keyfile=None,
|
||||
password=None,
|
||||
):
|
||||
self.load_cert_chain_count += 1
|
||||
|
||||
def load_verify_locations(
|
||||
self,
|
||||
cafile=None,
|
||||
capath=None,
|
||||
cadata=None,
|
||||
):
|
||||
self.load_ca_count += 1
|
||||
|
||||
|
||||
def create_file() -> str:
|
||||
with tempfile.NamedTemporaryFile(dir="/tmp", delete=False) as f:
|
||||
return f.name
|
||||
|
||||
|
||||
def touch_file(path: str) -> None:
|
||||
Path(path).touch()
|
||||
|
||||
|
||||
async def wait_for_counts(
|
||||
ssl_context: MockSSLContext,
|
||||
*,
|
||||
cert_chain_count: int,
|
||||
ca_count: int,
|
||||
timeout: float = 5.0,
|
||||
) -> None:
|
||||
deadline = asyncio.get_running_loop().time() + timeout
|
||||
while True:
|
||||
if (
|
||||
ssl_context.load_cert_chain_count >= cert_chain_count
|
||||
and ssl_context.load_ca_count >= ca_count
|
||||
):
|
||||
return
|
||||
|
||||
if asyncio.get_running_loop().time() >= deadline:
|
||||
assert ssl_context.load_cert_chain_count >= cert_chain_count
|
||||
assert ssl_context.load_ca_count >= ca_count
|
||||
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ssl_refresher():
|
||||
ssl_context = MockSSLContext()
|
||||
key_path = create_file()
|
||||
cert_path = create_file()
|
||||
ca_path = create_file()
|
||||
ssl_refresher = SSLCertRefresher(ssl_context, key_path, cert_path, ca_path)
|
||||
await asyncio.sleep(1)
|
||||
assert ssl_context.load_cert_chain_count == 0
|
||||
assert ssl_context.load_ca_count == 0
|
||||
|
||||
touch_file(key_path)
|
||||
await wait_for_counts(
|
||||
ssl_context,
|
||||
cert_chain_count=1,
|
||||
ca_count=0,
|
||||
)
|
||||
assert ssl_context.load_ca_count == 0
|
||||
|
||||
touch_file(cert_path)
|
||||
touch_file(ca_path)
|
||||
await wait_for_counts(
|
||||
ssl_context,
|
||||
cert_chain_count=2,
|
||||
ca_count=1,
|
||||
)
|
||||
|
||||
ssl_refresher.stop()
|
||||
await asyncio.sleep(0)
|
||||
cert_chain_count = ssl_context.load_cert_chain_count
|
||||
ca_count = ssl_context.load_ca_count
|
||||
|
||||
touch_file(cert_path)
|
||||
touch_file(ca_path)
|
||||
await asyncio.sleep(1)
|
||||
assert ssl_context.load_cert_chain_count == cert_chain_count
|
||||
assert ssl_context.load_ca_count == ca_count
|
||||
Reference in New Issue
Block a user