[Frontend] Share max_num_queued_reqs across API server processes (#54746)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-09-15 14:50:26 +00:00
committed by GitHub
parent 0136df94b0
commit f0fbc98c48
6 changed files with 173 additions and 6 deletions
@@ -5,6 +5,7 @@ import multiprocessing
import socket
import threading
import time
from types import SimpleNamespace
from unittest.mock import patch
import pytest
@@ -14,6 +15,7 @@ from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
exit_before_report_worker,
)
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
from vllm.v1.engine.admission_control import SharedAdmissionStats
from vllm.v1.utils import (
APIServerProcessManager,
get_engine_client_zmq_addr,
@@ -32,6 +34,18 @@ def mock_run_api_server_worker(listen_address, sock, args, client_config=None):
print("Mock worker completed successfully")
def update_admission_stats_worker(listen_address, sock, args, client_config):
stats = SharedAdmissionStats(
client_config,
client_config["client_count"],
client_config["client_index"],
)
value = client_config["client_index"] + 1
stats.set_num_requests(value)
args.barrier.wait()
args.results.put(stats.get_num_requests())
# Module-level stub for the gather_actual_addresses test. Must be
# importable by `multiprocessing.spawn` (no closures, no nesting).
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
@@ -131,6 +145,31 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):
assert not proc.is_alive()
@pytest.mark.skip_global_cleanup
def test_api_server_processes_share_admission_stats(api_server_args):
args = api_server_args.copy()
spawn_context = multiprocessing.get_context("spawn")
worker_args = SimpleNamespace(
barrier=spawn_context.Barrier(args["num_servers"]),
results=spawn_context.Queue(),
max_num_queued_reqs=10,
)
args["args"] = worker_args
args["target_server_fn"] = update_admission_stats_worker
manager = APIServerProcessManager(**args)
try:
for process in manager.processes:
process.join(timeout=30)
assert process.exitcode == 0
assert [worker_args.results.get(timeout=5) for _ in range(3)] == [6] * 3
finally:
manager.shutdown()
worker_args.results.close()
worker_args.results.join_thread()
@patch("vllm.v1.utils.run_api_server_worker_proc", mock_run_api_server_worker)
def test_wait_for_completion_or_failure(api_server_args):
"""Test that wait_for_completion_or_failure works with failures."""