[Frontend] Share max_num_queued_reqs across API server processes (#54746)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import multiprocessing
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -14,6 +15,7 @@ from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
|
||||
exit_before_report_worker,
|
||||
)
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.utils import (
|
||||
APIServerProcessManager,
|
||||
get_engine_client_zmq_addr,
|
||||
@@ -32,6 +34,18 @@ def mock_run_api_server_worker(listen_address, sock, args, client_config=None):
|
||||
print("Mock worker completed successfully")
|
||||
|
||||
|
||||
def update_admission_stats_worker(listen_address, sock, args, client_config):
|
||||
stats = SharedAdmissionStats(
|
||||
client_config,
|
||||
client_config["client_count"],
|
||||
client_config["client_index"],
|
||||
)
|
||||
value = client_config["client_index"] + 1
|
||||
stats.set_num_requests(value)
|
||||
args.barrier.wait()
|
||||
args.results.put(stats.get_num_requests())
|
||||
|
||||
|
||||
# Module-level stub for the gather_actual_addresses test. Must be
|
||||
# importable by `multiprocessing.spawn` (no closures, no nesting).
|
||||
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
|
||||
@@ -131,6 +145,31 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):
|
||||
assert not proc.is_alive()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_api_server_processes_share_admission_stats(api_server_args):
|
||||
args = api_server_args.copy()
|
||||
spawn_context = multiprocessing.get_context("spawn")
|
||||
worker_args = SimpleNamespace(
|
||||
barrier=spawn_context.Barrier(args["num_servers"]),
|
||||
results=spawn_context.Queue(),
|
||||
max_num_queued_reqs=10,
|
||||
)
|
||||
args["args"] = worker_args
|
||||
args["target_server_fn"] = update_admission_stats_worker
|
||||
manager = APIServerProcessManager(**args)
|
||||
|
||||
try:
|
||||
for process in manager.processes:
|
||||
process.join(timeout=30)
|
||||
assert process.exitcode == 0
|
||||
|
||||
assert [worker_args.results.get(timeout=5) for _ in range(3)] == [6] * 3
|
||||
finally:
|
||||
manager.shutdown()
|
||||
worker_args.results.close()
|
||||
worker_args.results.join_thread()
|
||||
|
||||
|
||||
@patch("vllm.v1.utils.run_api_server_worker_proc", mock_run_api_server_worker)
|
||||
def test_wait_for_completion_or_failure(api_server_args):
|
||||
"""Test that wait_for_completion_or_failure works with failures."""
|
||||
|
||||
@@ -14,6 +14,7 @@ These tests cover:
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import multiprocessing
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -34,6 +35,7 @@ from vllm.exceptions import (
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.argparse_utils import human_readable_int
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
|
||||
@@ -66,6 +68,7 @@ def _make_async_llm(
|
||||
llm.output_processor = MagicMock()
|
||||
llm.output_processor.get_num_unfinished_requests.return_value = num_unfinished
|
||||
llm.output_processor.get_num_queued_tokens.return_value = num_queued_tokens
|
||||
llm.admission_stats = None
|
||||
return llm
|
||||
|
||||
|
||||
@@ -84,6 +87,42 @@ def _make_scheduler_config(**kwargs) -> SchedulerConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_shared_stats(client_count: int = 2) -> list[SharedAdmissionStats]:
|
||||
counters = multiprocessing.RawArray(
|
||||
"q", SharedAdmissionStats.num_counters(client_count)
|
||||
)
|
||||
return [
|
||||
SharedAdmissionStats({"mp_admission_counters": counters}, client_count, index)
|
||||
for index in range(client_count)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_shared_admission_stats_aggregate_api_servers():
|
||||
server_0, server_1 = _make_shared_stats()
|
||||
|
||||
server_0.set_num_requests(2)
|
||||
assert server_0.get_num_requests() == 2
|
||||
server_1.set_num_requests(3)
|
||||
assert server_0.get_num_requests() == server_1.get_num_requests() == 5
|
||||
|
||||
server_0.set_num_requests(1)
|
||||
assert server_0.get_num_requests() == 4
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_shared_request_snapshot_enforces_global_limit():
|
||||
server_0, server_1 = _make_shared_stats()
|
||||
server_0.set_num_requests(2)
|
||||
llm = _make_async_llm(max_num_queued_reqs=2)
|
||||
llm.admission_stats = server_1
|
||||
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
|
||||
assert server_0.get_num_requests() == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exception classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user