[Frontend] Share max_num_queued_reqs across API server processes (#54746)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-09-15 14:50:26 +00:00
committed by GitHub
parent 0136df94b0
commit f0fbc98c48
6 changed files with 173 additions and 6 deletions
+19 -5
View File
@@ -44,6 +44,7 @@ from vllm.usage.usage_lib import UsageContext
from vllm.utils.async_utils import cancel_task_threadsafe
from vllm.utils.collection_utils import as_list
from vllm.v1.engine import EngineCoreRequest, PauseMode
from vllm.v1.engine.admission_control import SharedAdmissionStats
from vllm.v1.engine.core_client import EngineCoreClient
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm.v1.engine.input_processor import InputProcessor
@@ -144,12 +145,20 @@ class AsyncLLM(EngineClient):
# Convert EngineInput --> EngineCoreRequest.
self.input_processor = InputProcessor(self.vllm_config, renderer)
self.admission_stats = (
SharedAdmissionStats(client_addresses, client_count, client_index)
if client_addresses is not None
and "mp_admission_counters" in client_addresses
else None
)
# Converts EngineCoreOutputs --> RequestOutput.
self.output_processor = OutputProcessor(
renderer.tokenizer,
log_stats=self.log_stats,
stream_interval=self.vllm_config.scheduler_config.stream_interval,
tracing_enabled=tracing_endpoint is not None,
admission_stats=self.admission_stats,
)
# EngineCore (starts the engine in background process).
@@ -291,8 +300,9 @@ class AsyncLLM(EngineClient):
Both limits return HTTP 503 (Service Unavailable) so that load
balancers and client SDKs retry on a different instance.
- ``max_num_queued_reqs``: hard cap on the number of unfinished requests
(waiting + running). A request with ``n > 1`` counts as ``n`` slots.
- ``max_num_queued_reqs``: approximate global cap on the number of
unfinished requests (waiting + running). A request with ``n > 1``
counts as ``n`` slots.
- ``max_num_queued_tokens``: TTFT QoS — cap on the total prompt
tokens of requests still in prefill.
@@ -312,13 +322,17 @@ class AsyncLLM(EngineClient):
"""
max_num_reqs = self.scheduler_config.max_num_queued_reqs
if max_num_reqs is not None:
current = self.get_num_unfinished_requests()
if current + n > max_num_reqs:
current_requests = (
self.admission_stats.get_num_requests()
if self.admission_stats is not None
else self.get_num_unfinished_requests()
)
if current_requests + n > max_num_reqs:
logger.info(
"Request queue full - rejecting request %s "
"(current=%d, n=%d, max=%d).",
request_id,
current,
current_requests,
n,
max_num_reqs,
)