[Frontend] Share max_num_queued_reqs across API server processes (#54746)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -44,6 +44,7 @@ from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.async_utils import cancel_task_threadsafe
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.v1.engine import EngineCoreRequest, PauseMode
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.engine.core_client import EngineCoreClient
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
@@ -144,12 +145,20 @@ class AsyncLLM(EngineClient):
|
||||
# Convert EngineInput --> EngineCoreRequest.
|
||||
self.input_processor = InputProcessor(self.vllm_config, renderer)
|
||||
|
||||
self.admission_stats = (
|
||||
SharedAdmissionStats(client_addresses, client_count, client_index)
|
||||
if client_addresses is not None
|
||||
and "mp_admission_counters" in client_addresses
|
||||
else None
|
||||
)
|
||||
|
||||
# Converts EngineCoreOutputs --> RequestOutput.
|
||||
self.output_processor = OutputProcessor(
|
||||
renderer.tokenizer,
|
||||
log_stats=self.log_stats,
|
||||
stream_interval=self.vllm_config.scheduler_config.stream_interval,
|
||||
tracing_enabled=tracing_endpoint is not None,
|
||||
admission_stats=self.admission_stats,
|
||||
)
|
||||
|
||||
# EngineCore (starts the engine in background process).
|
||||
@@ -291,8 +300,9 @@ class AsyncLLM(EngineClient):
|
||||
Both limits return HTTP 503 (Service Unavailable) so that load
|
||||
balancers and client SDKs retry on a different instance.
|
||||
|
||||
- ``max_num_queued_reqs``: hard cap on the number of unfinished requests
|
||||
(waiting + running). A request with ``n > 1`` counts as ``n`` slots.
|
||||
- ``max_num_queued_reqs``: approximate global cap on the number of
|
||||
unfinished requests (waiting + running). A request with ``n > 1``
|
||||
counts as ``n`` slots.
|
||||
- ``max_num_queued_tokens``: TTFT QoS — cap on the total prompt
|
||||
tokens of requests still in prefill.
|
||||
|
||||
@@ -312,13 +322,17 @@ class AsyncLLM(EngineClient):
|
||||
"""
|
||||
max_num_reqs = self.scheduler_config.max_num_queued_reqs
|
||||
if max_num_reqs is not None:
|
||||
current = self.get_num_unfinished_requests()
|
||||
if current + n > max_num_reqs:
|
||||
current_requests = (
|
||||
self.admission_stats.get_num_requests()
|
||||
if self.admission_stats is not None
|
||||
else self.get_num_unfinished_requests()
|
||||
)
|
||||
if current_requests + n > max_num_reqs:
|
||||
logger.info(
|
||||
"Request queue full - rejecting request %s "
|
||||
"(current=%d, n=%d, max=%d).",
|
||||
request_id,
|
||||
current,
|
||||
current_requests,
|
||||
n,
|
||||
max_num_reqs,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user