[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+80 -6
View File
@@ -21,7 +21,13 @@ from vllm.distributed.weight_transfer.base import (
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
from vllm.exceptions import VLLMClientError, VLLMValidationError
from vllm.exceptions import (
GracefulHTTPError,
MaxQueuedTokensError,
QueueOverflowError,
VLLMClientError,
VLLMValidationError,
)
from vllm.inputs import EngineInput, PromptType
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -112,6 +118,7 @@ class AsyncLLM(EngineClient):
self.vllm_config = vllm_config
self._elastic_ep_lock = asyncio.Lock()
self.model_config = vllm_config.model_config
self.scheduler_config = vllm_config.scheduler_config
self.observability_config = vllm_config.observability_config
tracing_endpoint = self.observability_config.otlp_traces_endpoint
@@ -273,6 +280,64 @@ class AsyncLLM(EngineClient):
if handler is not None:
cancel_task_threadsafe(handler)
def get_num_unfinished_requests(self) -> int:
return self.output_processor.get_num_unfinished_requests()
def get_num_queued_tokens(self) -> int:
return self.output_processor.get_num_queued_tokens()
def check_admission(self, n: int = 1, request_id: str | None = None) -> None:
"""Reject the request if it would exceed queue limits.
Both limits return HTTP 503 (Service Unavailable) so that load
balancers and client SDKs retry on a different instance.
- ``max_num_queued_reqs``: hard cap on the number of unfinished requests
(waiting + running). A request with ``n > 1`` counts as ``n`` slots.
- ``max_num_queued_tokens``: TTFT QoS — cap on the total prompt
tokens of requests still in prefill.
Note: ``get_num_queued_tokens`` uses ``prompt_len`` for all prefilling requests.
Chunked prefill progress and prefix-cache hits are not subtracted because the
scheduler's ``num_computed_tokens`` and ``num_cached_tokens`` are only
propagated to the API server after prefill completes. The overestimation is
conservative — earlier rejection, preserving TTFT targets.
Args:
n: Number of sequences the request will occupy.
request_id: Request id, used for logging only.
Raises:
QueueOverflowError: If ``max_num_queued_reqs`` would be exceeded.
MaxQueuedTokensError: If ``max_num_queued_tokens`` would be exceeded.
"""
max_num_reqs = self.scheduler_config.max_num_queued_reqs
if max_num_reqs is not None:
current = self.get_num_unfinished_requests()
if current + n > max_num_reqs:
logger.info(
"Request queue full - rejecting request %s "
"(current=%d, n=%d, max=%d).",
request_id,
current,
n,
max_num_reqs,
)
raise QueueOverflowError()
max_queued_tokens = self.scheduler_config.max_num_queued_tokens
if max_queued_tokens is not None:
current_tokens = self.get_num_queued_tokens()
if current_tokens >= max_queued_tokens:
logger.info(
"Max queued tokens reached - rejecting request %s "
"(current_tokens=%d, max=%d).",
request_id,
current_tokens,
max_queued_tokens,
)
raise MaxQueuedTokensError()
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
if not hasattr(self, "_supported_tasks"):
# Cache the result
@@ -317,6 +382,10 @@ class AsyncLLM(EngineClient):
"prompt logprobs"
)
if isinstance(params, SamplingParams) and params.n > 1:
# TODO (NickLucche) Batch check admission check for all n requests
self.check_admission(params.n, request_id)
if isinstance(prompt, AsyncGenerator):
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
raise NotImplementedError
@@ -430,7 +499,12 @@ class AsyncLLM(EngineClient):
index: int,
queue: RequestOutputCollector,
):
# Add the request to OutputProcessor (this process).
if parent_req is None and not self.output_processor.has_request(
request.request_id
):
self.check_admission(request_id=request.request_id)
# Register locally before the first await so concurrent tasks see this request.
self.output_processor.add_request(request, prompt, parent_req, index, queue)
# Add the EngineCoreRequest to EngineCore (separate process).
@@ -630,8 +704,8 @@ class AsyncLLM(EngineClient):
logger.info("Request %s failed (engine dead).", request_id)
raise
# Request validation error.
except VLLMClientError as e:
# Request validation error or admission control rejection.
except (VLLMClientError, GracefulHTTPError) as e:
if self.log_requests:
logger.info("Request %s failed (bad request): %s.", request_id, e)
raise
@@ -907,8 +981,8 @@ class AsyncLLM(EngineClient):
logger.info("Request %s failed (engine dead).", request_id)
raise
# Request validation error.
except VLLMClientError:
# Request validation error or admission control rejection.
except (VLLMClientError, GracefulHTTPError):
if self.log_requests:
logger.info("Request %s failed (bad request).", request_id)
raise
+14
View File
@@ -467,6 +467,20 @@ class OutputProcessor:
def get_num_unfinished_requests(self):
return len(self.request_states)
def has_request(self, request_id: str) -> bool:
return request_id in self.request_states
def get_num_queued_tokens(self) -> int:
"""Total prompt tokens of requests currently in the prefill phase.
Uses ``prompt_len`` rather than remaining prefill work because the
scheduler's ``num_computed_tokens`` is not propagated to the API
server until prefill completes. See ``SchedulerConfig`` docs.
"""
return sum(
req.prompt_len for req in self.request_states.values() if req.is_prefilling
)
def has_unfinished_requests(self) -> bool:
return len(self.request_states) > 0