[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+31
View File
@@ -71,6 +71,37 @@ class SchedulerConfig:
"""For chunked prefill, a request is considered long if the prompt is
longer than this number of tokens. 0 disables the cap (default)."""
max_num_queued_reqs: int | None = Field(default=None, ge=0)
"""Maximum number of requests that can be in-flight (waiting or running)
at the same time, or None for no limit. When the limit is reached, new
requests are rejected with HTTP 503 so the client can retry on another
instance. This bounds vLLM's otherwise unbounded request queue and is
primarily a coarse capacity valve."""
max_num_queued_tokens: int | None = Field(default=None, ge=0)
"""Maximum total prompt tokens of requests currently in the prefill
phase, or None for no limit. When the limit is reached, new requests
are rejected with HTTP 503.
This is a TTFT QoS mechanism: by setting it to
``target_TTFT * prefill_throughput`` you reject requests when the
prefill backlog would exceed the latency target. In a disaggregated
prefill-decode setup this maps directly to the prefill pool's
capacity.
Note: the count is conservative. A partially prefilled request
still contributes its full ``prompt_len`` until it transitions out
of the prefill phase, because the scheduler's per-iteration
``num_computed_tokens`` progress is not propagated to the API
server process during prefill (``EngineCoreOutput`` is only
emitted once the request starts producing tokens). Similarly,
prefix-cache hits (``num_cached_tokens``) are only known to the
OutputProcessor after prefill completes. This overestimates the
real backlog, causing earlier rejection than strictly necessary
— the safe direction for QoS. The impact is limited to long
prompts under chunked prefill; short prompts that prefill in a
single iteration are unaffected."""
enable_chunked_prefill: bool = True
"""If True, prefill requests can be chunked based
on the remaining `max_num_batched_tokens`.
+12
View File
@@ -367,6 +367,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
"max_num_scheduled_tokens",
"kv_cache_memory_bytes",
"safetensors_prefetch_block_size",
"max_num_queued_tokens",
}
if name == "max_model_len":
kwargs[name]["type"] = human_readable_int_or_auto
@@ -544,6 +545,8 @@ class EngineArgs:
max_num_scheduled_tokens: int | None = None
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
max_num_seqs: int | None = None
max_num_queued_reqs: int | None = None
max_num_queued_tokens: int | None = None
max_logprobs: int = ModelConfig.max_logprobs
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
@@ -1554,6 +1557,13 @@ class EngineArgs:
"default": None,
},
)
scheduler_group.add_argument(
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
)
scheduler_group.add_argument(
"--max-num-queued-tokens",
**scheduler_kwargs["max_num_queued_tokens"],
)
scheduler_group.add_argument(
"--long-prefill-token-threshold",
**scheduler_kwargs["long_prefill_token_threshold"],
@@ -2348,6 +2358,8 @@ class EngineArgs:
max_num_batched_tokens=self.max_num_batched_tokens,
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
max_num_seqs=self.max_num_seqs,
max_num_queued_reqs=self.max_num_queued_reqs,
max_num_queued_tokens=self.max_num_queued_tokens,
max_model_len=model_config.max_model_len,
enable_chunked_prefill=self.enable_chunked_prefill,
disable_chunked_mm_input=self.disable_chunked_mm_input,
+17
View File
@@ -62,6 +62,23 @@ class EngineClient(ABC):
@abstractmethod
def dead_error(self) -> BaseException: ...
def check_admission( # noqa: B027
self, n: int = 1, request_id: str | None = None
) -> None:
"""Reject the request up front if it would exceed queue limits.
Called before a response is started so that overload rejections can
carry an HTTP status, which is not possible once a streaming response
has begun. Engines without admission control accept everything.
Args:
n: Number of sequences the request will occupy.
request_id: Request id, used for logging only.
Raises:
GracefulHTTPError: If the request cannot be admitted.
"""
@abstractmethod
def generate(
self,
+13
View File
@@ -171,6 +171,19 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
# Never fail server startup over the fingerprint.
self.system_fingerprint = None
def _preflight(self, n: int = 1) -> None:
"""Engine checks that must run before a response is started.
This is required for the streaming case, where we return a success
status before we actually start generating text :).
Args:
n: Number of sequences the request will occupy.
"""
if self.engine_client.errored:
raise self.engine_client.dead_error
self.engine_client.check_admission(n)
def create_streaming_error_response(
self,
message: str | Exception,
@@ -195,6 +195,7 @@ class ServingGenerativeScoring(BaseServing):
# Check if engine is alive
if self.engine_client.errored:
raise self.engine_client.dead_error
self.engine_client.check_admission(len(request.items))
# Get tokenizer
tokenizer = self.renderer.tokenizer
@@ -56,8 +56,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
logger.error("Error with model %s", error_check_ret)
return error_check_ret
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
renderer = self.online_renderer
@@ -235,11 +235,7 @@ class OpenAIServingChat(GenerateBaseServing):
logger.error("Error with model %s", error_check_ret)
return error_check_ret
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight(request.n or 1)
return await self.online_renderer.render_chat(request)
@@ -102,11 +102,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
if error_check_ret is not None:
return error_check_ret
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight(request.n or 1)
return await self.online_renderer.render_completion(request)
+1 -5
View File
@@ -350,11 +350,7 @@ class OpenAIServingResponses(GenerateBaseServing):
if maybe_validation_error is not None:
return maybe_validation_error
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
if request.store and not self.enable_store:
# Disable the store option.
@@ -110,11 +110,7 @@ class ServingTokens(GenerateBaseServing):
logger.error("Error with model %s", error_check_ret)
return error_check_ret
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
lora_request = None
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
@@ -26,6 +26,7 @@ def create_error_response(
)
from vllm.exceptions import (
GracefulHTTPError,
VLLMClientError,
VLLMNotFoundError,
VLLMServerError,
@@ -33,7 +34,11 @@ def create_error_response(
VLLMValidationError,
)
if isinstance(exc, VLLMValidationError):
if isinstance(exc, GracefulHTTPError):
err_type = HTTPStatus(exc.http_status).phrase
status_code = exc.http_status
param = None
elif isinstance(exc, VLLMValidationError):
err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST
param = exc.parameter
@@ -440,11 +440,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
if not request.model:
request.model = self.models.model_name()
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
if request.response_format not in [
"text",
+45
View File
@@ -125,3 +125,48 @@ class GenerationError(VLLMServerError):
def __init__(self, message: str = "Internal server error"):
super().__init__(message)
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
class GracefulHTTPError(VLLMError):
"""Exception that should be translated into an HTTP error response.
These are expected to occur during normal operation (e.g. admission
control rejections) and should be surfaced to the client with the
explicit HTTP status code they carry, rather than being mapped to a
generic 4xx/5xx by the client/server split.
"""
def __init__(self, message: str, http_status: HTTPStatus):
super().__init__(message)
self.message = message
self.http_status = http_status
class QueueOverflowError(GracefulHTTPError):
"""Raised when admitting a request would exceed the request queue limit.
Returns HTTP 503 (Service Unavailable) so that load balancers and
client SDKs retry the request on a different instance.
"""
def __init__(self):
super().__init__(
"The engine is currently busy and cannot accept new requests. "
"Please try again later or on a different instance.",
HTTPStatus.SERVICE_UNAVAILABLE,
)
class MaxQueuedTokensError(GracefulHTTPError):
"""Raised when the pending prefill tokens exceed the configured limit.
Returns HTTP 503 (Service Unavailable) so that load balancers and
client SDKs retry the request on a different instance.
"""
def __init__(self):
super().__init__(
"The engine has reached its prefill token backlog limit. "
"Please try again later or on a different instance.",
HTTPStatus.SERVICE_UNAVAILABLE,
)
+80 -6
View File
@@ -21,7 +21,13 @@ from vllm.distributed.weight_transfer.base import (
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
from vllm.exceptions import VLLMClientError, VLLMValidationError
from vllm.exceptions import (
GracefulHTTPError,
MaxQueuedTokensError,
QueueOverflowError,
VLLMClientError,
VLLMValidationError,
)
from vllm.inputs import EngineInput, PromptType
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -112,6 +118,7 @@ class AsyncLLM(EngineClient):
self.vllm_config = vllm_config
self._elastic_ep_lock = asyncio.Lock()
self.model_config = vllm_config.model_config
self.scheduler_config = vllm_config.scheduler_config
self.observability_config = vllm_config.observability_config
tracing_endpoint = self.observability_config.otlp_traces_endpoint
@@ -273,6 +280,64 @@ class AsyncLLM(EngineClient):
if handler is not None:
cancel_task_threadsafe(handler)
def get_num_unfinished_requests(self) -> int:
return self.output_processor.get_num_unfinished_requests()
def get_num_queued_tokens(self) -> int:
return self.output_processor.get_num_queued_tokens()
def check_admission(self, n: int = 1, request_id: str | None = None) -> None:
"""Reject the request if it would exceed queue limits.
Both limits return HTTP 503 (Service Unavailable) so that load
balancers and client SDKs retry on a different instance.
- ``max_num_queued_reqs``: hard cap on the number of unfinished requests
(waiting + running). A request with ``n > 1`` counts as ``n`` slots.
- ``max_num_queued_tokens``: TTFT QoS — cap on the total prompt
tokens of requests still in prefill.
Note: ``get_num_queued_tokens`` uses ``prompt_len`` for all prefilling requests.
Chunked prefill progress and prefix-cache hits are not subtracted because the
scheduler's ``num_computed_tokens`` and ``num_cached_tokens`` are only
propagated to the API server after prefill completes. The overestimation is
conservative — earlier rejection, preserving TTFT targets.
Args:
n: Number of sequences the request will occupy.
request_id: Request id, used for logging only.
Raises:
QueueOverflowError: If ``max_num_queued_reqs`` would be exceeded.
MaxQueuedTokensError: If ``max_num_queued_tokens`` would be exceeded.
"""
max_num_reqs = self.scheduler_config.max_num_queued_reqs
if max_num_reqs is not None:
current = self.get_num_unfinished_requests()
if current + n > max_num_reqs:
logger.info(
"Request queue full - rejecting request %s "
"(current=%d, n=%d, max=%d).",
request_id,
current,
n,
max_num_reqs,
)
raise QueueOverflowError()
max_queued_tokens = self.scheduler_config.max_num_queued_tokens
if max_queued_tokens is not None:
current_tokens = self.get_num_queued_tokens()
if current_tokens >= max_queued_tokens:
logger.info(
"Max queued tokens reached - rejecting request %s "
"(current_tokens=%d, max=%d).",
request_id,
current_tokens,
max_queued_tokens,
)
raise MaxQueuedTokensError()
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
if not hasattr(self, "_supported_tasks"):
# Cache the result
@@ -317,6 +382,10 @@ class AsyncLLM(EngineClient):
"prompt logprobs"
)
if isinstance(params, SamplingParams) and params.n > 1:
# TODO (NickLucche) Batch check admission check for all n requests
self.check_admission(params.n, request_id)
if isinstance(prompt, AsyncGenerator):
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
raise NotImplementedError
@@ -430,7 +499,12 @@ class AsyncLLM(EngineClient):
index: int,
queue: RequestOutputCollector,
):
# Add the request to OutputProcessor (this process).
if parent_req is None and not self.output_processor.has_request(
request.request_id
):
self.check_admission(request_id=request.request_id)
# Register locally before the first await so concurrent tasks see this request.
self.output_processor.add_request(request, prompt, parent_req, index, queue)
# Add the EngineCoreRequest to EngineCore (separate process).
@@ -630,8 +704,8 @@ class AsyncLLM(EngineClient):
logger.info("Request %s failed (engine dead).", request_id)
raise
# Request validation error.
except VLLMClientError as e:
# Request validation error or admission control rejection.
except (VLLMClientError, GracefulHTTPError) as e:
if self.log_requests:
logger.info("Request %s failed (bad request): %s.", request_id, e)
raise
@@ -907,8 +981,8 @@ class AsyncLLM(EngineClient):
logger.info("Request %s failed (engine dead).", request_id)
raise
# Request validation error.
except VLLMClientError:
# Request validation error or admission control rejection.
except (VLLMClientError, GracefulHTTPError):
if self.log_requests:
logger.info("Request %s failed (bad request).", request_id)
raise
+14
View File
@@ -467,6 +467,20 @@ class OutputProcessor:
def get_num_unfinished_requests(self):
return len(self.request_states)
def has_request(self, request_id: str) -> bool:
return request_id in self.request_states
def get_num_queued_tokens(self) -> int:
"""Total prompt tokens of requests currently in the prefill phase.
Uses ``prompt_len`` rather than remaining prefill work because the
scheduler's ``num_computed_tokens`` is not propagated to the API
server until prefill completes. See ``SchedulerConfig`` docs.
"""
return sum(
req.prompt_len for req in self.request_states.values() if req.is_prefilling
)
def has_unfinished_requests(self) -> bool:
return len(self.request_states) > 0