[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -71,6 +71,37 @@ class SchedulerConfig:
|
||||
"""For chunked prefill, a request is considered long if the prompt is
|
||||
longer than this number of tokens. 0 disables the cap (default)."""
|
||||
|
||||
max_num_queued_reqs: int | None = Field(default=None, ge=0)
|
||||
"""Maximum number of requests that can be in-flight (waiting or running)
|
||||
at the same time, or None for no limit. When the limit is reached, new
|
||||
requests are rejected with HTTP 503 so the client can retry on another
|
||||
instance. This bounds vLLM's otherwise unbounded request queue and is
|
||||
primarily a coarse capacity valve."""
|
||||
|
||||
max_num_queued_tokens: int | None = Field(default=None, ge=0)
|
||||
"""Maximum total prompt tokens of requests currently in the prefill
|
||||
phase, or None for no limit. When the limit is reached, new requests
|
||||
are rejected with HTTP 503.
|
||||
|
||||
This is a TTFT QoS mechanism: by setting it to
|
||||
``target_TTFT * prefill_throughput`` you reject requests when the
|
||||
prefill backlog would exceed the latency target. In a disaggregated
|
||||
prefill-decode setup this maps directly to the prefill pool's
|
||||
capacity.
|
||||
|
||||
Note: the count is conservative. A partially prefilled request
|
||||
still contributes its full ``prompt_len`` until it transitions out
|
||||
of the prefill phase, because the scheduler's per-iteration
|
||||
``num_computed_tokens`` progress is not propagated to the API
|
||||
server process during prefill (``EngineCoreOutput`` is only
|
||||
emitted once the request starts producing tokens). Similarly,
|
||||
prefix-cache hits (``num_cached_tokens``) are only known to the
|
||||
OutputProcessor after prefill completes. This overestimates the
|
||||
real backlog, causing earlier rejection than strictly necessary
|
||||
— the safe direction for QoS. The impact is limited to long
|
||||
prompts under chunked prefill; short prompts that prefill in a
|
||||
single iteration are unaffected."""
|
||||
|
||||
enable_chunked_prefill: bool = True
|
||||
"""If True, prefill requests can be chunked based
|
||||
on the remaining `max_num_batched_tokens`.
|
||||
|
||||
@@ -367,6 +367,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
|
||||
"max_num_scheduled_tokens",
|
||||
"kv_cache_memory_bytes",
|
||||
"safetensors_prefetch_block_size",
|
||||
"max_num_queued_tokens",
|
||||
}
|
||||
if name == "max_model_len":
|
||||
kwargs[name]["type"] = human_readable_int_or_auto
|
||||
@@ -544,6 +545,8 @@ class EngineArgs:
|
||||
max_num_scheduled_tokens: int | None = None
|
||||
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
|
||||
max_num_seqs: int | None = None
|
||||
max_num_queued_reqs: int | None = None
|
||||
max_num_queued_tokens: int | None = None
|
||||
max_logprobs: int = ModelConfig.max_logprobs
|
||||
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
|
||||
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
|
||||
@@ -1554,6 +1557,13 @@ class EngineArgs:
|
||||
"default": None,
|
||||
},
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-queued-tokens",
|
||||
**scheduler_kwargs["max_num_queued_tokens"],
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--long-prefill-token-threshold",
|
||||
**scheduler_kwargs["long_prefill_token_threshold"],
|
||||
@@ -2348,6 +2358,8 @@ class EngineArgs:
|
||||
max_num_batched_tokens=self.max_num_batched_tokens,
|
||||
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
|
||||
max_num_seqs=self.max_num_seqs,
|
||||
max_num_queued_reqs=self.max_num_queued_reqs,
|
||||
max_num_queued_tokens=self.max_num_queued_tokens,
|
||||
max_model_len=model_config.max_model_len,
|
||||
enable_chunked_prefill=self.enable_chunked_prefill,
|
||||
disable_chunked_mm_input=self.disable_chunked_mm_input,
|
||||
|
||||
@@ -62,6 +62,23 @@ class EngineClient(ABC):
|
||||
@abstractmethod
|
||||
def dead_error(self) -> BaseException: ...
|
||||
|
||||
def check_admission( # noqa: B027
|
||||
self, n: int = 1, request_id: str | None = None
|
||||
) -> None:
|
||||
"""Reject the request up front if it would exceed queue limits.
|
||||
|
||||
Called before a response is started so that overload rejections can
|
||||
carry an HTTP status, which is not possible once a streaming response
|
||||
has begun. Engines without admission control accept everything.
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
request_id: Request id, used for logging only.
|
||||
|
||||
Raises:
|
||||
GracefulHTTPError: If the request cannot be admitted.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def generate(
|
||||
self,
|
||||
|
||||
@@ -171,6 +171,19 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
# Never fail server startup over the fingerprint.
|
||||
self.system_fingerprint = None
|
||||
|
||||
def _preflight(self, n: int = 1) -> None:
|
||||
"""Engine checks that must run before a response is started.
|
||||
|
||||
This is required for the streaming case, where we return a success
|
||||
status before we actually start generating text :).
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
"""
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self.engine_client.check_admission(n)
|
||||
|
||||
def create_streaming_error_response(
|
||||
self,
|
||||
message: str | Exception,
|
||||
|
||||
@@ -195,6 +195,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
# Check if engine is alive
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self.engine_client.check_admission(len(request.items))
|
||||
|
||||
# Get tokenizer
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
@@ -56,8 +56,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
renderer = self.online_renderer
|
||||
|
||||
|
||||
@@ -235,11 +235,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight(request.n or 1)
|
||||
|
||||
return await self.online_renderer.render_chat(request)
|
||||
|
||||
|
||||
@@ -102,11 +102,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight(request.n or 1)
|
||||
|
||||
return await self.online_renderer.render_completion(request)
|
||||
|
||||
|
||||
@@ -350,11 +350,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
if maybe_validation_error is not None:
|
||||
return maybe_validation_error
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
if request.store and not self.enable_store:
|
||||
# Disable the store option.
|
||||
|
||||
@@ -110,11 +110,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
lora_request = None
|
||||
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
|
||||
|
||||
@@ -26,6 +26,7 @@ def create_error_response(
|
||||
)
|
||||
|
||||
from vllm.exceptions import (
|
||||
GracefulHTTPError,
|
||||
VLLMClientError,
|
||||
VLLMNotFoundError,
|
||||
VLLMServerError,
|
||||
@@ -33,7 +34,11 @@ def create_error_response(
|
||||
VLLMValidationError,
|
||||
)
|
||||
|
||||
if isinstance(exc, VLLMValidationError):
|
||||
if isinstance(exc, GracefulHTTPError):
|
||||
err_type = HTTPStatus(exc.http_status).phrase
|
||||
status_code = exc.http_status
|
||||
param = None
|
||||
elif isinstance(exc, VLLMValidationError):
|
||||
err_type = "BadRequestError"
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
param = exc.parameter
|
||||
|
||||
@@ -440,11 +440,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
if not request.model:
|
||||
request.model = self.models.model_name()
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
if request.response_format not in [
|
||||
"text",
|
||||
|
||||
@@ -125,3 +125,48 @@ class GenerationError(VLLMServerError):
|
||||
def __init__(self, message: str = "Internal server error"):
|
||||
super().__init__(message)
|
||||
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
|
||||
|
||||
class GracefulHTTPError(VLLMError):
|
||||
"""Exception that should be translated into an HTTP error response.
|
||||
|
||||
These are expected to occur during normal operation (e.g. admission
|
||||
control rejections) and should be surfaced to the client with the
|
||||
explicit HTTP status code they carry, rather than being mapped to a
|
||||
generic 4xx/5xx by the client/server split.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, http_status: HTTPStatus):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.http_status = http_status
|
||||
|
||||
|
||||
class QueueOverflowError(GracefulHTTPError):
|
||||
"""Raised when admitting a request would exceed the request queue limit.
|
||||
|
||||
Returns HTTP 503 (Service Unavailable) so that load balancers and
|
||||
client SDKs retry the request on a different instance.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"The engine is currently busy and cannot accept new requests. "
|
||||
"Please try again later or on a different instance.",
|
||||
HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
|
||||
class MaxQueuedTokensError(GracefulHTTPError):
|
||||
"""Raised when the pending prefill tokens exceed the configured limit.
|
||||
|
||||
Returns HTTP 503 (Service Unavailable) so that load balancers and
|
||||
client SDKs retry the request on a different instance.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"The engine has reached its prefill token backlog limit. "
|
||||
"Please try again later or on a different instance.",
|
||||
HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
@@ -21,7 +21,13 @@ from vllm.distributed.weight_transfer.base import (
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient, StreamingInput
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
|
||||
from vllm.exceptions import VLLMClientError, VLLMValidationError
|
||||
from vllm.exceptions import (
|
||||
GracefulHTTPError,
|
||||
MaxQueuedTokensError,
|
||||
QueueOverflowError,
|
||||
VLLMClientError,
|
||||
VLLMValidationError,
|
||||
)
|
||||
from vllm.inputs import EngineInput, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -112,6 +118,7 @@ class AsyncLLM(EngineClient):
|
||||
self.vllm_config = vllm_config
|
||||
self._elastic_ep_lock = asyncio.Lock()
|
||||
self.model_config = vllm_config.model_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
|
||||
tracing_endpoint = self.observability_config.otlp_traces_endpoint
|
||||
@@ -273,6 +280,64 @@ class AsyncLLM(EngineClient):
|
||||
if handler is not None:
|
||||
cancel_task_threadsafe(handler)
|
||||
|
||||
def get_num_unfinished_requests(self) -> int:
|
||||
return self.output_processor.get_num_unfinished_requests()
|
||||
|
||||
def get_num_queued_tokens(self) -> int:
|
||||
return self.output_processor.get_num_queued_tokens()
|
||||
|
||||
def check_admission(self, n: int = 1, request_id: str | None = None) -> None:
|
||||
"""Reject the request if it would exceed queue limits.
|
||||
|
||||
Both limits return HTTP 503 (Service Unavailable) so that load
|
||||
balancers and client SDKs retry on a different instance.
|
||||
|
||||
- ``max_num_queued_reqs``: hard cap on the number of unfinished requests
|
||||
(waiting + running). A request with ``n > 1`` counts as ``n`` slots.
|
||||
- ``max_num_queued_tokens``: TTFT QoS — cap on the total prompt
|
||||
tokens of requests still in prefill.
|
||||
|
||||
Note: ``get_num_queued_tokens`` uses ``prompt_len`` for all prefilling requests.
|
||||
Chunked prefill progress and prefix-cache hits are not subtracted because the
|
||||
scheduler's ``num_computed_tokens`` and ``num_cached_tokens`` are only
|
||||
propagated to the API server after prefill completes. The overestimation is
|
||||
conservative — earlier rejection, preserving TTFT targets.
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
request_id: Request id, used for logging only.
|
||||
|
||||
Raises:
|
||||
QueueOverflowError: If ``max_num_queued_reqs`` would be exceeded.
|
||||
MaxQueuedTokensError: If ``max_num_queued_tokens`` would be exceeded.
|
||||
"""
|
||||
max_num_reqs = self.scheduler_config.max_num_queued_reqs
|
||||
if max_num_reqs is not None:
|
||||
current = self.get_num_unfinished_requests()
|
||||
if current + n > max_num_reqs:
|
||||
logger.info(
|
||||
"Request queue full - rejecting request %s "
|
||||
"(current=%d, n=%d, max=%d).",
|
||||
request_id,
|
||||
current,
|
||||
n,
|
||||
max_num_reqs,
|
||||
)
|
||||
raise QueueOverflowError()
|
||||
|
||||
max_queued_tokens = self.scheduler_config.max_num_queued_tokens
|
||||
if max_queued_tokens is not None:
|
||||
current_tokens = self.get_num_queued_tokens()
|
||||
if current_tokens >= max_queued_tokens:
|
||||
logger.info(
|
||||
"Max queued tokens reached - rejecting request %s "
|
||||
"(current_tokens=%d, max=%d).",
|
||||
request_id,
|
||||
current_tokens,
|
||||
max_queued_tokens,
|
||||
)
|
||||
raise MaxQueuedTokensError()
|
||||
|
||||
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
if not hasattr(self, "_supported_tasks"):
|
||||
# Cache the result
|
||||
@@ -317,6 +382,10 @@ class AsyncLLM(EngineClient):
|
||||
"prompt logprobs"
|
||||
)
|
||||
|
||||
if isinstance(params, SamplingParams) and params.n > 1:
|
||||
# TODO (NickLucche) Batch check admission check for all n requests
|
||||
self.check_admission(params.n, request_id)
|
||||
|
||||
if isinstance(prompt, AsyncGenerator):
|
||||
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
|
||||
raise NotImplementedError
|
||||
@@ -430,7 +499,12 @@ class AsyncLLM(EngineClient):
|
||||
index: int,
|
||||
queue: RequestOutputCollector,
|
||||
):
|
||||
# Add the request to OutputProcessor (this process).
|
||||
if parent_req is None and not self.output_processor.has_request(
|
||||
request.request_id
|
||||
):
|
||||
self.check_admission(request_id=request.request_id)
|
||||
|
||||
# Register locally before the first await so concurrent tasks see this request.
|
||||
self.output_processor.add_request(request, prompt, parent_req, index, queue)
|
||||
|
||||
# Add the EngineCoreRequest to EngineCore (separate process).
|
||||
@@ -630,8 +704,8 @@ class AsyncLLM(EngineClient):
|
||||
logger.info("Request %s failed (engine dead).", request_id)
|
||||
raise
|
||||
|
||||
# Request validation error.
|
||||
except VLLMClientError as e:
|
||||
# Request validation error or admission control rejection.
|
||||
except (VLLMClientError, GracefulHTTPError) as e:
|
||||
if self.log_requests:
|
||||
logger.info("Request %s failed (bad request): %s.", request_id, e)
|
||||
raise
|
||||
@@ -907,8 +981,8 @@ class AsyncLLM(EngineClient):
|
||||
logger.info("Request %s failed (engine dead).", request_id)
|
||||
raise
|
||||
|
||||
# Request validation error.
|
||||
except VLLMClientError:
|
||||
# Request validation error or admission control rejection.
|
||||
except (VLLMClientError, GracefulHTTPError):
|
||||
if self.log_requests:
|
||||
logger.info("Request %s failed (bad request).", request_id)
|
||||
raise
|
||||
|
||||
@@ -467,6 +467,20 @@ class OutputProcessor:
|
||||
def get_num_unfinished_requests(self):
|
||||
return len(self.request_states)
|
||||
|
||||
def has_request(self, request_id: str) -> bool:
|
||||
return request_id in self.request_states
|
||||
|
||||
def get_num_queued_tokens(self) -> int:
|
||||
"""Total prompt tokens of requests currently in the prefill phase.
|
||||
|
||||
Uses ``prompt_len`` rather than remaining prefill work because the
|
||||
scheduler's ``num_computed_tokens`` is not propagated to the API
|
||||
server until prefill completes. See ``SchedulerConfig`` docs.
|
||||
"""
|
||||
return sum(
|
||||
req.prompt_len for req in self.request_states.values() if req.is_prefilling
|
||||
)
|
||||
|
||||
def has_unfinished_requests(self) -> bool:
|
||||
return len(self.request_states) > 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user