[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -21,7 +21,13 @@ from vllm.distributed.weight_transfer.base import (
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient, StreamingInput
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
|
||||
from vllm.exceptions import VLLMClientError, VLLMValidationError
|
||||
from vllm.exceptions import (
|
||||
GracefulHTTPError,
|
||||
MaxQueuedTokensError,
|
||||
QueueOverflowError,
|
||||
VLLMClientError,
|
||||
VLLMValidationError,
|
||||
)
|
||||
from vllm.inputs import EngineInput, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -112,6 +118,7 @@ class AsyncLLM(EngineClient):
|
||||
self.vllm_config = vllm_config
|
||||
self._elastic_ep_lock = asyncio.Lock()
|
||||
self.model_config = vllm_config.model_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
|
||||
tracing_endpoint = self.observability_config.otlp_traces_endpoint
|
||||
@@ -273,6 +280,64 @@ class AsyncLLM(EngineClient):
|
||||
if handler is not None:
|
||||
cancel_task_threadsafe(handler)
|
||||
|
||||
def get_num_unfinished_requests(self) -> int:
|
||||
return self.output_processor.get_num_unfinished_requests()
|
||||
|
||||
def get_num_queued_tokens(self) -> int:
|
||||
return self.output_processor.get_num_queued_tokens()
|
||||
|
||||
def check_admission(self, n: int = 1, request_id: str | None = None) -> None:
|
||||
"""Reject the request if it would exceed queue limits.
|
||||
|
||||
Both limits return HTTP 503 (Service Unavailable) so that load
|
||||
balancers and client SDKs retry on a different instance.
|
||||
|
||||
- ``max_num_queued_reqs``: hard cap on the number of unfinished requests
|
||||
(waiting + running). A request with ``n > 1`` counts as ``n`` slots.
|
||||
- ``max_num_queued_tokens``: TTFT QoS — cap on the total prompt
|
||||
tokens of requests still in prefill.
|
||||
|
||||
Note: ``get_num_queued_tokens`` uses ``prompt_len`` for all prefilling requests.
|
||||
Chunked prefill progress and prefix-cache hits are not subtracted because the
|
||||
scheduler's ``num_computed_tokens`` and ``num_cached_tokens`` are only
|
||||
propagated to the API server after prefill completes. The overestimation is
|
||||
conservative — earlier rejection, preserving TTFT targets.
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
request_id: Request id, used for logging only.
|
||||
|
||||
Raises:
|
||||
QueueOverflowError: If ``max_num_queued_reqs`` would be exceeded.
|
||||
MaxQueuedTokensError: If ``max_num_queued_tokens`` would be exceeded.
|
||||
"""
|
||||
max_num_reqs = self.scheduler_config.max_num_queued_reqs
|
||||
if max_num_reqs is not None:
|
||||
current = self.get_num_unfinished_requests()
|
||||
if current + n > max_num_reqs:
|
||||
logger.info(
|
||||
"Request queue full - rejecting request %s "
|
||||
"(current=%d, n=%d, max=%d).",
|
||||
request_id,
|
||||
current,
|
||||
n,
|
||||
max_num_reqs,
|
||||
)
|
||||
raise QueueOverflowError()
|
||||
|
||||
max_queued_tokens = self.scheduler_config.max_num_queued_tokens
|
||||
if max_queued_tokens is not None:
|
||||
current_tokens = self.get_num_queued_tokens()
|
||||
if current_tokens >= max_queued_tokens:
|
||||
logger.info(
|
||||
"Max queued tokens reached - rejecting request %s "
|
||||
"(current_tokens=%d, max=%d).",
|
||||
request_id,
|
||||
current_tokens,
|
||||
max_queued_tokens,
|
||||
)
|
||||
raise MaxQueuedTokensError()
|
||||
|
||||
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
if not hasattr(self, "_supported_tasks"):
|
||||
# Cache the result
|
||||
@@ -317,6 +382,10 @@ class AsyncLLM(EngineClient):
|
||||
"prompt logprobs"
|
||||
)
|
||||
|
||||
if isinstance(params, SamplingParams) and params.n > 1:
|
||||
# TODO (NickLucche) Batch check admission check for all n requests
|
||||
self.check_admission(params.n, request_id)
|
||||
|
||||
if isinstance(prompt, AsyncGenerator):
|
||||
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
|
||||
raise NotImplementedError
|
||||
@@ -430,7 +499,12 @@ class AsyncLLM(EngineClient):
|
||||
index: int,
|
||||
queue: RequestOutputCollector,
|
||||
):
|
||||
# Add the request to OutputProcessor (this process).
|
||||
if parent_req is None and not self.output_processor.has_request(
|
||||
request.request_id
|
||||
):
|
||||
self.check_admission(request_id=request.request_id)
|
||||
|
||||
# Register locally before the first await so concurrent tasks see this request.
|
||||
self.output_processor.add_request(request, prompt, parent_req, index, queue)
|
||||
|
||||
# Add the EngineCoreRequest to EngineCore (separate process).
|
||||
@@ -630,8 +704,8 @@ class AsyncLLM(EngineClient):
|
||||
logger.info("Request %s failed (engine dead).", request_id)
|
||||
raise
|
||||
|
||||
# Request validation error.
|
||||
except VLLMClientError as e:
|
||||
# Request validation error or admission control rejection.
|
||||
except (VLLMClientError, GracefulHTTPError) as e:
|
||||
if self.log_requests:
|
||||
logger.info("Request %s failed (bad request): %s.", request_id, e)
|
||||
raise
|
||||
@@ -907,8 +981,8 @@ class AsyncLLM(EngineClient):
|
||||
logger.info("Request %s failed (engine dead).", request_id)
|
||||
raise
|
||||
|
||||
# Request validation error.
|
||||
except VLLMClientError:
|
||||
# Request validation error or admission control rejection.
|
||||
except (VLLMClientError, GracefulHTTPError):
|
||||
if self.log_requests:
|
||||
logger.info("Request %s failed (bad request).", request_id)
|
||||
raise
|
||||
|
||||
@@ -467,6 +467,20 @@ class OutputProcessor:
|
||||
def get_num_unfinished_requests(self):
|
||||
return len(self.request_states)
|
||||
|
||||
def has_request(self, request_id: str) -> bool:
|
||||
return request_id in self.request_states
|
||||
|
||||
def get_num_queued_tokens(self) -> int:
|
||||
"""Total prompt tokens of requests currently in the prefill phase.
|
||||
|
||||
Uses ``prompt_len`` rather than remaining prefill work because the
|
||||
scheduler's ``num_computed_tokens`` is not propagated to the API
|
||||
server until prefill completes. See ``SchedulerConfig`` docs.
|
||||
"""
|
||||
return sum(
|
||||
req.prompt_len for req in self.request_states.values() if req.is_prefilling
|
||||
)
|
||||
|
||||
def has_unfinished_requests(self) -> bool:
|
||||
return len(self.request_states) > 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user