[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+45
View File
@@ -125,3 +125,48 @@ class GenerationError(VLLMServerError):
def __init__(self, message: str = "Internal server error"):
super().__init__(message)
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
class GracefulHTTPError(VLLMError):
"""Exception that should be translated into an HTTP error response.
These are expected to occur during normal operation (e.g. admission
control rejections) and should be surfaced to the client with the
explicit HTTP status code they carry, rather than being mapped to a
generic 4xx/5xx by the client/server split.
"""
def __init__(self, message: str, http_status: HTTPStatus):
super().__init__(message)
self.message = message
self.http_status = http_status
class QueueOverflowError(GracefulHTTPError):
"""Raised when admitting a request would exceed the request queue limit.
Returns HTTP 503 (Service Unavailable) so that load balancers and
client SDKs retry the request on a different instance.
"""
def __init__(self):
super().__init__(
"The engine is currently busy and cannot accept new requests. "
"Please try again later or on a different instance.",
HTTPStatus.SERVICE_UNAVAILABLE,
)
class MaxQueuedTokensError(GracefulHTTPError):
"""Raised when the pending prefill tokens exceed the configured limit.
Returns HTTP 503 (Service Unavailable) so that load balancers and
client SDKs retry the request on a different instance.
"""
def __init__(self):
super().__init__(
"The engine has reached its prefill token backlog limit. "
"Please try again later or on a different instance.",
HTTPStatus.SERVICE_UNAVAILABLE,
)