[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
@@ -26,6 +26,7 @@ def create_error_response(
)
from vllm.exceptions import (
GracefulHTTPError,
VLLMClientError,
VLLMNotFoundError,
VLLMServerError,
@@ -33,7 +34,11 @@ def create_error_response(
VLLMValidationError,
)
if isinstance(exc, VLLMValidationError):
if isinstance(exc, GracefulHTTPError):
err_type = HTTPStatus(exc.http_status).phrase
status_code = exc.http_status
param = None
elif isinstance(exc, VLLMValidationError):
err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST
param = exc.parameter