[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -26,6 +26,7 @@ def create_error_response(
|
||||
)
|
||||
|
||||
from vllm.exceptions import (
|
||||
GracefulHTTPError,
|
||||
VLLMClientError,
|
||||
VLLMNotFoundError,
|
||||
VLLMServerError,
|
||||
@@ -33,7 +34,11 @@ def create_error_response(
|
||||
VLLMValidationError,
|
||||
)
|
||||
|
||||
if isinstance(exc, VLLMValidationError):
|
||||
if isinstance(exc, GracefulHTTPError):
|
||||
err_type = HTTPStatus(exc.http_status).phrase
|
||||
status_code = exc.http_status
|
||||
param = None
|
||||
elif isinstance(exc, VLLMValidationError):
|
||||
err_type = "BadRequestError"
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
param = exc.parameter
|
||||
|
||||
Reference in New Issue
Block a user