[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+13
View File
@@ -171,6 +171,19 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
# Never fail server startup over the fingerprint.
self.system_fingerprint = None
def _preflight(self, n: int = 1) -> None:
"""Engine checks that must run before a response is started.
This is required for the streaming case, where we return a success
status before we actually start generating text :).
Args:
n: Number of sequences the request will occupy.
"""
if self.engine_client.errored:
raise self.engine_client.dead_error
self.engine_client.check_admission(n)
def create_streaming_error_response(
self,
message: str | Exception,
@@ -195,6 +195,7 @@ class ServingGenerativeScoring(BaseServing):
# Check if engine is alive
if self.engine_client.errored:
raise self.engine_client.dead_error
self.engine_client.check_admission(len(request.items))
# Get tokenizer
tokenizer = self.renderer.tokenizer
@@ -56,8 +56,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
logger.error("Error with model %s", error_check_ret)
return error_check_ret
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
renderer = self.online_renderer
@@ -235,11 +235,7 @@ class OpenAIServingChat(GenerateBaseServing):
logger.error("Error with model %s", error_check_ret)
return error_check_ret
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight(request.n or 1)
return await self.online_renderer.render_chat(request)
@@ -102,11 +102,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
if error_check_ret is not None:
return error_check_ret
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight(request.n or 1)
return await self.online_renderer.render_completion(request)
+1 -5
View File
@@ -350,11 +350,7 @@ class OpenAIServingResponses(GenerateBaseServing):
if maybe_validation_error is not None:
return maybe_validation_error
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
if request.store and not self.enable_store:
# Disable the store option.
@@ -110,11 +110,7 @@ class ServingTokens(GenerateBaseServing):
logger.error("Error with model %s", error_check_ret)
return error_check_ret
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
lora_request = None
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
@@ -26,6 +26,7 @@ def create_error_response(
)
from vllm.exceptions import (
GracefulHTTPError,
VLLMClientError,
VLLMNotFoundError,
VLLMServerError,
@@ -33,7 +34,11 @@ def create_error_response(
VLLMValidationError,
)
if isinstance(exc, VLLMValidationError):
if isinstance(exc, GracefulHTTPError):
err_type = HTTPStatus(exc.http_status).phrase
status_code = exc.http_status
param = None
elif isinstance(exc, VLLMValidationError):
err_type = "BadRequestError"
status_code = HTTPStatus.BAD_REQUEST
param = exc.parameter
@@ -440,11 +440,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
if not request.model:
request.model = self.models.model_name()
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
self._preflight()
if request.response_format not in [
"text",