[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -171,6 +171,19 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
|
||||
# Never fail server startup over the fingerprint.
|
||||
self.system_fingerprint = None
|
||||
|
||||
def _preflight(self, n: int = 1) -> None:
|
||||
"""Engine checks that must run before a response is started.
|
||||
|
||||
This is required for the streaming case, where we return a success
|
||||
status before we actually start generating text :).
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
"""
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self.engine_client.check_admission(n)
|
||||
|
||||
def create_streaming_error_response(
|
||||
self,
|
||||
message: str | Exception,
|
||||
|
||||
@@ -195,6 +195,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
# Check if engine is alive
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self.engine_client.check_admission(len(request.items))
|
||||
|
||||
# Get tokenizer
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
@@ -56,8 +56,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
renderer = self.online_renderer
|
||||
|
||||
|
||||
@@ -235,11 +235,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight(request.n or 1)
|
||||
|
||||
return await self.online_renderer.render_chat(request)
|
||||
|
||||
|
||||
@@ -102,11 +102,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight(request.n or 1)
|
||||
|
||||
return await self.online_renderer.render_completion(request)
|
||||
|
||||
|
||||
@@ -350,11 +350,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
if maybe_validation_error is not None:
|
||||
return maybe_validation_error
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
if request.store and not self.enable_store:
|
||||
# Disable the store option.
|
||||
|
||||
@@ -110,11 +110,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
lora_request = None
|
||||
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
|
||||
|
||||
@@ -26,6 +26,7 @@ def create_error_response(
|
||||
)
|
||||
|
||||
from vllm.exceptions import (
|
||||
GracefulHTTPError,
|
||||
VLLMClientError,
|
||||
VLLMNotFoundError,
|
||||
VLLMServerError,
|
||||
@@ -33,7 +34,11 @@ def create_error_response(
|
||||
VLLMValidationError,
|
||||
)
|
||||
|
||||
if isinstance(exc, VLLMValidationError):
|
||||
if isinstance(exc, GracefulHTTPError):
|
||||
err_type = HTTPStatus(exc.http_status).phrase
|
||||
status_code = exc.http_status
|
||||
param = None
|
||||
elif isinstance(exc, VLLMValidationError):
|
||||
err_type = "BadRequestError"
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
param = exc.parameter
|
||||
|
||||
@@ -440,11 +440,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
if not request.model:
|
||||
request.model = self.models.model_name()
|
||||
|
||||
# If the engine is dead, raise the engine's DEAD_ERROR.
|
||||
# This is required for the streaming case, where we return a
|
||||
# success status before we actually start generating text :).
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self._preflight()
|
||||
|
||||
if request.response_format not in [
|
||||
"text",
|
||||
|
||||
Reference in New Issue
Block a user