[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+13
View File
@@ -171,6 +171,19 @@ class GenerateBaseServing(BaseServing, BeamSearchOnlineMixin):
# Never fail server startup over the fingerprint.
self.system_fingerprint = None
def _preflight(self, n: int = 1) -> None:
"""Engine checks that must run before a response is started.
This is required for the streaming case, where we return a success
status before we actually start generating text :).
Args:
n: Number of sequences the request will occupy.
"""
if self.engine_client.errored:
raise self.engine_client.dead_error
self.engine_client.check_admission(n)
def create_streaming_error_response(
self,
message: str | Exception,
@@ -195,6 +195,7 @@ class ServingGenerativeScoring(BaseServing):
# Check if engine is alive
if self.engine_client.errored:
raise self.engine_client.dead_error
self.engine_client.check_admission(len(request.items))
# Get tokenizer
tokenizer = self.renderer.tokenizer