[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -195,6 +195,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
# Check if engine is alive
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
self.engine_client.check_admission(len(request.items))
|
||||
|
||||
# Get tokenizer
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
Reference in New Issue
Block a user