[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+17
View File
@@ -62,6 +62,23 @@ class EngineClient(ABC):
@abstractmethod
def dead_error(self) -> BaseException: ...
def check_admission( # noqa: B027
self, n: int = 1, request_id: str | None = None
) -> None:
"""Reject the request up front if it would exceed queue limits.
Called before a response is started so that overload rejections can
carry an HTTP status, which is not possible once a streaming response
has begun. Engines without admission control accept everything.
Args:
n: Number of sequences the request will occupy.
request_id: Request id, used for logging only.
Raises:
GracefulHTTPError: If the request cannot be admitted.
"""
@abstractmethod
def generate(
self,