[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -62,6 +62,23 @@ class EngineClient(ABC):
|
||||
@abstractmethod
|
||||
def dead_error(self) -> BaseException: ...
|
||||
|
||||
def check_admission( # noqa: B027
|
||||
self, n: int = 1, request_id: str | None = None
|
||||
) -> None:
|
||||
"""Reject the request up front if it would exceed queue limits.
|
||||
|
||||
Called before a response is started so that overload rejections can
|
||||
carry an HTTP status, which is not possible once a streaming response
|
||||
has begun. Engines without admission control accept everything.
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
request_id: Request id, used for logging only.
|
||||
|
||||
Raises:
|
||||
GracefulHTTPError: If the request cannot be admitted.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def generate(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user