[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-31 14:26:49 +00:00
committed by GitHub
parent 76ff0cdff2
commit dafbef15a1
18 changed files with 644 additions and 35 deletions
+12
View File
@@ -367,6 +367,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
"max_num_scheduled_tokens",
"kv_cache_memory_bytes",
"safetensors_prefetch_block_size",
"max_num_queued_tokens",
}
if name == "max_model_len":
kwargs[name]["type"] = human_readable_int_or_auto
@@ -544,6 +545,8 @@ class EngineArgs:
max_num_scheduled_tokens: int | None = None
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
max_num_seqs: int | None = None
max_num_queued_reqs: int | None = None
max_num_queued_tokens: int | None = None
max_logprobs: int = ModelConfig.max_logprobs
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
@@ -1554,6 +1557,13 @@ class EngineArgs:
"default": None,
},
)
scheduler_group.add_argument(
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
)
scheduler_group.add_argument(
"--max-num-queued-tokens",
**scheduler_kwargs["max_num_queued_tokens"],
)
scheduler_group.add_argument(
"--long-prefill-token-threshold",
**scheduler_kwargs["long_prefill_token_threshold"],
@@ -2348,6 +2358,8 @@ class EngineArgs:
max_num_batched_tokens=self.max_num_batched_tokens,
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
max_num_seqs=self.max_num_seqs,
max_num_queued_reqs=self.max_num_queued_reqs,
max_num_queued_tokens=self.max_num_queued_tokens,
max_model_len=model_config.max_model_len,
enable_chunked_prefill=self.enable_chunked_prefill,
disable_chunked_mm_input=self.disable_chunked_mm_input,
+17
View File
@@ -62,6 +62,23 @@ class EngineClient(ABC):
@abstractmethod
def dead_error(self) -> BaseException: ...
def check_admission( # noqa: B027
self, n: int = 1, request_id: str | None = None
) -> None:
"""Reject the request up front if it would exceed queue limits.
Called before a response is started so that overload rejections can
carry an HTTP status, which is not possible once a streaming response
has begun. Engines without admission control accept everything.
Args:
n: Number of sequences the request will occupy.
request_id: Request id, used for logging only.
Raises:
GracefulHTTPError: If the request cannot be admitted.
"""
@abstractmethod
def generate(
self,