[Core] Add max_num_queued_reqs and max_num_queued_tokens for queue size management (#49445)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -367,6 +367,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:
|
||||
"max_num_scheduled_tokens",
|
||||
"kv_cache_memory_bytes",
|
||||
"safetensors_prefetch_block_size",
|
||||
"max_num_queued_tokens",
|
||||
}
|
||||
if name == "max_model_len":
|
||||
kwargs[name]["type"] = human_readable_int_or_auto
|
||||
@@ -544,6 +545,8 @@ class EngineArgs:
|
||||
max_num_scheduled_tokens: int | None = None
|
||||
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
|
||||
max_num_seqs: int | None = None
|
||||
max_num_queued_reqs: int | None = None
|
||||
max_num_queued_tokens: int | None = None
|
||||
max_logprobs: int = ModelConfig.max_logprobs
|
||||
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
|
||||
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
|
||||
@@ -1554,6 +1557,13 @@ class EngineArgs:
|
||||
"default": None,
|
||||
},
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-queued-tokens",
|
||||
**scheduler_kwargs["max_num_queued_tokens"],
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--long-prefill-token-threshold",
|
||||
**scheduler_kwargs["long_prefill_token_threshold"],
|
||||
@@ -2348,6 +2358,8 @@ class EngineArgs:
|
||||
max_num_batched_tokens=self.max_num_batched_tokens,
|
||||
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
|
||||
max_num_seqs=self.max_num_seqs,
|
||||
max_num_queued_reqs=self.max_num_queued_reqs,
|
||||
max_num_queued_tokens=self.max_num_queued_tokens,
|
||||
max_model_len=model_config.max_model_len,
|
||||
enable_chunked_prefill=self.enable_chunked_prefill,
|
||||
disable_chunked_mm_input=self.disable_chunked_mm_input,
|
||||
|
||||
@@ -62,6 +62,23 @@ class EngineClient(ABC):
|
||||
@abstractmethod
|
||||
def dead_error(self) -> BaseException: ...
|
||||
|
||||
def check_admission( # noqa: B027
|
||||
self, n: int = 1, request_id: str | None = None
|
||||
) -> None:
|
||||
"""Reject the request up front if it would exceed queue limits.
|
||||
|
||||
Called before a response is started so that overload rejections can
|
||||
carry an HTTP status, which is not possible once a streaming response
|
||||
has begun. Engines without admission control accept everything.
|
||||
|
||||
Args:
|
||||
n: Number of sequences the request will occupy.
|
||||
request_id: Request id, used for logging only.
|
||||
|
||||
Raises:
|
||||
GracefulHTTPError: If the request cannot be admitted.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def generate(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user