[Scheduler] Add --max-num-active-seqs to cap RUNNING admission (#56758)

Signed-off-by: Li <chuali@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Chuan (Richard) Li
2026-09-17 10:22:12 +08:00
committed by GitHub
co-authored by Cursor
parent b1f9ebc0c3
commit f730a93d2b
5 changed files with 51 additions and 1 deletions
+2
View File
@@ -52,6 +52,7 @@ def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int
def create_scheduler(
model: str = "facebook/opt-125m",
max_num_seqs: int = 16,
max_num_active_seqs: int | None = None,
max_num_batched_tokens: int = 8192,
enable_chunked_prefill: bool = True,
enable_prefix_caching: bool = False,
@@ -107,6 +108,7 @@ def create_scheduler(
max_model_len = max_num_batched_tokens
scheduler_config = SchedulerConfig(
max_num_seqs=max_num_seqs,
max_num_active_seqs=max_num_active_seqs,
max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len,
long_prefill_token_threshold=long_prefill_token_threshold,