[Scheduler] Add --max-num-active-seqs to cap RUNNING admission (#56758)

Signed-off-by: Li <chuali@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Chuan (Richard) Li
2026-09-17 10:22:12 +08:00
committed by GitHub
co-authored by Cursor
parent b1f9ebc0c3
commit f730a93d2b
5 changed files with 51 additions and 1 deletions
+16
View File
@@ -193,6 +193,22 @@ def test_get_num_unfinished_requests():
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
def test_max_num_active_seqs_caps_admission():
"""Admission is capped by max_num_active_seqs, not max_num_seqs."""
scheduler = create_scheduler(max_num_seqs=16, max_num_active_seqs=3)
requests = create_requests(num_requests=10)
for request in requests:
scheduler.add_request(request)
output = scheduler.schedule()
assert len(output.scheduled_new_reqs) == 3
assert len(scheduler.running) == 3
assert len(scheduler.waiting) == 7
# Execution capacity is untouched.
assert scheduler.max_num_running_reqs == 16
def _bind_hisparse_connector(scheduler):
connector = object.__new__(HiSparseConnector)
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
+2
View File
@@ -52,6 +52,7 @@ def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int
def create_scheduler(
model: str = "facebook/opt-125m",
max_num_seqs: int = 16,
max_num_active_seqs: int | None = None,
max_num_batched_tokens: int = 8192,
enable_chunked_prefill: bool = True,
enable_prefix_caching: bool = False,
@@ -107,6 +108,7 @@ def create_scheduler(
max_model_len = max_num_batched_tokens
scheduler_config = SchedulerConfig(
max_num_seqs=max_num_seqs,
max_num_active_seqs=max_num_active_seqs,
max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len,
long_prefill_token_threshold=long_prefill_token_threshold,