[Scheduler] Add --max-num-active-seqs to cap RUNNING admission (#56758)
Signed-off-by: Li <chuali@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
b1f9ebc0c3
commit
f730a93d2b
@@ -193,6 +193,22 @@ def test_get_num_unfinished_requests():
|
||||
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
|
||||
|
||||
|
||||
def test_max_num_active_seqs_caps_admission():
|
||||
"""Admission is capped by max_num_active_seqs, not max_num_seqs."""
|
||||
scheduler = create_scheduler(max_num_seqs=16, max_num_active_seqs=3)
|
||||
requests = create_requests(num_requests=10)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert len(output.scheduled_new_reqs) == 3
|
||||
assert len(scheduler.running) == 3
|
||||
assert len(scheduler.waiting) == 7
|
||||
# Execution capacity is untouched.
|
||||
assert scheduler.max_num_running_reqs == 16
|
||||
|
||||
|
||||
def _bind_hisparse_connector(scheduler):
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
|
||||
@@ -52,6 +52,7 @@ def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int
|
||||
def create_scheduler(
|
||||
model: str = "facebook/opt-125m",
|
||||
max_num_seqs: int = 16,
|
||||
max_num_active_seqs: int | None = None,
|
||||
max_num_batched_tokens: int = 8192,
|
||||
enable_chunked_prefill: bool = True,
|
||||
enable_prefix_caching: bool = False,
|
||||
@@ -107,6 +108,7 @@ def create_scheduler(
|
||||
max_model_len = max_num_batched_tokens
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_active_seqs=max_num_active_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_model_len=max_model_len,
|
||||
long_prefill_token_threshold=long_prefill_token_threshold,
|
||||
|
||||
Reference in New Issue
Block a user