[Scheduler] Add --max-num-active-seqs to cap RUNNING admission (#56758)

Signed-off-by: Li <chuali@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Chuan (Richard) Li
2026-09-17 10:22:12 +08:00
committed by GitHub
co-authored by Cursor
parent b1f9ebc0c3
commit f730a93d2b
5 changed files with 51 additions and 1 deletions
+16
View File
@@ -193,6 +193,22 @@ def test_get_num_unfinished_requests():
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
def test_max_num_active_seqs_caps_admission():
"""Admission is capped by max_num_active_seqs, not max_num_seqs."""
scheduler = create_scheduler(max_num_seqs=16, max_num_active_seqs=3)
requests = create_requests(num_requests=10)
for request in requests:
scheduler.add_request(request)
output = scheduler.schedule()
assert len(output.scheduled_new_reqs) == 3
assert len(scheduler.running) == 3
assert len(scheduler.waiting) == 7
# Execution capacity is untouched.
assert scheduler.max_num_running_reqs == 16
def _bind_hisparse_connector(scheduler):
connector = object.__new__(HiSparseConnector)
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
+2
View File
@@ -52,6 +52,7 @@ def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int
def create_scheduler(
model: str = "facebook/opt-125m",
max_num_seqs: int = 16,
max_num_active_seqs: int | None = None,
max_num_batched_tokens: int = 8192,
enable_chunked_prefill: bool = True,
enable_prefix_caching: bool = False,
@@ -107,6 +108,7 @@ def create_scheduler(
max_model_len = max_num_batched_tokens
scheduler_config = SchedulerConfig(
max_num_seqs=max_num_seqs,
max_num_active_seqs=max_num_active_seqs,
max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len,
long_prefill_token_threshold=long_prefill_token_threshold,
+19
View File
@@ -67,6 +67,16 @@ class SchedulerConfig:
In real usage, this should be set in `EngineArgs.create_engine_config`.
"""
max_num_active_seqs: int | None = Field(default=None, ge=1)
"""Maximum number of requests the scheduler admits into RUNNING.
``max_num_seqs`` sizes the model runner (per-request buffers and CUDA
graph capture) and is also the default admission limit. Setting this
lowers only the number of requests that may occupy RUNNING, so decode
batches stay smaller without shrinking runner or graph capacity. Must
be ``<= max_num_seqs``. ``None`` (default) keeps current behavior.
"""
long_prefill_token_threshold: int = Field(default=0, ge=0)
"""For chunked prefill, a request is considered long if the prompt is
longer than this number of tokens. 0 disables the cap (default)."""
@@ -310,6 +320,15 @@ class SchedulerConfig:
f"({self.max_num_seqs})."
)
if (
self.max_num_active_seqs is not None
and self.max_num_active_seqs > self.max_num_seqs
):
raise ValueError(
f"max_num_active_seqs ({self.max_num_active_seqs}) cannot be "
f"greater than max_num_seqs ({self.max_num_seqs})."
)
if self.max_num_batched_tokens > self.max_num_seqs * max_model_len:
logger.warning(
"max_num_batched_tokens (%d) exceeds max_num_seqs "
+6
View File
@@ -556,6 +556,7 @@ class EngineArgs:
max_num_scheduled_tokens: int | None = None
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
max_num_seqs: int | None = None
max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs
max_num_queued_reqs: int | None = None
max_num_queued_tokens: int | None = None
max_logprobs: int = ModelConfig.max_logprobs
@@ -1589,6 +1590,10 @@ class EngineArgs:
"default": None,
},
)
scheduler_group.add_argument(
"--max-num-active-seqs",
**scheduler_kwargs["max_num_active_seqs"],
)
scheduler_group.add_argument(
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
)
@@ -2421,6 +2426,7 @@ class EngineArgs:
max_num_batched_tokens=self.max_num_batched_tokens,
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
max_num_seqs=self.max_num_seqs,
max_num_active_seqs=self.max_num_active_seqs,
max_num_queued_reqs=self.max_num_queued_reqs,
max_num_queued_tokens=self.max_num_queued_tokens,
max_model_len=model_config.max_model_len,
+8 -1
View File
@@ -123,6 +123,13 @@ class Scheduler(SchedulerInterface):
# Scheduling constraints.
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
# Admission into RUNNING. Separate from max_num_running_reqs, which
# is the model-runner slot count.
self.max_num_active_reqs = (
self.scheduler_config.max_num_active_seqs
if self.scheduler_config.max_num_active_seqs is not None
else self.max_num_running_reqs
)
self.max_num_scheduled_tokens = (
self.scheduler_config.max_num_scheduled_tokens
if self.scheduler_config.max_num_scheduled_tokens is not None
@@ -850,7 +857,7 @@ class Scheduler(SchedulerInterface):
# Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not
# in `running` but still hold a model-runner request slot.
num_running = len(self.running) + self.num_waiting_for_streaming_input
if num_running >= self.max_num_running_reqs:
if num_running >= self.max_num_active_reqs:
break
request_queue = self._select_waiting_queue_for_scheduling()