[Scheduler] Add --max-num-active-seqs to cap RUNNING admission (#56758)
Signed-off-by: Li <chuali@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
b1f9ebc0c3
commit
f730a93d2b
@@ -193,6 +193,22 @@ def test_get_num_unfinished_requests():
|
||||
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
|
||||
|
||||
|
||||
def test_max_num_active_seqs_caps_admission():
|
||||
"""Admission is capped by max_num_active_seqs, not max_num_seqs."""
|
||||
scheduler = create_scheduler(max_num_seqs=16, max_num_active_seqs=3)
|
||||
requests = create_requests(num_requests=10)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert len(output.scheduled_new_reqs) == 3
|
||||
assert len(scheduler.running) == 3
|
||||
assert len(scheduler.waiting) == 7
|
||||
# Execution capacity is untouched.
|
||||
assert scheduler.max_num_running_reqs == 16
|
||||
|
||||
|
||||
def _bind_hisparse_connector(scheduler):
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
|
||||
@@ -52,6 +52,7 @@ def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int
|
||||
def create_scheduler(
|
||||
model: str = "facebook/opt-125m",
|
||||
max_num_seqs: int = 16,
|
||||
max_num_active_seqs: int | None = None,
|
||||
max_num_batched_tokens: int = 8192,
|
||||
enable_chunked_prefill: bool = True,
|
||||
enable_prefix_caching: bool = False,
|
||||
@@ -107,6 +108,7 @@ def create_scheduler(
|
||||
max_model_len = max_num_batched_tokens
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_active_seqs=max_num_active_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_model_len=max_model_len,
|
||||
long_prefill_token_threshold=long_prefill_token_threshold,
|
||||
|
||||
@@ -67,6 +67,16 @@ class SchedulerConfig:
|
||||
In real usage, this should be set in `EngineArgs.create_engine_config`.
|
||||
"""
|
||||
|
||||
max_num_active_seqs: int | None = Field(default=None, ge=1)
|
||||
"""Maximum number of requests the scheduler admits into RUNNING.
|
||||
|
||||
``max_num_seqs`` sizes the model runner (per-request buffers and CUDA
|
||||
graph capture) and is also the default admission limit. Setting this
|
||||
lowers only the number of requests that may occupy RUNNING, so decode
|
||||
batches stay smaller without shrinking runner or graph capacity. Must
|
||||
be ``<= max_num_seqs``. ``None`` (default) keeps current behavior.
|
||||
"""
|
||||
|
||||
long_prefill_token_threshold: int = Field(default=0, ge=0)
|
||||
"""For chunked prefill, a request is considered long if the prompt is
|
||||
longer than this number of tokens. 0 disables the cap (default)."""
|
||||
@@ -310,6 +320,15 @@ class SchedulerConfig:
|
||||
f"({self.max_num_seqs})."
|
||||
)
|
||||
|
||||
if (
|
||||
self.max_num_active_seqs is not None
|
||||
and self.max_num_active_seqs > self.max_num_seqs
|
||||
):
|
||||
raise ValueError(
|
||||
f"max_num_active_seqs ({self.max_num_active_seqs}) cannot be "
|
||||
f"greater than max_num_seqs ({self.max_num_seqs})."
|
||||
)
|
||||
|
||||
if self.max_num_batched_tokens > self.max_num_seqs * max_model_len:
|
||||
logger.warning(
|
||||
"max_num_batched_tokens (%d) exceeds max_num_seqs "
|
||||
|
||||
@@ -556,6 +556,7 @@ class EngineArgs:
|
||||
max_num_scheduled_tokens: int | None = None
|
||||
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
|
||||
max_num_seqs: int | None = None
|
||||
max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs
|
||||
max_num_queued_reqs: int | None = None
|
||||
max_num_queued_tokens: int | None = None
|
||||
max_logprobs: int = ModelConfig.max_logprobs
|
||||
@@ -1589,6 +1590,10 @@ class EngineArgs:
|
||||
"default": None,
|
||||
},
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-active-seqs",
|
||||
**scheduler_kwargs["max_num_active_seqs"],
|
||||
)
|
||||
scheduler_group.add_argument(
|
||||
"--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"]
|
||||
)
|
||||
@@ -2421,6 +2426,7 @@ class EngineArgs:
|
||||
max_num_batched_tokens=self.max_num_batched_tokens,
|
||||
max_num_scheduled_tokens=self.max_num_scheduled_tokens,
|
||||
max_num_seqs=self.max_num_seqs,
|
||||
max_num_active_seqs=self.max_num_active_seqs,
|
||||
max_num_queued_reqs=self.max_num_queued_reqs,
|
||||
max_num_queued_tokens=self.max_num_queued_tokens,
|
||||
max_model_len=model_config.max_model_len,
|
||||
|
||||
@@ -123,6 +123,13 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Scheduling constraints.
|
||||
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
|
||||
# Admission into RUNNING. Separate from max_num_running_reqs, which
|
||||
# is the model-runner slot count.
|
||||
self.max_num_active_reqs = (
|
||||
self.scheduler_config.max_num_active_seqs
|
||||
if self.scheduler_config.max_num_active_seqs is not None
|
||||
else self.max_num_running_reqs
|
||||
)
|
||||
self.max_num_scheduled_tokens = (
|
||||
self.scheduler_config.max_num_scheduled_tokens
|
||||
if self.scheduler_config.max_num_scheduled_tokens is not None
|
||||
@@ -850,7 +857,7 @@ class Scheduler(SchedulerInterface):
|
||||
# Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not
|
||||
# in `running` but still hold a model-runner request slot.
|
||||
num_running = len(self.running) + self.num_waiting_for_streaming_input
|
||||
if num_running >= self.max_num_running_reqs:
|
||||
if num_running >= self.max_num_active_reqs:
|
||||
break
|
||||
|
||||
request_queue = self._select_waiting_queue_for_scheduling()
|
||||
|
||||
Reference in New Issue
Block a user