diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 3239ec25a0..3add7beda0 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -193,6 +193,22 @@ def test_get_num_unfinished_requests(): assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1 +def test_max_num_active_seqs_caps_admission(): + """Admission is capped by max_num_active_seqs, not max_num_seqs.""" + scheduler = create_scheduler(max_num_seqs=16, max_num_active_seqs=3) + requests = create_requests(num_requests=10) + for request in requests: + scheduler.add_request(request) + + output = scheduler.schedule() + + assert len(output.scheduled_new_reqs) == 3 + assert len(scheduler.running) == 3 + assert len(scheduler.waiting) == 7 + # Execution capacity is untouched. + assert scheduler.max_num_running_reqs == 16 + + def _bind_hisparse_connector(scheduler): connector = object.__new__(HiSparseConnector) connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False) diff --git a/tests/v1/core/utils.py b/tests/v1/core/utils.py index 28d1f71ac4..f69b10ced3 100644 --- a/tests/v1/core/utils.py +++ b/tests/v1/core/utils.py @@ -52,6 +52,7 @@ def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int def create_scheduler( model: str = "facebook/opt-125m", max_num_seqs: int = 16, + max_num_active_seqs: int | None = None, max_num_batched_tokens: int = 8192, enable_chunked_prefill: bool = True, enable_prefix_caching: bool = False, @@ -107,6 +108,7 @@ def create_scheduler( max_model_len = max_num_batched_tokens scheduler_config = SchedulerConfig( max_num_seqs=max_num_seqs, + max_num_active_seqs=max_num_active_seqs, max_num_batched_tokens=max_num_batched_tokens, max_model_len=max_model_len, long_prefill_token_threshold=long_prefill_token_threshold, diff --git a/vllm/config/scheduler.py b/vllm/config/scheduler.py index d3264b2d80..321c76fe30 100644 --- a/vllm/config/scheduler.py +++ b/vllm/config/scheduler.py @@ -67,6 +67,16 @@ class SchedulerConfig: In real usage, this should be set in `EngineArgs.create_engine_config`. """ + max_num_active_seqs: int | None = Field(default=None, ge=1) + """Maximum number of requests the scheduler admits into RUNNING. + + ``max_num_seqs`` sizes the model runner (per-request buffers and CUDA + graph capture) and is also the default admission limit. Setting this + lowers only the number of requests that may occupy RUNNING, so decode + batches stay smaller without shrinking runner or graph capacity. Must + be ``<= max_num_seqs``. ``None`` (default) keeps current behavior. + """ + long_prefill_token_threshold: int = Field(default=0, ge=0) """For chunked prefill, a request is considered long if the prompt is longer than this number of tokens. 0 disables the cap (default).""" @@ -310,6 +320,15 @@ class SchedulerConfig: f"({self.max_num_seqs})." ) + if ( + self.max_num_active_seqs is not None + and self.max_num_active_seqs > self.max_num_seqs + ): + raise ValueError( + f"max_num_active_seqs ({self.max_num_active_seqs}) cannot be " + f"greater than max_num_seqs ({self.max_num_seqs})." + ) + if self.max_num_batched_tokens > self.max_num_seqs * max_model_len: logger.warning( "max_num_batched_tokens (%d) exceeds max_num_seqs " diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index ccb30a2597..d637391cd3 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -556,6 +556,7 @@ class EngineArgs: max_num_scheduled_tokens: int | None = None long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold max_num_seqs: int | None = None + max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs max_num_queued_reqs: int | None = None max_num_queued_tokens: int | None = None max_logprobs: int = ModelConfig.max_logprobs @@ -1589,6 +1590,10 @@ class EngineArgs: "default": None, }, ) + scheduler_group.add_argument( + "--max-num-active-seqs", + **scheduler_kwargs["max_num_active_seqs"], + ) scheduler_group.add_argument( "--max-num-queued-reqs", **scheduler_kwargs["max_num_queued_reqs"] ) @@ -2421,6 +2426,7 @@ class EngineArgs: max_num_batched_tokens=self.max_num_batched_tokens, max_num_scheduled_tokens=self.max_num_scheduled_tokens, max_num_seqs=self.max_num_seqs, + max_num_active_seqs=self.max_num_active_seqs, max_num_queued_reqs=self.max_num_queued_reqs, max_num_queued_tokens=self.max_num_queued_tokens, max_model_len=model_config.max_model_len, diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 19ea411aa1..22c231cc43 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -123,6 +123,13 @@ class Scheduler(SchedulerInterface): # Scheduling constraints. self.max_num_running_reqs = self.scheduler_config.max_num_seqs + # Admission into RUNNING. Separate from max_num_running_reqs, which + # is the model-runner slot count. + self.max_num_active_reqs = ( + self.scheduler_config.max_num_active_seqs + if self.scheduler_config.max_num_active_seqs is not None + else self.max_num_running_reqs + ) self.max_num_scheduled_tokens = ( self.scheduler_config.max_num_scheduled_tokens if self.scheduler_config.max_num_scheduled_tokens is not None @@ -850,7 +857,7 @@ class Scheduler(SchedulerInterface): # Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not # in `running` but still hold a model-runner request slot. num_running = len(self.running) + self.num_waiting_for_streaming_input - if num_running >= self.max_num_running_reqs: + if num_running >= self.max_num_active_reqs: break request_queue = self._select_waiting_queue_for_scheduling()