diff --git a/tests/v1/core/test_deferred_block_free.py b/tests/v1/core/test_deferred_block_free.py index ebbf492e20..44cdf567f7 100644 --- a/tests/v1/core/test_deferred_block_free.py +++ b/tests/v1/core/test_deferred_block_free.py @@ -417,7 +417,7 @@ def test_abort_mid_prefill_defers_free(): chunk is still in flight must withhold its blocks. """ scheduler = create_scheduler( - model=MODEL, async_scheduling=True, long_prefill_token_threshold=16 + model=MODEL, async_scheduling=True, max_num_batched_tokens=16 ) scheduler.defer_block_free = True pool = scheduler.kv_cache_manager.block_pool diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index a58bcab7d5..e31163923b 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -762,6 +762,36 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool): assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224 +def test_long_prefill_threshold_ignored_when_alone(): + """A lone long prefill is not capped by the threshold: it has no other + request to starve.""" + scheduler = create_scheduler( + max_num_batched_tokens=1024, + long_prefill_token_threshold=400, + ) + request = create_requests(num_requests=1, num_tokens=2000)[0] + scheduler.add_request(request) + + output = scheduler.schedule() + assert output.num_scheduled_tokens[request.request_id] == 1024 + + +def test_long_prefill_threshold_applies_with_other_requests(): + """The threshold caps the prefill as soon as another request is queued.""" + scheduler = create_scheduler( + max_num_batched_tokens=1024, + long_prefill_token_threshold=400, + ) + long_req = create_requests(num_requests=1, num_tokens=2000)[0] + short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0] + for request in [long_req, short_req]: + scheduler.add_request(request) + + output = scheduler.schedule() + assert output.num_scheduled_tokens[long_req.request_id] == 400 + assert output.num_scheduled_tokens[short_req.request_id] == 10 + + def test_update_from_output_routes_sampling_masks_by_request(): """Each request receives the sampler row at its own batch index.""" scheduler = create_scheduler() diff --git a/tests/v1/e2e/general/test_pooling_chunked_prefill.py b/tests/v1/e2e/general/test_pooling_chunked_prefill.py index e6b63eaf61..ab36e9427d 100644 --- a/tests/v1/e2e/general/test_pooling_chunked_prefill.py +++ b/tests/v1/e2e/general/test_pooling_chunked_prefill.py @@ -77,13 +77,13 @@ def test_pooling_chunked_prefill(vllm_runner, monkeypatch): chunk_size = 10 - # Set chunking parameters to force chunked prefill - # Note: Chunked prefill is automatically handled by vLLM - # internally based on the model size and prompt + # Cap the token budget to force chunked prefill; the + # long_prefill_token_threshold is not applied to a lone request. with vllm_runner( model_id, runner="pooling", - long_prefill_token_threshold=chunk_size, + max_num_batched_tokens=chunk_size, + max_num_seqs=1, tensor_parallel_size=1, enforce_eager=True, enable_chunked_prefill=True, diff --git a/vllm/config/scheduler.py b/vllm/config/scheduler.py index 321c76fe30..9185c26c4d 100644 --- a/vllm/config/scheduler.py +++ b/vllm/config/scheduler.py @@ -79,7 +79,10 @@ class SchedulerConfig: long_prefill_token_threshold: int = Field(default=0, ge=0) """For chunked prefill, a request is considered long if the prompt is - longer than this number of tokens. 0 disables the cap (default).""" + longer than this number of tokens. 0 disables the cap (default). + + The cap is not applied when the request is the only one in the batch, + since there is no other request for it to starve.""" max_num_queued_reqs: int | None = Field(default=None, ge=0) """Maximum number of requests that can be in-flight (waiting or running) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 36a09f9adc..18bbc10283 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -599,6 +599,15 @@ class Scheduler(SchedulerInterface): throttle_prefills and not self.prefill_capacity_bound ) and any(not r.is_prefill_chunk for r in self.running) + # `long_prefill_token_threshold` exists to stop a long prefill from + # starving other requests of the token budget. When it is the only + # request there is nobody to starve, so let it use the whole budget. + long_prefill_token_threshold = ( + self.scheduler_config.long_prefill_token_threshold + if len(self.running) + len(self.waiting) + len(self.skipped_waiting) > 1 + else 0 + ) + # First, schedule the RUNNING requests. req_index = 0 while req_index < len(self.running) and token_budget > 0: @@ -650,8 +659,8 @@ class Scheduler(SchedulerInterface): + request.num_output_placeholders - request.num_computed_tokens ) - if 0 < self.scheduler_config.long_prefill_token_threshold < num_new_tokens: - num_new_tokens = self.scheduler_config.long_prefill_token_threshold + if 0 < long_prefill_token_threshold < num_new_tokens: + num_new_tokens = long_prefill_token_threshold num_new_tokens = min( num_new_tokens, token_budget, input_budget - draft_slots ) @@ -1090,9 +1099,8 @@ class Scheduler(SchedulerInterface): num_new_tokens = padded_num_tokens pad_spec_decode = True - threshold = self.scheduler_config.long_prefill_token_threshold - if 0 < threshold < num_new_tokens: - num_new_tokens = threshold + if 0 < long_prefill_token_threshold < num_new_tokens: + num_new_tokens = long_prefill_token_threshold # chunked prefill has to be enabled explicitly to allow # pooling requests to be chunked