[Scheduler] Tune --long-prefill-token-threshold adaptiveness (#58459)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
Robert Shaw
2026-09-23 22:23:18 -04:00
committed by GitHub
parent e9f167129c
commit 6697a7dd30
5 changed files with 54 additions and 1 deletions
+20
View File
@@ -796,6 +796,26 @@ def test_long_prefill_threshold_applies_with_other_requests():
assert output.num_scheduled_tokens[short_req.request_id] == 10 assert output.num_scheduled_tokens[short_req.request_id] == 10
def test_long_prefill_threshold_floored_by_fair_share():
"""With the adaptive flag, the effective threshold never falls below the
fair share of the token budget: max_num_batched_tokens / num queued +
running requests."""
scheduler = create_scheduler(
max_num_batched_tokens=1024,
long_prefill_token_threshold=100,
long_prefill_token_threshold_adaptive=True,
)
long_req = create_requests(num_requests=1, num_tokens=2000)[0]
short_req = create_requests(num_requests=1, num_tokens=10, req_ids=["short"])[0]
for request in [long_req, short_req]:
scheduler.add_request(request)
output = scheduler.schedule()
# 100 is below the fair share (1024 // 2 = 512), so the floor binds.
assert output.num_scheduled_tokens[long_req.request_id] == 512
assert output.num_scheduled_tokens[short_req.request_id] == 10
def test_update_from_output_routes_sampling_masks_by_request(): def test_update_from_output_routes_sampling_masks_by_request():
"""Each request receives the sampler row at its own batch index.""" """Each request receives the sampler row at its own batch index."""
scheduler = create_scheduler() scheduler = create_scheduler()
+5
View File
@@ -58,6 +58,7 @@ def create_scheduler(
enable_chunked_prefill: bool = True, enable_chunked_prefill: bool = True,
enable_prefix_caching: bool = False, enable_prefix_caching: bool = False,
long_prefill_token_threshold: int = 0, long_prefill_token_threshold: int = 0,
long_prefill_token_threshold_adaptive: bool = False,
disable_chunked_mm_input: bool = False, disable_chunked_mm_input: bool = False,
use_kv_connector: None | bool | str | MockKVConfig = None, use_kv_connector: None | bool | str | MockKVConfig = None,
kv_role: str = "kv_both", kv_role: str = "kv_both",
@@ -90,6 +91,9 @@ def create_scheduler(
enable_prefix_caching: optionally force APC config enable_prefix_caching: optionally force APC config
(True/False) or use default (True/False) or use default
(False) (False)
long_prefill_token_threshold: cap on prefill chunk size
long_prefill_token_threshold_adaptive: floor the cap at a
fair share of the token budget
Returns: Returns:
{class}`Scheduler` instance {class}`Scheduler` instance
@@ -115,6 +119,7 @@ def create_scheduler(
max_num_batched_tokens=max_num_batched_tokens, max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len, max_model_len=max_model_len,
long_prefill_token_threshold=long_prefill_token_threshold, long_prefill_token_threshold=long_prefill_token_threshold,
long_prefill_token_threshold_adaptive=(long_prefill_token_threshold_adaptive),
disable_chunked_mm_input=disable_chunked_mm_input, disable_chunked_mm_input=disable_chunked_mm_input,
enable_chunked_prefill=enable_chunked_prefill, enable_chunked_prefill=enable_chunked_prefill,
async_scheduling=async_scheduling, async_scheduling=async_scheduling,
+6
View File
@@ -84,6 +84,12 @@ class SchedulerConfig:
The cap is not applied when the request is the only one in the batch, The cap is not applied when the request is the only one in the batch,
since there is no other request for it to starve.""" since there is no other request for it to starve."""
long_prefill_token_threshold_adaptive: bool = Field(default=False)
"""Floor the effective long prefill token threshold at a fair share of
the token budget: max_num_batched_tokens divided by the number of
queued and running requests. Only applies when
long_prefill_token_threshold is nonzero."""
max_num_queued_reqs: int | None = Field(default=None, ge=0) max_num_queued_reqs: int | None = Field(default=None, ge=0)
"""Maximum number of requests that can be in-flight (waiting or running) """Maximum number of requests that can be in-flight (waiting or running)
at the same time, or None for no limit. When the limit is reached, new at the same time, or None for no limit. When the limit is reached, new
+10
View File
@@ -573,6 +573,9 @@ class EngineArgs:
max_num_batched_tokens: int | None = None max_num_batched_tokens: int | None = None
max_num_scheduled_tokens: int | None = None max_num_scheduled_tokens: int | None = None
long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold long_prefill_token_threshold: int = SchedulerConfig.long_prefill_token_threshold
long_prefill_token_threshold_adaptive: bool = (
SchedulerConfig.long_prefill_token_threshold_adaptive
)
max_num_seqs: int | None = None max_num_seqs: int | None = None
max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs max_num_active_seqs: int | None = SchedulerConfig.max_num_active_seqs
max_num_queued_reqs: int | None = None max_num_queued_reqs: int | None = None
@@ -1643,6 +1646,10 @@ class EngineArgs:
"--long-prefill-token-threshold", "--long-prefill-token-threshold",
**scheduler_kwargs["long_prefill_token_threshold"], **scheduler_kwargs["long_prefill_token_threshold"],
) )
scheduler_group.add_argument(
"--long-prefill-token-threshold-adaptive",
**scheduler_kwargs["long_prefill_token_threshold_adaptive"],
)
# multi-step scheduling has been removed; corresponding arguments # multi-step scheduling has been removed; corresponding arguments
# are no longer supported. # are no longer supported.
scheduler_group.add_argument( scheduler_group.add_argument(
@@ -2489,6 +2496,9 @@ class EngineArgs:
policy=self.scheduling_policy, policy=self.scheduling_policy,
scheduler_cls=self.scheduler_cls, scheduler_cls=self.scheduler_cls,
long_prefill_token_threshold=self.long_prefill_token_threshold, long_prefill_token_threshold=self.long_prefill_token_threshold,
long_prefill_token_threshold_adaptive=(
self.long_prefill_token_threshold_adaptive
),
scheduler_reserve_full_isl=self.scheduler_reserve_full_isl, scheduler_reserve_full_isl=self.scheduler_reserve_full_isl,
watermark=self.watermark, watermark=self.watermark,
prefill_schedule_interval=self.prefill_schedule_interval, prefill_schedule_interval=self.prefill_schedule_interval,
+13 -1
View File
@@ -134,6 +134,9 @@ class Scheduler(SchedulerInterface):
if self.scheduler_config.max_num_scheduled_tokens is not None if self.scheduler_config.max_num_scheduled_tokens is not None
else self.scheduler_config.max_num_batched_tokens else self.scheduler_config.max_num_batched_tokens
) )
self.adaptive_long_prefill_threshold = (
self.scheduler_config.long_prefill_token_threshold_adaptive
)
self.max_model_len = vllm_config.model_config.max_model_len self.max_model_len = vllm_config.model_config.max_model_len
self.enable_kv_cache_events = ( self.enable_kv_cache_events = (
self.kv_events_config is not None self.kv_events_config is not None
@@ -602,11 +605,20 @@ class Scheduler(SchedulerInterface):
# `long_prefill_token_threshold` exists to stop a long prefill from # `long_prefill_token_threshold` exists to stop a long prefill from
# starving other requests of the token budget. When it is the only # starving other requests of the token budget. When it is the only
# request there is nobody to starve, so let it use the whole budget. # request there is nobody to starve, so let it use the whole budget.
num_eligible_reqs = (
len(self.running) + len(self.waiting) + len(self.skipped_waiting)
)
long_prefill_token_threshold = ( long_prefill_token_threshold = (
self.scheduler_config.long_prefill_token_threshold self.scheduler_config.long_prefill_token_threshold
if len(self.running) + len(self.waiting) + len(self.skipped_waiting) > 1 if num_eligible_reqs > 1
else 0 else 0
) )
if long_prefill_token_threshold > 0 and self.adaptive_long_prefill_threshold:
# Floor the cap at a fair share of the input budget so it never
# cuts a request below max_num_batched_tokens / num requests.
long_prefill_token_threshold = max(
long_prefill_token_threshold, input_budget // num_eligible_reqs
)
# First, schedule the RUNNING requests. # First, schedule the RUNNING requests.
req_index = 0 req_index = 0