[Core] Apply prefill_schedule_interval outside data parallelism (#54627)

Signed-off-by: lifulu <fululi12@amd.com>
Signed-off-by: lifulu <lifulu@users.noreply.github.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: lifulu <lifulu@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
fululi12
2026-10-15 11:03:13 -05:00
committed by GitHub
co-authored by Cursor Agent lifulu Claude Opus 5
parent c825e1d235
commit ed3050add4
4 changed files with 329 additions and 3 deletions
+6 -3
View File
@@ -201,9 +201,12 @@ class SchedulerConfig:
(the default) disables the watermark."""
prefill_schedule_interval: int = Field(default=1, ge=1)
"""For data-parallel deployments, only admit new prefill requests
once every N engine steps, aligned across DP ranks, to better balance
per-step forward-pass times."""
"""Only admit prefill once every N engine steps, leaving the steps in
between to decode alone. 1 (the default) admits prefill on every step.
A deferred step carries no prefill at all, including chunks of an
already-admitted prompt, so a long prompt can take up to N times as many
steps to finish prefilling."""
async_scheduling: bool | None = None
"""If set to False, disable async scheduling. Async scheduling helps to
+78
View File
@@ -368,6 +368,36 @@ class Scheduler(SchedulerInterface):
# prefill batch fully drained the waiting queue. Prefill throttling
# is disabled in this case.
self.prefill_capacity_bound = False
# The DP engine core drives this cadence from a counter shared across
# ranks; an engine without one never throttles, so apply the interval
# locally, timed from the last prefill-carrying step. Three regimes:
# 1. no DP -- local cadence applies;
# 2. dense DP rank -- `reconfigure_for_independent_dp_rank` already
# set data_parallel_size=1, so each rank gets its own cadence.
# Intended, but a behaviour change: this used to be a no-op;
# 3. MoE DP rank (size > 1) -- the engine core drives it, no-op here.
self.local_prefill_interval = (
self.scheduler_config.prefill_schedule_interval
if self.parallel_config.data_parallel_size == 1
else 1
)
if self.local_prefill_interval > 1:
# Neither the hold nor its release is otherwise visible in logs or
# stats, which makes a misapplied cadence hard to diagnose.
logger.info_once(
"Prefill cadence: applying prefill_schedule_interval=%d "
"locally (this engine is not DP-coordinated); prefill is held "
"until %d executed steps have passed since the last "
"prefill-carrying step.",
self.local_prefill_interval,
self.local_prefill_interval,
)
# A full interval back, so the first step is never gated.
self.last_prefill_step = -self.local_prefill_interval
# Steps that actually scheduled tokens. `current_step` counts every
# schedule() call including empty ones, so it would expire the hold
# early; it keeps its own meaning (next_decode_eligible_step).
self.num_executed_steps = 0
self.scheduler_reserve_full_isl = (
self.scheduler_config.scheduler_reserve_full_isl
)
@@ -623,6 +653,9 @@ class Scheduler(SchedulerInterface):
scheduled_spec_decode_tokens: dict[str, list[int]] = {}
# Whether the running batch contains any prefill requests.
prefill_scheduled = False
# Whether a waiting request was admitted with prompt left to compute.
# Separate from `prefill_scheduled`, which covers only the running loop.
prefill_admitted = False
# Whether any scheduled request has a synchronous connector KV load.
has_sync_kv_loads = False
@@ -654,6 +687,28 @@ class Scheduler(SchedulerInterface):
long_prefill_token_threshold, input_budget // num_eligible_reqs
)
# Local prefill cadence: hold prefill until the interval since the last
# prefill-carrying step has elapsed, so decode runs in longer
# uninterrupted stretches. Mirrors the DP gate above, including its
# capacity escape: once a release step cannot drain the waiting queue,
# throttling stops (#44558, test_throttle_capacity_bound_guard_admits).
# Without that escape a backed-up engine holds both admissions and
# in-flight chunks for interval-1 of every interval steps, capping
# prefill at roughly one budget per interval and growing the waiting
# queue without bound.
if (
not defer_prefills
and self.local_prefill_interval > 1
and not self.prefill_capacity_bound
# This call becomes executed step num_executed_steps + 1 if it
# schedules anything, and last_prefill_step is stamped on the same
# convention below.
and (self.num_executed_steps + 1) - self.last_prefill_step
< self.local_prefill_interval
and any(not r.is_prefill_chunk for r in self.running)
):
defer_prefills = True
# First, schedule the RUNNING requests.
req_index = 0
while req_index < len(self.running) and token_budget > 0:
@@ -1318,6 +1373,12 @@ class Scheduler(SchedulerInterface):
)
if load_kv_async:
# This path deliberately does not reach `prefill_admitted`
# below, so it does not restart the prefill cadence:
# `num_new_tokens` was set to 0 above, so the step carries
# no prefill compute for this request. The interval is
# timed from steps that actually prefilled; refreshing it
# here would start the cadence from a step that did none.
# If loading async, allocate memory and put request
# into the WAITING_FOR_REMOTE_KV state.
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
@@ -1373,6 +1434,14 @@ class Scheduler(SchedulerInterface):
scheduled_spec_decode_tokens[request_id] = [
-1
] * self.num_spec_tokens
# Must use the SAME predicate as the deferral gate in the
# waiting loop (`num_computed_tokens < num_tokens - 1`).
# Stamping on `< num_prompt_tokens` instead let a request with
# exactly one token left pass the gate on a held step and still
# restart the interval -- which is every full prefix-cache hit,
# and every remote-KV resume, since those set
# num_computed_tokens = num_tokens - 1.
prefill_admitted |= num_computed_tokens < request.num_tokens - 1
# Only track requests that will still be prefilling after this chunk.
if num_computed_tokens + num_new_tokens < request.num_tokens:
self._inflight_prefills.add(request)
@@ -1403,6 +1472,15 @@ class Scheduler(SchedulerInterface):
# Check if the scheduling constraints are satisfied.
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
assert total_num_scheduled_tokens <= self.max_num_scheduled_tokens
# Count this step before stamping the cadence below, so that
# last_prefill_step includes the step that carried the prefill.
if total_num_scheduled_tokens > 0:
self.num_executed_steps += 1
# Restart the cadence interval from any step that carried prefill,
# whether a new admission or a running chunk.
if self.local_prefill_interval > 1 and (prefill_scheduled or prefill_admitted):
self.last_prefill_step = self.num_executed_steps
assert token_budget >= 0
assert input_budget >= 0