[Core][DP] Throttle prefills based on local prefill work (#46532)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-06-24 02:27:12 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e2bdc24612
commit 6af0559ddb
2 changed files with 42 additions and 20 deletions
+3 -4
View File
@@ -783,10 +783,9 @@ class Scheduler(SchedulerInterface):
# KVTransfer: loading remote KV, do not allocate for new work.
assert num_external_computed_tokens > 0
num_new_tokens = 0
elif defer_prefills and request.num_computed_tokens == 0:
# DP prefill balancing: async KV loads (the branch above) are
# allowed to start even on throttled steps, but committing new
# prefill compute is deferred to a cadence-aligned step.
elif defer_prefills and num_computed_tokens < request.num_tokens - 1:
# DP prefill balancing: defer this step's local prefill
# compute to a cadence-aligned step.
break
else:
# Number of tokens to be scheduled.