[Core][DP] Throttle prefills based on local prefill work (#46532)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e2bdc24612
commit
6af0559ddb
@@ -783,10 +783,9 @@ class Scheduler(SchedulerInterface):
|
||||
# KVTransfer: loading remote KV, do not allocate for new work.
|
||||
assert num_external_computed_tokens > 0
|
||||
num_new_tokens = 0
|
||||
elif defer_prefills and request.num_computed_tokens == 0:
|
||||
# DP prefill balancing: async KV loads (the branch above) are
|
||||
# allowed to start even on throttled steps, but committing new
|
||||
# prefill compute is deferred to a cadence-aligned step.
|
||||
elif defer_prefills and num_computed_tokens < request.num_tokens - 1:
|
||||
# DP prefill balancing: defer this step's local prefill
|
||||
# compute to a cadence-aligned step.
|
||||
break
|
||||
else:
|
||||
# Number of tokens to be scheduled.
|
||||
|
||||
Reference in New Issue
Block a user