[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)

Signed-off-by: LostFox11 <wangziyue17@huawei.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: LostFox11 <wangziyue17@huawei.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
LostFox11
2026-09-30 19:18:55 -07:00
committed by GitHub
co-authored by LostFox11 Nick Hill
parent 34c458d27f
commit 7314360c9e
7 changed files with 111 additions and 6 deletions
+10 -5
View File
@@ -37,14 +37,19 @@ class PendingRecv:
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
that produce a sampled token this step, and therefore must have that token
(and the draft block proposed from it) propagated to the earlier PP stages.
Returns None if no request in the batch produces a sample."""
with outputs that might be needed in a subsequent (decode) step.
Returns None if no sampled outputs are needed in the requests' next step."""
old_computed = input_batch.num_computed_tokens_np
prefill_len = input_batch.prefill_len_np
# Exclude non-final prefill chunks (they don't produce a sample).
produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len
return produces_sample if produces_sample.any() else None
need_sampled_mask = old_computed + input_batch.num_scheduled_tokens >= prefill_len
if input_batch.max_seq_len_np is not None:
# Also exclude final prefill chunks whose single sampled token reaches
# the request's length cap.
finished_prefill = prefill_len + 1 >= input_batch.max_seq_len_np
finished_prefill &= input_batch.is_prefilling_np
need_sampled_mask &= ~finished_prefill
return need_sampled_mask if need_sampled_mask.any() else None
class PPHandler: