[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)
Signed-off-by: LostFox11 <wangziyue17@huawei.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: LostFox11 <wangziyue17@huawei.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
LostFox11
Nick Hill
parent
34c458d27f
commit
7314360c9e
@@ -37,14 +37,19 @@ class PendingRecv:
|
||||
|
||||
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
|
||||
that produce a sampled token this step, and therefore must have that token
|
||||
(and the draft block proposed from it) propagated to the earlier PP stages.
|
||||
Returns None if no request in the batch produces a sample."""
|
||||
with outputs that might be needed in a subsequent (decode) step.
|
||||
Returns None if no sampled outputs are needed in the requests' next step."""
|
||||
old_computed = input_batch.num_computed_tokens_np
|
||||
prefill_len = input_batch.prefill_len_np
|
||||
# Exclude non-final prefill chunks (they don't produce a sample).
|
||||
produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len
|
||||
return produces_sample if produces_sample.any() else None
|
||||
need_sampled_mask = old_computed + input_batch.num_scheduled_tokens >= prefill_len
|
||||
if input_batch.max_seq_len_np is not None:
|
||||
# Also exclude final prefill chunks whose single sampled token reaches
|
||||
# the request's length cap.
|
||||
finished_prefill = prefill_len + 1 >= input_batch.max_seq_len_np
|
||||
finished_prefill &= input_batch.is_prefilling_np
|
||||
need_sampled_mask &= ~finished_prefill
|
||||
return need_sampled_mask if need_sampled_mask.any() else None
|
||||
|
||||
|
||||
class PPHandler:
|
||||
|
||||
Reference in New Issue
Block a user