[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)

Signed-off-by: LostFox11 <wangziyue17@huawei.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: LostFox11 <wangziyue17@huawei.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
LostFox11
2026-09-30 19:18:55 -07:00
committed by GitHub
co-authored by LostFox11 Nick Hill
parent 34c458d27f
commit 7314360c9e
7 changed files with 111 additions and 6 deletions
+4
View File
@@ -103,6 +103,9 @@ class InputBatch:
# [num_reqs] per-request prompt length, only populated for R-SWA.
prompt_lens: torch.Tensor | None
# [num_reqs] only populated for non-dummy pipeline-parallel batches.
max_seq_len_np: np.ndarray | None = None
# Longest query the batch may contain. Set when a cudagraph descriptor promises
# a query length this batch's own split does not reach, so attention metadata
# stays valid for every replay the graph serves.
@@ -203,6 +206,7 @@ class InputBatch:
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
has_prefill=False,
max_seq_len_np=None,
decode_graph_eligible=True,
input_ids=input_ids,
positions=positions,
+7
View File
@@ -1274,6 +1274,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
]
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
has_prefill = bool(is_prefilling_np.any())
max_seq_len_np = None
if self.use_pp and has_prefill:
# consumed by the PP `compute_need_sampled_mask` for batches with prefill.
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
num_draft_tokens_np = None
if draft_tokens:
@@ -1312,6 +1316,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
has_prefill=has_prefill,
prefill_runs_as_decode_np=prefill_runs_as_decode_np,
decode_graph_eligible=decode_graph_eligible,
max_seq_len_np=max_seq_len_np,
)
return batch_state, get_uniform_decode_token_count(
num_reqs, num_toks, max_query_len, decode_graph_eligible
@@ -1499,6 +1504,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
has_prefill=batch_req_state.has_prefill,
prefill_runs_as_decode_np=batch_req_state.prefill_runs_as_decode_np,
decode_graph_eligible=batch_req_state.decode_graph_eligible,
max_seq_len_np=batch_req_state.max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
is_padding=is_padding,
@@ -2392,6 +2398,7 @@ class BatchReqState(NamedTuple):
prefill_len_np: np.ndarray # [num_reqs]
num_computed_prefill_tokens_np: np.ndarray # [num_reqs]
is_prefilling_np: np.ndarray # [num_reqs]
max_seq_len_np: np.ndarray | None # [num_reqs], None in non-pp case
has_prefill: bool
prefill_runs_as_decode_np: np.ndarray | None # [num_reqs]
decode_graph_eligible: bool
+1
View File
@@ -649,6 +649,7 @@ class PCPManager:
prefill_len_np=local_prefill_len_np,
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
is_prefilling_np=local_is_prefilling_np,
max_seq_len_np=None,
has_prefill=local_has_prefill,
decode_graph_eligible=not local_has_prefill,
prefill_runs_as_decode_np=None,
+10 -5
View File
@@ -37,14 +37,19 @@ class PendingRecv:
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
that produce a sampled token this step, and therefore must have that token
(and the draft block proposed from it) propagated to the earlier PP stages.
Returns None if no request in the batch produces a sample."""
with outputs that might be needed in a subsequent (decode) step.
Returns None if no sampled outputs are needed in the requests' next step."""
old_computed = input_batch.num_computed_tokens_np
prefill_len = input_batch.prefill_len_np
# Exclude non-final prefill chunks (they don't produce a sample).
produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len
return produces_sample if produces_sample.any() else None
need_sampled_mask = old_computed + input_batch.num_scheduled_tokens >= prefill_len
if input_batch.max_seq_len_np is not None:
# Also exclude final prefill chunks whose single sampled token reaches
# the request's length cap.
finished_prefill = prefill_len + 1 >= input_batch.max_seq_len_np
finished_prefill &= input_batch.is_prefilling_np
need_sampled_mask &= ~finished_prefill
return need_sampled_mask if need_sampled_mask.any() else None
class PPHandler:
+1
View File
@@ -194,6 +194,7 @@ def _slice_input_batch(
if input_batch.prompt_lens is None
else input_batch.prompt_lens[req_start:req_stop]
),
max_seq_len_np=None,
fast_prefill=None,
)