[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)

Signed-off-by: LostFox11 <wangziyue17@huawei.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: LostFox11 <wangziyue17@huawei.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
LostFox11
2026-09-30 19:18:55 -07:00
committed by GitHub
co-authored by LostFox11 Nick Hill
parent 34c458d27f
commit 7314360c9e
7 changed files with 111 additions and 6 deletions
+7
View File
@@ -1274,6 +1274,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
]
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
has_prefill = bool(is_prefilling_np.any())
max_seq_len_np = None
if self.use_pp and has_prefill:
# consumed by the PP `compute_need_sampled_mask` for batches with prefill.
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
num_draft_tokens_np = None
if draft_tokens:
@@ -1312,6 +1316,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
has_prefill=has_prefill,
prefill_runs_as_decode_np=prefill_runs_as_decode_np,
decode_graph_eligible=decode_graph_eligible,
max_seq_len_np=max_seq_len_np,
)
return batch_state, get_uniform_decode_token_count(
num_reqs, num_toks, max_query_len, decode_graph_eligible
@@ -1499,6 +1504,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
has_prefill=batch_req_state.has_prefill,
prefill_runs_as_decode_np=batch_req_state.prefill_runs_as_decode_np,
decode_graph_eligible=batch_req_state.decode_graph_eligible,
max_seq_len_np=batch_req_state.max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
is_padding=is_padding,
@@ -2392,6 +2398,7 @@ class BatchReqState(NamedTuple):
prefill_len_np: np.ndarray # [num_reqs]
num_computed_prefill_tokens_np: np.ndarray # [num_reqs]
is_prefilling_np: np.ndarray # [num_reqs]
max_seq_len_np: np.ndarray | None # [num_reqs], None in non-pp case
has_prefill: bool
prefill_runs_as_decode_np: np.ndarray | None # [num_reqs]
decode_graph_eligible: bool