[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)

Signed-off-by: LostFox11 <wangziyue17@huawei.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: LostFox11 <wangziyue17@huawei.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
LostFox11
2026-09-30 19:18:55 -07:00
committed by GitHub
co-authored by LostFox11 Nick Hill
parent 34c458d27f
commit 7314360c9e
7 changed files with 111 additions and 6 deletions
+4
View File
@@ -103,6 +103,9 @@ class InputBatch:
# [num_reqs] per-request prompt length, only populated for R-SWA.
prompt_lens: torch.Tensor | None
# [num_reqs] only populated for non-dummy pipeline-parallel batches.
max_seq_len_np: np.ndarray | None = None
# Longest query the batch may contain. Set when a cudagraph descriptor promises
# a query length this batch's own split does not reach, so attention metadata
# stays valid for every replay the graph serves.
@@ -203,6 +206,7 @@ class InputBatch:
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
has_prefill=False,
max_seq_len_np=None,
decode_graph_eligible=True,
input_ids=input_ids,
positions=positions,