[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)
Signed-off-by: LostFox11 <wangziyue17@huawei.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: LostFox11 <wangziyue17@huawei.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
LostFox11
Nick Hill
parent
34c458d27f
commit
7314360c9e
@@ -103,6 +103,9 @@ class InputBatch:
|
||||
# [num_reqs] per-request prompt length, only populated for R-SWA.
|
||||
prompt_lens: torch.Tensor | None
|
||||
|
||||
# [num_reqs] only populated for non-dummy pipeline-parallel batches.
|
||||
max_seq_len_np: np.ndarray | None = None
|
||||
|
||||
# Longest query the batch may contain. Set when a cudagraph descriptor promises
|
||||
# a query length this batch's own split does not reach, so attention metadata
|
||||
# stays valid for every replay the graph serves.
|
||||
@@ -203,6 +206,7 @@ class InputBatch:
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
has_prefill=False,
|
||||
max_seq_len_np=None,
|
||||
decode_graph_eligible=True,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
|
||||
Reference in New Issue
Block a user