[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yongye Zhu
parent
ef42093a36
commit
e7c903609f
@@ -81,6 +81,8 @@ class InputBatch:
|
||||
is_prefilling_np: np.ndarray
|
||||
# == np.any(is_prefilling_np)
|
||||
has_prefill: bool
|
||||
# No prefills, or only prefill rows that run as decodes.
|
||||
decode_graph_eligible: bool
|
||||
|
||||
# [num_tokens_after_padding]
|
||||
input_ids: torch.Tensor
|
||||
@@ -113,6 +115,11 @@ class InputBatch:
|
||||
# [num_reqs] set only under PCP+DCP (see CommonAttentionMetadata).
|
||||
dcp_local_seq_lens_cpu_upper_bound: torch.Tensor | None = None
|
||||
|
||||
# [num_reqs] prefilling rows that schedule one new prompt token (excluding
|
||||
# drafts) over existing context and so compute exactly like decodes.
|
||||
# None if there are no prefills.
|
||||
prefill_runs_as_decode_np: np.ndarray | None = None
|
||||
|
||||
@classmethod
|
||||
def make_dummy(
|
||||
cls,
|
||||
@@ -196,6 +203,7 @@ class InputBatch:
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
is_padding=is_padding,
|
||||
|
||||
Reference in New Issue
Block a user