[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yongye Zhu
parent
ef42093a36
commit
e7c903609f
@@ -595,6 +595,7 @@ class PCPManager:
|
||||
)
|
||||
local_is_prefilling_np = np.zeros(num_reqs_after_padding, dtype=np.bool_)
|
||||
local_is_prefilling_np[:num_local_reqs] = real_local_is_prefilling_np
|
||||
local_has_prefill = bool(local_is_prefilling_np.any())
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_after_padding, dtype=np.int32)
|
||||
seq_lens_cpu_upper_bound_np[:num_local_reqs] = (
|
||||
local_start_pos_np + local_num_scheduled_tokens
|
||||
@@ -645,7 +646,9 @@ class PCPManager:
|
||||
prefill_len_np=local_prefill_len_np,
|
||||
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=local_is_prefilling_np,
|
||||
has_prefill=bool(local_is_prefilling_np.any()),
|
||||
has_prefill=local_has_prefill,
|
||||
decode_graph_eligible=not local_has_prefill,
|
||||
prefill_runs_as_decode_np=None,
|
||||
input_ids=input_buffers.input_ids[:num_local_tokens_padded],
|
||||
positions=input_buffers.positions[:num_local_tokens_padded],
|
||||
is_padding=is_padding,
|
||||
|
||||
Reference in New Issue
Block a user