[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Nick Hill
2026-09-28 12:48:52 -07:00
committed by GitHub
co-authored by Nicolò Lucchesi Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Yongye Zhu
parent ef42093a36
commit e7c903609f
12 changed files with 144 additions and 42 deletions
@@ -282,7 +282,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
# the target model during FULL cudagraph.
num_tokens,
max_query_len,
input_batch.has_prefill,
input_batch.decode_graph_eligible,
)
prefill_batch_desc, prefill_batch_sync = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
@@ -190,7 +190,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
# When all requests are decoding (no true prefills), each has
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
uniform_token_count = get_uniform_decode_token_count(
num_reqs, num_tokens, max_query_len, input_batch.has_prefill
num_reqs, num_tokens, max_query_len, input_batch.decode_graph_eligible
)
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
self.cudagraph_manager,