[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yongye Zhu
parent
ef42093a36
commit
e7c903609f
@@ -282,7 +282,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
# the target model during FULL cudagraph.
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
input_batch.has_prefill,
|
||||
input_batch.decode_graph_eligible,
|
||||
)
|
||||
prefill_batch_desc, prefill_batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.prefill_cudagraph_manager,
|
||||
|
||||
@@ -190,7 +190,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator):
|
||||
# When all requests are decoding (no true prefills), each has
|
||||
# num_speculative_steps + 1 tokens, enabling FULL graph replay.
|
||||
uniform_token_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, max_query_len, input_batch.has_prefill
|
||||
num_reqs, num_tokens, max_query_len, input_batch.decode_graph_eligible
|
||||
)
|
||||
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.cudagraph_manager,
|
||||
|
||||
Reference in New Issue
Block a user