[Spec Decode] Remove eager metadata rebuild during MTP fused multi-step decode (#58463)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-10-06 12:17:30 -04:00
committed by GitHub
parent 4ea0c28bc4
commit fcf53d240c
8 changed files with 182 additions and 50 deletions
@@ -99,10 +99,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.use_fused_multi_step_decode = False
return
if not self.advance_draft_positions:
self.use_fused_multi_step_decode = True
return
unsupported_backends = sorted(
{
attn_group.backend.get_name()
@@ -574,6 +570,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
idx_mapping = self.idx_mapping[:num_reqs]
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
return
attn_metadata = None
slot_mappings_by_layer = None
if not skip_attn:
@@ -583,10 +584,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
positions,
batch_desc.num_tokens,
)
if batch_desc.cg_mode != CUDAGraphMode.FULL:
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
attn_metadata = self._build_uniform_attn_metadata(
num_reqs=num_reqs,
batch_desc=batch_desc,
@@ -595,11 +595,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
step=1,
)
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
return
self._generate_fused_drafts(
num_reqs,
batch_desc.num_tokens,
@@ -623,13 +618,19 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
idx_mapping = self.idx_mapping[:num_reqs]
positions = self.input_buffers.positions[:num_reqs]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
attn_groups = (
[group for groups in self.attn_groups for group in groups]
if attn_metadata is not None
else []
)
for step in range(1, num_speculative_steps):
if attn_metadata is not None and (
self.advance_draft_positions or step == 1
):
self.block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
num_tokens_padded,
)
self._update_draft_decode_metadata(attn_metadata, num_reqs)
self.current_draft_step.fill_(step)
self._generate_draft(
num_reqs,
@@ -640,19 +641,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
cudagraph_runtime_mode,
num_speculative_steps,
)
if (
step < num_speculative_steps - 1
and attn_metadata is not None
and self.advance_draft_positions
):
self.block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
num_tokens_padded,
)
for attn_group in attn_groups:
attn_group.update_draft_decode_metadata(attn_metadata)
def _generate_draft(
self,
@@ -562,3 +562,19 @@ class DraftModelSpeculator(BaseSpeculator):
causal=causal,
dcp_local_seq_lens=dcp_local_seq_lens,
)
def _update_draft_decode_metadata(
self, attn_metadata: dict[str, Any], num_reqs: int
) -> None:
if self.block_tables.cp_size > 1:
prepare_dcp_local_seq_lens(
self.input_buffers.dcp_local_seq_lens,
self.input_buffers.seq_lens,
num_reqs,
self.block_tables.cp_size,
self.block_tables.cp_rank,
self.block_tables.cp_interleave,
)
for groups in self.attn_groups:
for group in groups:
group.update_draft_decode_metadata(attn_metadata)