[Spec Decode] Remove eager metadata rebuild during MTP fused multi-step decode (#58463)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -99,10 +99,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
self.use_fused_multi_step_decode = False
|
||||
return
|
||||
|
||||
if not self.advance_draft_positions:
|
||||
self.use_fused_multi_step_decode = True
|
||||
return
|
||||
|
||||
unsupported_backends = sorted(
|
||||
{
|
||||
attn_group.backend.get_name()
|
||||
@@ -574,6 +570,11 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
return
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
if not skip_attn:
|
||||
@@ -583,10 +584,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
positions,
|
||||
batch_desc.num_tokens,
|
||||
)
|
||||
if batch_desc.cg_mode != CUDAGraphMode.FULL:
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
attn_metadata = self._build_uniform_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
batch_desc=batch_desc,
|
||||
@@ -595,11 +595,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
step=1,
|
||||
)
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
return
|
||||
|
||||
self._generate_fused_drafts(
|
||||
num_reqs,
|
||||
batch_desc.num_tokens,
|
||||
@@ -623,13 +618,19 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
attn_groups = (
|
||||
[group for groups in self.attn_groups for group in groups]
|
||||
if attn_metadata is not None
|
||||
else []
|
||||
)
|
||||
|
||||
for step in range(1, num_speculative_steps):
|
||||
if attn_metadata is not None and (
|
||||
self.advance_draft_positions or step == 1
|
||||
):
|
||||
self.block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded,
|
||||
)
|
||||
self._update_draft_decode_metadata(attn_metadata, num_reqs)
|
||||
|
||||
self.current_draft_step.fill_(step)
|
||||
self._generate_draft(
|
||||
num_reqs,
|
||||
@@ -640,19 +641,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
cudagraph_runtime_mode,
|
||||
num_speculative_steps,
|
||||
)
|
||||
if (
|
||||
step < num_speculative_steps - 1
|
||||
and attn_metadata is not None
|
||||
and self.advance_draft_positions
|
||||
):
|
||||
self.block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded,
|
||||
)
|
||||
for attn_group in attn_groups:
|
||||
attn_group.update_draft_decode_metadata(attn_metadata)
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
|
||||
@@ -562,3 +562,19 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
causal=causal,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
)
|
||||
|
||||
def _update_draft_decode_metadata(
|
||||
self, attn_metadata: dict[str, Any], num_reqs: int
|
||||
) -> None:
|
||||
if self.block_tables.cp_size > 1:
|
||||
prepare_dcp_local_seq_lens(
|
||||
self.input_buffers.dcp_local_seq_lens,
|
||||
self.input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
self.block_tables.cp_size,
|
||||
self.block_tables.cp_rank,
|
||||
self.block_tables.cp_interleave,
|
||||
)
|
||||
for groups in self.attn_groups:
|
||||
for group in groups:
|
||||
group.update_draft_decode_metadata(attn_metadata)
|
||||
|
||||
Reference in New Issue
Block a user