[MRV2][Spec] Fuse AR speculator multi-step decodes back into one CUDA graph (#46849)

Signed-off-by: Yizhou Liu <liu_yizhou@outlook.com>
This commit is contained in:
Yizhou
2026-08-11 15:06:22 +08:00
committed by GitHub
parent 1044d19998
commit a311916a29
10 changed files with 492 additions and 49 deletions
@@ -10,17 +10,21 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
get_uniform_token_count,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
@@ -41,6 +45,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.use_fused_multi_step_decode = False
def load_model(self, target_model: nn.Module) -> None:
super().load_model(target_model)
@@ -76,6 +81,49 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
"""
return True
def set_attn(
self,
model_state: ModelState,
kv_cache_config: KVCacheConfig,
block_tables: BlockTables,
target_input_buffers: InputBuffers,
target_attn_groups: list[list[AttentionGroup]],
) -> None:
super().set_attn(
model_state,
kv_cache_config,
block_tables,
target_input_buffers,
target_attn_groups,
)
self._configure_fused_multi_step_decode()
def _configure_fused_multi_step_decode(self) -> None:
if self.num_speculative_steps == 1:
self.use_fused_multi_step_decode = False
return
if not self.advance_draft_positions:
self.use_fused_multi_step_decode = True
return
unsupported_backends = sorted(
{
attn_group.backend.get_name()
for attn_groups in self.attn_groups
for attn_group in attn_groups
if not attn_group.supports_draft_decode_metadata_update
}
)
self.use_fused_multi_step_decode = not unsupported_backends
if unsupported_backends:
logger.info_once(
"Fused multi-step draft decode is not supported by attention "
"backend(s) %s; falling back to rebuilding attention metadata "
"between draft steps.",
", ".join(unsupported_backends),
)
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# Initialize cudagraph manager for draft prefill (draft position 0).
self.prefill_cudagraph_manager = SpeculatorCudaGraphManager(
@@ -104,6 +152,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
# Reset indices to zeros to prevent stale values from prior
# dummy runs to cause out-of-bounds indexing during capture.
self.last_token_indices.zero_()
self.idx_mapping.zero_()
# Capture the prefill routine (model forward + compute_logits +
# sample).
@@ -133,12 +182,15 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
return
self.on_multi_step_decode_begin(self.max_num_reqs)
# Capture the decode draft generation routine (model forward +
# sample + update_draft_inputs) for a single
# step.
# Capture either the fused decode loop or one decode step per graph.
assert self.decode_cudagraph_manager is not None
decode_fn = (
self._generate_fused_drafts
if self.use_fused_multi_step_decode
else self._generate_draft
)
self.decode_cudagraph_manager.capture(
self._generate_draft,
decode_fn,
self.model_state,
self.input_buffers,
self.block_tables,
@@ -290,7 +342,12 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.on_multi_step_decode_begin(num_reqs)
# Generate the remaining num_speculative_steps - 1 draft tokens.
self._multi_step_decode(
decode_fn = (
self._fused_multi_step_decode
if self.use_fused_multi_step_decode
else self._multi_step_decode
)
decode_fn(
num_reqs,
dummy_run and skip_attn_for_dummy_run,
decode_batch_desc,
@@ -435,10 +492,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
step=step,
)
# Update the current draft step.
self.current_draft_step.fill_(step)
# Generate draft tokens for the current step.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
@@ -452,6 +507,95 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
cudagraph_runtime_mode=batch_desc.cg_mode,
)
def _fused_multi_step_decode(
self,
num_reqs: int,
skip_attn: bool,
batch_desc: BatchExecutionDescriptor,
num_tokens_across_dp: torch.Tensor | None,
seq_lens_cpu_upper_bound: torch.Tensor,
) -> None:
positions = self.input_buffers.positions[:num_reqs]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
idx_mapping = self.idx_mapping[:num_reqs]
attn_metadata = None
slot_mappings_by_layer = None
if not skip_attn:
slot_mappings = self.block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
batch_desc.num_tokens,
)
if batch_desc.cg_mode != CUDAGraphMode.FULL:
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
attn_metadata = self._build_draft_attn_metadata(
num_reqs=num_reqs,
num_reqs_padded=batch_desc.num_reqs or num_reqs,
num_tokens_padded=batch_desc.num_tokens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
step=1,
)
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.decode_cudagraph_manager is not None
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
return
self._generate_fused_drafts(
num_reqs,
batch_desc.num_tokens,
attn_metadata,
slot_mappings_by_layer,
num_tokens_across_dp,
batch_desc.cg_mode,
)
def _generate_fused_drafts(
self,
num_reqs: int,
num_tokens_padded: int,
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
) -> None:
idx_mapping = self.idx_mapping[:num_reqs]
positions = self.input_buffers.positions[:num_reqs]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
attn_groups = (
[group for groups in self.attn_groups for group in groups]
if attn_metadata is not None
else []
)
for step in range(1, self.num_speculative_steps):
self.current_draft_step.fill_(step)
self._generate_draft(
num_reqs,
num_tokens_padded,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cudagraph_runtime_mode,
)
if (
step < self.num_speculative_steps - 1
and attn_metadata is not None
and self.advance_draft_positions
):
self.block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
num_tokens_padded,
)
for attn_group in attn_groups:
attn_group.update_draft_decode_metadata(attn_metadata)
def _generate_draft(
self,
num_reqs: int,