[MRV2][Spec] Fuse AR speculator multi-step decodes back into one CUDA graph (#46849)
Signed-off-by: Yizhou Liu <liu_yizhou@outlook.com>
This commit is contained in:
@@ -10,17 +10,21 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -41,6 +45,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
self.use_fused_multi_step_decode = False
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
super().load_model(target_model)
|
||||
@@ -76,6 +81,49 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
block_tables: BlockTables,
|
||||
target_input_buffers: InputBuffers,
|
||||
target_attn_groups: list[list[AttentionGroup]],
|
||||
) -> None:
|
||||
super().set_attn(
|
||||
model_state,
|
||||
kv_cache_config,
|
||||
block_tables,
|
||||
target_input_buffers,
|
||||
target_attn_groups,
|
||||
)
|
||||
self._configure_fused_multi_step_decode()
|
||||
|
||||
def _configure_fused_multi_step_decode(self) -> None:
|
||||
if self.num_speculative_steps == 1:
|
||||
self.use_fused_multi_step_decode = False
|
||||
return
|
||||
|
||||
if not self.advance_draft_positions:
|
||||
self.use_fused_multi_step_decode = True
|
||||
return
|
||||
|
||||
unsupported_backends = sorted(
|
||||
{
|
||||
attn_group.backend.get_name()
|
||||
for attn_groups in self.attn_groups
|
||||
for attn_group in attn_groups
|
||||
if not attn_group.supports_draft_decode_metadata_update
|
||||
}
|
||||
)
|
||||
self.use_fused_multi_step_decode = not unsupported_backends
|
||||
if unsupported_backends:
|
||||
logger.info_once(
|
||||
"Fused multi-step draft decode is not supported by attention "
|
||||
"backend(s) %s; falling back to rebuilding attention metadata "
|
||||
"between draft steps.",
|
||||
", ".join(unsupported_backends),
|
||||
)
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# Initialize cudagraph manager for draft prefill (draft position 0).
|
||||
self.prefill_cudagraph_manager = SpeculatorCudaGraphManager(
|
||||
@@ -104,6 +152,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
# Reset indices to zeros to prevent stale values from prior
|
||||
# dummy runs to cause out-of-bounds indexing during capture.
|
||||
self.last_token_indices.zero_()
|
||||
self.idx_mapping.zero_()
|
||||
|
||||
# Capture the prefill routine (model forward + compute_logits +
|
||||
# sample).
|
||||
@@ -133,12 +182,15 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
return
|
||||
|
||||
self.on_multi_step_decode_begin(self.max_num_reqs)
|
||||
# Capture the decode draft generation routine (model forward +
|
||||
# sample + update_draft_inputs) for a single
|
||||
# step.
|
||||
# Capture either the fused decode loop or one decode step per graph.
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
decode_fn = (
|
||||
self._generate_fused_drafts
|
||||
if self.use_fused_multi_step_decode
|
||||
else self._generate_draft
|
||||
)
|
||||
self.decode_cudagraph_manager.capture(
|
||||
self._generate_draft,
|
||||
decode_fn,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
self.block_tables,
|
||||
@@ -290,7 +342,12 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
self.on_multi_step_decode_begin(num_reqs)
|
||||
# Generate the remaining num_speculative_steps - 1 draft tokens.
|
||||
self._multi_step_decode(
|
||||
decode_fn = (
|
||||
self._fused_multi_step_decode
|
||||
if self.use_fused_multi_step_decode
|
||||
else self._multi_step_decode
|
||||
)
|
||||
decode_fn(
|
||||
num_reqs,
|
||||
dummy_run and skip_attn_for_dummy_run,
|
||||
decode_batch_desc,
|
||||
@@ -435,10 +492,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
step=step,
|
||||
)
|
||||
|
||||
# Update the current draft step.
|
||||
self.current_draft_step.fill_(step)
|
||||
|
||||
# Generate draft tokens for the current step.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
@@ -452,6 +507,95 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
)
|
||||
|
||||
def _fused_multi_step_decode(
|
||||
self,
|
||||
num_reqs: int,
|
||||
skip_attn: bool,
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor,
|
||||
) -> None:
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
if not skip_attn:
|
||||
slot_mappings = self.block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
batch_desc.num_tokens,
|
||||
)
|
||||
if batch_desc.cg_mode != CUDAGraphMode.FULL:
|
||||
slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
attn_metadata = self._build_draft_attn_metadata(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=batch_desc.num_reqs or num_reqs,
|
||||
num_tokens_padded=batch_desc.num_tokens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
step=1,
|
||||
)
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
return
|
||||
|
||||
self._generate_fused_drafts(
|
||||
num_reqs,
|
||||
batch_desc.num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings_by_layer,
|
||||
num_tokens_across_dp,
|
||||
batch_desc.cg_mode,
|
||||
)
|
||||
|
||||
def _generate_fused_drafts(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens_padded: int,
|
||||
attn_metadata: dict[str, Any] | None,
|
||||
slot_mappings: dict[str, torch.Tensor] | None,
|
||||
num_tokens_across_dp: torch.Tensor | None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> None:
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
positions = self.input_buffers.positions[:num_reqs]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
attn_groups = (
|
||||
[group for groups in self.attn_groups for group in groups]
|
||||
if attn_metadata is not None
|
||||
else []
|
||||
)
|
||||
|
||||
for step in range(1, self.num_speculative_steps):
|
||||
self.current_draft_step.fill_(step)
|
||||
self._generate_draft(
|
||||
num_reqs,
|
||||
num_tokens_padded,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
if (
|
||||
step < self.num_speculative_steps - 1
|
||||
and attn_metadata is not None
|
||||
and self.advance_draft_positions
|
||||
):
|
||||
self.block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded,
|
||||
)
|
||||
for attn_group in attn_groups:
|
||||
attn_group.update_draft_decode_metadata(attn_metadata)
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
|
||||
Reference in New Issue
Block a user