[Spec Decode] Skip metadata build for dflash/dspark when all model backends support it (#60692)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Rishi Puri <20074092+puririshi98@users.noreply.github.com>
This commit is contained in:
Giancarlo Delfin
2026-10-12 00:09:10 -07:00
committed by GitHub
co-authored by Rishi Puri
parent 6a70913590
commit 36f3972812
11 changed files with 360 additions and 172 deletions
@@ -828,3 +828,57 @@ def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start():
# Decode row is copied through; prefill rows follow the bounded window.
assert bounded[0].cpu().tolist() == plain[0].cpu().tolist()
assert bounded[1:].cpu().tolist() == rows[1:]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_dspark_noncausal_draft_decode_update_matches_build():
"""A DSpark draft graph refreshes its non-causal SWA metadata in place
instead of rebuilding it, so the refresh must match a fresh build."""
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
num_spec = 4
query_lens = [num_spec] * 3
def make_dspark_builder():
builder = make_builder(vision=False)
builder.is_dspark = True
builder.noncausal_index_width = get_dspark_swa_index_width(WINDOW, num_spec)
builder.decode_threshold = num_spec
return builder
def common_metadata(seq_lens):
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
seq_lens, query_lens, torch.device("cuda")
)
return CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=seq_lens_t,
seq_lens_cpu_upper_bound=seq_lens_t.cpu(),
num_reqs=len(seq_lens),
num_actual_tokens=int(query_start_loc[-1]),
max_query_len=num_spec,
max_seq_len=max(seq_lens),
block_table_tensor=block_table,
slot_mapping=slot_mapping,
causal=False,
)
captured_seq_lens = [6, 40, 130]
replay_seq_lens = [130, 9, 90]
builder = make_dspark_builder()
captured = common_metadata(captured_seq_lens)
md = builder.build(0, captured)
replay = common_metadata(replay_seq_lens)
expected = make_dspark_builder().build(0, replay)
# A replay sees new inputs through the same device buffers.
captured.seq_lens.copy_(replay.seq_lens)
captured.block_table_tensor.copy_(replay.block_table_tensor)
builder.update_draft_decode_metadata(md)
assert md.decode_swa_width == builder.noncausal_index_width
torch.testing.assert_close(md.decode_swa_lens, expected.decode_swa_lens)
torch.testing.assert_close(md.decode_swa_indices, expected.decode_swa_indices)
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import CUDAGraphMode
from vllm.platforms import current_platform
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
from vllm.v1.attention.backends.triton_attn_diffkv import (
TritonAttentionDiffKVMetadataBuilder,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
# the 64-segment scratch holds every query token (e.g. multi-query verify).
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
@pytest.mark.parametrize(
"capability,num_kv_heads,num_seqs,query_len,segments",
[
((12, 0), 1, 11, 1, 64),
((12, 0), 1, 12, 1, 16),
((12, 0), 8, 1, 1, 64),
((12, 0), 8, 2, 1, 16),
((9, 0), 1, 1, 1, 16),
((12, 0), 1, 2, 4, 64),
((12, 0), 1, 10, 4, 16),
],
)
def test_split_k_segments_follow_sm_occupancy(
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
current_platform, "is_device_capability", lambda cap: cap == capability
)
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
config = SimpleNamespace(
model_config=SimpleNamespace(
get_num_attention_heads=lambda _: 8,
get_num_kv_heads=lambda _: num_kv_heads,
get_head_size=lambda: 128,
rswa_window=None,
),
parallel_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=16),
speculative_config=None,
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
),
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
)
builder = builder_cls(spec, ["layer.0"], config, "cpu")
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
assert metadata.num_par_softmax_segments == segments
assert metadata.softmax_segm_output.shape[2] == segments
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
# DiffKV re-allocates the output; it must stay sized like the base buffers.
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
assert (
metadata.softmax_segm_output.data_ptr()
== builder.softmax_segm_output.data_ptr()
)
def _draft_builder(builder_cls, rswa_window: int | None):
config = SimpleNamespace(
model_config=SimpleNamespace(
get_num_attention_heads=lambda _: 8,
get_num_kv_heads=lambda _: 1,
get_head_size=lambda: 128,
rswa_window=rswa_window,
),
parallel_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=16),
speculative_config=None,
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
),
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
)
return builder_cls(spec, ["layer.0"], config, "cpu")
def _draft_decode_batch(seq_lens: list[int]):
batch = BatchSpec(seq_lens=seq_lens, query_lens=[1] * len(seq_lens))
return create_common_attn_metadata(batch, 16, "cpu")
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
@pytest.mark.parametrize("rswa_window", [None, 64])
def test_draft_decode_update_flag_matches_build_writes(builder_cls, rswa_window):
"""The update is a no-op, so the builder may claim support only when
build() writes no builder-owned tensor that a captured graph reads."""
builder = _draft_builder(builder_cls, rswa_window)
common_metadata = _draft_decode_batch([5, 9])
common_metadata.rswa_prefix_lens = torch.tensor([3, 7], dtype=torch.int32)
def owned_bytes():
# Bytes, not values: uninitialized scratch may hold NaNs.
return {
name: value.view(torch.uint8).clone()
for name, value in vars(builder).items()
if isinstance(value, torch.Tensor)
}
before = owned_bytes()
builder.build(0, common_metadata)
after = owned_bytes()
written = [name for name in before if not torch.equal(before[name], after[name])]
assert builder.supports_draft_decode_metadata_update is (not written)
assert builder.supports_draft_decode_metadata_update is (rswa_window is None)
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
def test_draft_decode_update_matches_fresh_build(builder_cls):
"""A replayed graph reads metadata built at capture; after an in-place
update it must match a fresh build of the replayed batch."""
torch.manual_seed(0)
builder = _draft_builder(builder_cls, rswa_window=None)
captured = _draft_decode_batch([100, 120])
metadata = builder.build(0, captured)
replayed = _draft_decode_batch([128, 30])
# The replayed batch arrives through the same persistent input buffers.
captured.seq_lens.copy_(replayed.seq_lens)
captured.block_table_tensor.copy_(replayed.block_table_tensor)
captured.slot_mapping.copy_(replayed.slot_mapping)
builder.update_draft_decode_metadata(metadata)
expected = _draft_builder(builder_cls, rswa_window=None).build(0, replayed)
assert torch.equal(metadata.seq_lens, expected.seq_lens)
assert torch.equal(metadata.block_table, expected.block_table)
assert torch.equal(metadata.slot_mapping, expected.slot_mapping)
assert torch.equal(metadata.query_start_loc, expected.query_start_loc)
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import CUDAGraphMode
from vllm.platforms import current_platform
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
from vllm.v1.attention.backends.triton_attn_diffkv import (
TritonAttentionDiffKVMetadataBuilder,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
# the 64-segment scratch holds every query token (e.g. multi-query verify).
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
@pytest.mark.parametrize(
"capability,num_kv_heads,num_seqs,query_len,segments",
[
((12, 0), 1, 11, 1, 64),
((12, 0), 1, 12, 1, 16),
((12, 0), 8, 1, 1, 64),
((12, 0), 8, 2, 1, 16),
((9, 0), 1, 1, 1, 16),
((12, 0), 1, 2, 4, 64),
((12, 0), 1, 10, 4, 16),
],
)
def test_split_k_segments_follow_sm_occupancy(
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
current_platform, "is_device_capability", lambda cap: cap == capability
)
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
config = SimpleNamespace(
model_config=SimpleNamespace(
get_num_attention_heads=lambda _: 8,
get_num_kv_heads=lambda _: num_kv_heads,
get_head_size=lambda: 128,
rswa_window=None,
),
parallel_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=16),
speculative_config=None,
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
),
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
)
builder = builder_cls(spec, ["layer.0"], config, "cpu")
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
assert metadata.num_par_softmax_segments == segments
assert metadata.softmax_segm_output.shape[2] == segments
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
# DiffKV re-allocates the output; it must stay sized like the base buffers.
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
assert (
metadata.softmax_segm_output.data_ptr()
== builder.softmax_segm_output.data_ptr()
)
@@ -609,3 +609,25 @@ def test_update_draft_decode_metadata_skips_without_scheduler_metadata(monkeypat
assert not called
assert metadata.scheduler_metadata is None
@pytest.mark.parametrize(("num_speculative_steps", "expected"), [(1, False), (3, True)])
def test_decode_metadata_update_needs_draft_decode_steps(
num_speculative_steps, expected
):
"""One speculative step runs no draft decodes, so the in-place metadata
update (and its fallback log) does not apply even on a capable backend."""
speculator = object.__new__(_TestSpeculator)
speculator.num_speculative_steps = num_speculative_steps
speculator.attn_groups = [
[
SimpleNamespace(
supports_draft_decode_metadata_update=True,
backend=SimpleNamespace(get_name=lambda: "CAPABLE"),
)
]
]
speculator._configure_decode_metadata_update()
assert speculator.supports_decode_metadata_update is expected
@@ -206,24 +206,36 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui
metadata = super().build(
common_prefix_len, common_attn_metadata, fast_build, replay_start
)
num_tokens = metadata.num_decode_tokens
if not common_attn_metadata.causal and num_tokens > 0:
assert metadata.decode_swa_lens is not None
assert metadata.seq_lens is not None
assert metadata.token_to_req_indices is not None
topk_lens = self._decode_topk_lens[:num_tokens]
seq_lens = self._decode_seq_lens[:num_tokens]
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
torch.index_select(
metadata.seq_lens,
0,
metadata.token_to_req_indices[:num_tokens],
out=seq_lens,
)
metadata.flashinfer_decode_topk_lens = topk_lens
metadata.flashinfer_decode_seq_lens = seq_lens
if not common_attn_metadata.causal and metadata.num_decode_tokens > 0:
self._fill_noncausal_decode_lens(metadata)
return metadata
def update_draft_decode_metadata(
self, metadata: "DeepseekSparseSWAMetadata"
) -> None:
super().update_draft_decode_metadata(metadata)
if metadata.flashinfer_decode_topk_lens is not None:
self._fill_noncausal_decode_lens(metadata)
def _fill_noncausal_decode_lens(
self, metadata: "DeepseekSparseSWAMetadata"
) -> None:
num_tokens = metadata.num_decode_tokens
assert metadata.decode_swa_lens is not None
assert metadata.seq_lens is not None
assert metadata.token_to_req_indices is not None
topk_lens = self._decode_topk_lens[:num_tokens]
seq_lens = self._decode_seq_lens[:num_tokens]
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
torch.index_select(
metadata.seq_lens,
0,
metadata.token_to_req_indices[:num_tokens],
out=seq_lens,
)
metadata.flashinfer_decode_topk_lens = topk_lens
metadata.flashinfer_decode_seq_lens = seq_lens
class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend):
@staticmethod
+38 -17
View File
@@ -852,23 +852,44 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
num_tokens,
)
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL(
metadata.decode_swa_indices,
metadata.decode_swa_lens,
metadata.decode_swa_indices.shape[-1],
metadata.decode_swa_indices.shape[-1],
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
metadata.query_start_loc,
metadata.seq_lens,
metadata.token_to_req_indices,
metadata.is_valid_token,
metadata.block_table,
self.block_size,
metadata.replay_start,
num_tokens=metadata.num_decode_tokens,
token_offset=0,
)
# build() points non-causal (DSpark draft) metadata at its own buffer.
noncausal = self.decode_swa_indices_noncausal
if (
noncausal is not None
and metadata.decode_swa_indices.data_ptr() == noncausal.data_ptr()
):
_COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL(
metadata.decode_swa_indices,
metadata.decode_swa_lens,
self.window_size,
self.noncausal_index_width,
metadata.query_start_loc,
metadata.seq_lens,
metadata.token_to_req_indices,
metadata.is_valid_token,
metadata.block_table,
self.block_size,
num_tokens=num_tokens,
token_offset=0,
)
else:
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL(
metadata.decode_swa_indices,
metadata.decode_swa_lens,
metadata.decode_swa_indices.shape[-1],
metadata.decode_swa_indices.shape[-1],
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
metadata.decode_swa_lens, # unused (HAS_IMAGE=False)
metadata.query_start_loc,
metadata.seq_lens,
metadata.token_to_req_indices,
metadata.is_valid_token,
metadata.block_table,
self.block_size,
metadata.replay_start,
num_tokens=metadata.num_decode_tokens,
token_offset=0,
)
tile_sched = self.build_tile_scheduler(metadata.num_decode_tokens)
metadata.tile_sched_swaonly = tile_sched[_LAYER_TYPE_SWAONLY]
metadata.tile_sched_c4a = tile_sched[_LAYER_TYPE_C4A]
@@ -193,6 +193,9 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
device=device,
)
self.rswa_window = model_config.rswa_window
# R-SWA restages prefix lengths in build(), which the no-op
# update_draft_decode_metadata() does not refresh.
self.supports_draft_decode_metadata_update = self.rswa_window is None
self.persistent_rswa_prefix_lens: torch.Tensor | None = None
if self.rswa_window is not None:
self.persistent_rswa_prefix_lens = torch.empty(
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Mapping
from typing import Any
import torch
@@ -84,11 +85,11 @@ class DFlashCudaGraphManager(CudaGraphManager):
kv_cache_config: KVCacheConfig,
max_model_len: int,
causal: bool | Mapping[int, bool],
precompute_context_kv: Callable[[int], None],
prepare_forward: Callable[[int, dict[str, Any] | None], None],
progress_bar_desc: str = "Capturing CUDA graphs",
) -> None:
"""``precompute_context_kv(num_reqs)`` is captured ahead of the query
forward."""
"""``prepare_forward(num_reqs, attn_metadata)`` is captured ahead of the
query forward."""
def create_forward_fn(
desc: BatchExecutionDescriptor,
@@ -115,7 +116,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
attn_metadata, slot_mappings = attn_state
def forward(cg_mode: CUDAGraphMode) -> None:
precompute_context_kv(num_reqs)
prepare_forward(num_reqs, attn_metadata)
forward_fn(
num_reqs,
num_tokens,
@@ -166,12 +166,18 @@ class DFlashSpeculator(DraftModelSpeculator):
self.kv_cache_config,
self.max_model_len,
causal=self._group_causal,
precompute_context_kv=lambda num_reqs: self._precompute_context_kv(
0, self._num_graph_context_tokens(num_reqs)
),
prepare_forward=self._prepare_graph_forward,
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
)
def _prepare_graph_forward(
self, num_reqs: int, attn_metadata: dict[str, Any] | None
) -> None:
self._precompute_context_kv(0, self._num_graph_context_tokens(num_reqs))
# Recorded in the FULL graph so replays skip the eager metadata build.
if attn_metadata is not None and self.supports_decode_metadata_update:
self._update_draft_decode_metadata(attn_metadata, num_reqs)
def load_draft_model(
self,
target_model: nn.Module,
@@ -486,29 +492,32 @@ class DFlashSpeculator(DraftModelSpeculator):
else:
self._precompute_context_kv(0, num_target_tokens, dummy_run)
# Rebuild the draft attention metadata even when replaying the FULL
# graph so that any attention metadata builder state is updated.
draft_attn_metadata = self._build_uniform_attn_metadata(
num_reqs=num_reqs,
batch_desc=batch_desc,
num_query_per_req=self.num_query_per_req,
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound,
step=self.num_query_per_req,
causal=self._group_causal,
)
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
self.block_tables.slot_mappings[:, :num_tokens_padded],
self.kv_cache_config,
)
is_full_cudagraph = batch_desc.cg_mode == CUDAGraphMode.FULL
# During FULL cudagraph with a model that supports decode metadata updates,
# the metadata update is captured in the graph, and we can skip the full
# build below.
if not (is_full_cudagraph and self.supports_decode_metadata_update):
draft_attn_metadata = self._build_uniform_attn_metadata(
num_reqs=num_reqs,
batch_desc=batch_desc,
num_query_per_req=self.num_query_per_req,
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound,
step=self.num_query_per_req,
causal=self._group_causal,
)
# DFlash processes all speculative tokens in one forward pass,
# so the real token count is num_query_tokens.
self._prepare_eplb_forward(num_query_tokens)
if batch_desc.cg_mode == CUDAGraphMode.FULL:
if is_full_cudagraph:
assert self.query_cudagraph_manager is not None
self.query_cudagraph_manager.run_fullgraph(batch_desc)
else:
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
self.block_tables.slot_mappings[:, :num_tokens_padded],
self.kv_cache_config,
)
self._generate_draft(
num_reqs,
num_tokens_padded,
@@ -207,6 +207,9 @@ class DraftModelSpeculator(BaseSpeculator):
self.supports_mm_inputs = False
self.pcp_manager: PCPManager | None = None
# Whether every draft attention builder can refresh its decode metadata
# in place, so the refresh can be recorded in the draft CUDA graph.
self.supports_decode_metadata_update = False
@abstractmethod
def load_draft_model(
@@ -299,6 +302,24 @@ class DraftModelSpeculator(BaseSpeculator):
# builders and buffers.
self.target_input_buffers = target_input_buffers
self.target_attn_groups = target_attn_groups
self._configure_decode_metadata_update()
def _configure_decode_metadata_update(self) -> None:
unsupported_backends = sorted(
{
attn_group.backend.get_name()
for attn_groups in self.attn_groups
for attn_group in attn_groups
if not attn_group.supports_draft_decode_metadata_update
}
)
self.supports_decode_metadata_update = not unsupported_backends
if unsupported_backends:
logger.info_once(
"Attention backend(s) %s cannot update draft decode metadata in "
"place; falling back to rebuilding attention metadata eagerly.",
", ".join(unsupported_backends),
)
def _build_attn_metadata(
self,
@@ -10,18 +10,15 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count
from vllm.v1.worker.utils import get_uniform_decode_token_count
logger = init_logger(__name__)
@@ -51,7 +48,6 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.use_fused_multi_step_decode = False
def load_model(self, target_model: nn.Module) -> None:
super().load_model(target_model)
@@ -83,44 +79,12 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
"""
return True
def set_attn(
self,
model_state: ModelState,
kv_cache_config: KVCacheConfig,
block_tables: BlockTables,
target_input_buffers: InputBuffers,
target_attn_groups: list[list[AttentionGroup]],
) -> None:
super().set_attn(
model_state,
kv_cache_config,
block_tables,
target_input_buffers,
target_attn_groups,
)
self._configure_fused_multi_step_decode()
def _configure_fused_multi_step_decode(self) -> None:
def _configure_decode_metadata_update(self) -> None:
# With one speculative step there are no draft decode steps to update.
if self.num_speculative_steps == 1:
self.use_fused_multi_step_decode = False
self.supports_decode_metadata_update = False
return
unsupported_backends = sorted(
{
attn_group.backend.get_name()
for attn_groups in self.attn_groups
for attn_group in attn_groups
if not attn_group.supports_draft_decode_metadata_update
}
)
self.use_fused_multi_step_decode = not unsupported_backends
if unsupported_backends:
logger.info_once(
"Fused multi-step draft decode is not supported by attention "
"backend(s) %s; falling back to rebuilding attention metadata "
"between draft steps.",
", ".join(unsupported_backends),
)
super()._configure_decode_metadata_update()
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# Initialize cudagraph manager for draft prefill (draft position 0).
@@ -181,7 +145,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
assert self.decode_cudagraph_manager is not None
decode_fn = (
self._generate_fused_drafts
if self.use_fused_multi_step_decode
if self.supports_decode_metadata_update
else self._generate_draft
)
self.decode_cudagraph_manager.capture(
@@ -192,7 +156,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
self.attn_groups,
self.kv_cache_config,
progress_bar_desc="Capturing decode CUDA graphs",
specialize_spec_tokens=self.use_fused_multi_step_decode
specialize_spec_tokens=self.supports_decode_metadata_update
and self.speculative_config.uses_dynamic_speculative_decoding(),
)
self.on_multi_step_decode_end(self.max_num_reqs)
@@ -384,7 +348,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
# Generate the remaining num_speculative_steps - 1 draft tokens.
decode_fn = (
self._fused_multi_step_decode
if self.use_fused_multi_step_decode
if self.supports_decode_metadata_update
else self._multi_step_decode
)
decode_fn(