From 36f397281295f5b58852a7eaf429b90348161079 Mon Sep 17 00:00:00 2001 From: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:09:10 -0700 Subject: [PATCH] [Spec Decode] Skip metadata build for dflash/dspark when all model backends support it (#60692) Signed-off-by: Giancarlo Delfin Co-authored-by: Rishi Puri <20074092+puririshi98@users.noreply.github.com> --- .../attention/test_deepseek_v4_swa_visible.py | 54 ++++++ .../test_triton_attn_metadata_builder.py | 154 ++++++++++++++++++ .../v1/attention/test_triton_attn_segments.py | 73 --------- ...test_gpu_target_dependent_ar_speculator.py | 22 +++ .../deepseek_v41/nvidia/flashinfer_sparse.py | 44 +++-- vllm/v1/attention/backends/mla/sparse_swa.py | 55 +++++-- vllm/v1/attention/backends/triton_attn.py | 3 + .../gpu/spec_decode/dflash/cudagraph.py | 9 +- .../gpu/spec_decode/dflash/speculator.py | 45 +++-- vllm/v1/worker/gpu/spec_decode/speculator.py | 21 +++ .../target_dependent_ar/speculator.py | 52 +----- 11 files changed, 360 insertions(+), 172 deletions(-) create mode 100644 tests/v1/attention/test_triton_attn_metadata_builder.py delete mode 100644 tests/v1/attention/test_triton_attn_segments.py diff --git a/tests/v1/attention/test_deepseek_v4_swa_visible.py b/tests/v1/attention/test_deepseek_v4_swa_visible.py index 8f08929549..5ccd721808 100644 --- a/tests/v1/attention/test_deepseek_v4_swa_visible.py +++ b/tests/v1/attention/test_deepseek_v4_swa_visible.py @@ -828,3 +828,57 @@ def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start(): # Decode row is copied through; prefill rows follow the bounded window. assert bounded[0].cpu().tolist() == plain[0].cpu().tolist() assert bounded[1:].cpu().tolist() == rows[1:] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_dspark_noncausal_draft_decode_update_matches_build(): + """A DSpark draft graph refreshes its non-causal SWA metadata in place + instead of rebuilding it, so the refresh must match a fresh build.""" + from vllm.v1.attention.backends.mla.compressor_utils import ( + get_dspark_swa_index_width, + ) + + num_spec = 4 + query_lens = [num_spec] * 3 + + def make_dspark_builder(): + builder = make_builder(vision=False) + builder.is_dspark = True + builder.noncausal_index_width = get_dspark_swa_index_width(WINDOW, num_spec) + builder.decode_threshold = num_spec + return builder + + def common_metadata(seq_lens): + query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch( + seq_lens, query_lens, torch.device("cuda") + ) + return CommonAttentionMetadata( + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc.cpu(), + seq_lens=seq_lens_t, + seq_lens_cpu_upper_bound=seq_lens_t.cpu(), + num_reqs=len(seq_lens), + num_actual_tokens=int(query_start_loc[-1]), + max_query_len=num_spec, + max_seq_len=max(seq_lens), + block_table_tensor=block_table, + slot_mapping=slot_mapping, + causal=False, + ) + + captured_seq_lens = [6, 40, 130] + replay_seq_lens = [130, 9, 90] + builder = make_dspark_builder() + captured = common_metadata(captured_seq_lens) + md = builder.build(0, captured) + replay = common_metadata(replay_seq_lens) + expected = make_dspark_builder().build(0, replay) + + # A replay sees new inputs through the same device buffers. + captured.seq_lens.copy_(replay.seq_lens) + captured.block_table_tensor.copy_(replay.block_table_tensor) + builder.update_draft_decode_metadata(md) + + assert md.decode_swa_width == builder.noncausal_index_width + torch.testing.assert_close(md.decode_swa_lens, expected.decode_swa_lens) + torch.testing.assert_close(md.decode_swa_indices, expected.decode_swa_indices) diff --git a/tests/v1/attention/test_triton_attn_metadata_builder.py b/tests/v1/attention/test_triton_attn_metadata_builder.py new file mode 100644 index 0000000000..9ec8145566 --- /dev/null +++ b/tests/v1/attention/test_triton_attn_metadata_builder.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata +from vllm.config import CUDAGraphMode +from vllm.platforms import current_platform +from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder +from vllm.v1.attention.backends.triton_attn_diffkv import ( + TritonAttentionDiffKVMetadataBuilder, +) +from vllm.v1.kv_cache_interface import FullAttentionSpec + + +# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and +# the 64-segment scratch holds every query token (e.g. multi-query verify). +@pytest.mark.parametrize( + "builder_cls", + [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder], +) +@pytest.mark.parametrize( + "capability,num_kv_heads,num_seqs,query_len,segments", + [ + ((12, 0), 1, 11, 1, 64), + ((12, 0), 1, 12, 1, 16), + ((12, 0), 8, 1, 1, 64), + ((12, 0), 8, 2, 1, 16), + ((9, 0), 1, 1, 1, 16), + ((12, 0), 1, 2, 4, 64), + ((12, 0), 1, 10, 4, 16), + ], +) +def test_split_k_segments_follow_sm_occupancy( + monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments +): + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr( + current_platform, "is_device_capability", lambda cap: cap == capability + ) + monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188) + config = SimpleNamespace( + model_config=SimpleNamespace( + get_num_attention_heads=lambda _: 8, + get_num_kv_heads=lambda _: num_kv_heads, + get_head_size=lambda: 128, + rswa_window=None, + ), + parallel_config=None, + scheduler_config=SimpleNamespace(max_num_seqs=16), + speculative_config=None, + compilation_config=SimpleNamespace( + cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={} + ), + ) + spec = FullAttentionSpec( + block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16 + ) + builder = builder_cls(spec, ["layer.0"], config, "cpu") + batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs) + metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu")) + assert metadata.num_par_softmax_segments == segments + assert metadata.softmax_segm_output.shape[2] == segments + assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len + # DiffKV re-allocates the output; it must stay sized like the base buffers. + assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0] + assert ( + metadata.softmax_segm_output.data_ptr() + == builder.softmax_segm_output.data_ptr() + ) + + +def _draft_builder(builder_cls, rswa_window: int | None): + config = SimpleNamespace( + model_config=SimpleNamespace( + get_num_attention_heads=lambda _: 8, + get_num_kv_heads=lambda _: 1, + get_head_size=lambda: 128, + rswa_window=rswa_window, + ), + parallel_config=None, + scheduler_config=SimpleNamespace(max_num_seqs=16), + speculative_config=None, + compilation_config=SimpleNamespace( + cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={} + ), + ) + spec = FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16 + ) + return builder_cls(spec, ["layer.0"], config, "cpu") + + +def _draft_decode_batch(seq_lens: list[int]): + batch = BatchSpec(seq_lens=seq_lens, query_lens=[1] * len(seq_lens)) + return create_common_attn_metadata(batch, 16, "cpu") + + +@pytest.mark.parametrize( + "builder_cls", + [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder], +) +@pytest.mark.parametrize("rswa_window", [None, 64]) +def test_draft_decode_update_flag_matches_build_writes(builder_cls, rswa_window): + """The update is a no-op, so the builder may claim support only when + build() writes no builder-owned tensor that a captured graph reads.""" + builder = _draft_builder(builder_cls, rswa_window) + common_metadata = _draft_decode_batch([5, 9]) + common_metadata.rswa_prefix_lens = torch.tensor([3, 7], dtype=torch.int32) + + def owned_bytes(): + # Bytes, not values: uninitialized scratch may hold NaNs. + return { + name: value.view(torch.uint8).clone() + for name, value in vars(builder).items() + if isinstance(value, torch.Tensor) + } + + before = owned_bytes() + builder.build(0, common_metadata) + after = owned_bytes() + + written = [name for name in before if not torch.equal(before[name], after[name])] + assert builder.supports_draft_decode_metadata_update is (not written) + assert builder.supports_draft_decode_metadata_update is (rswa_window is None) + + +@pytest.mark.parametrize( + "builder_cls", + [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder], +) +def test_draft_decode_update_matches_fresh_build(builder_cls): + """A replayed graph reads metadata built at capture; after an in-place + update it must match a fresh build of the replayed batch.""" + torch.manual_seed(0) + builder = _draft_builder(builder_cls, rswa_window=None) + captured = _draft_decode_batch([100, 120]) + metadata = builder.build(0, captured) + + replayed = _draft_decode_batch([128, 30]) + # The replayed batch arrives through the same persistent input buffers. + captured.seq_lens.copy_(replayed.seq_lens) + captured.block_table_tensor.copy_(replayed.block_table_tensor) + captured.slot_mapping.copy_(replayed.slot_mapping) + builder.update_draft_decode_metadata(metadata) + + expected = _draft_builder(builder_cls, rswa_window=None).build(0, replayed) + assert torch.equal(metadata.seq_lens, expected.seq_lens) + assert torch.equal(metadata.block_table, expected.block_table) + assert torch.equal(metadata.slot_mapping, expected.slot_mapping) + assert torch.equal(metadata.query_start_loc, expected.query_start_loc) diff --git a/tests/v1/attention/test_triton_attn_segments.py b/tests/v1/attention/test_triton_attn_segments.py deleted file mode 100644 index 874c61d237..0000000000 --- a/tests/v1/attention/test_triton_attn_segments.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from types import SimpleNamespace - -import pytest -import torch - -from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata -from vllm.config import CUDAGraphMode -from vllm.platforms import current_platform -from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder -from vllm.v1.attention.backends.triton_attn_diffkv import ( - TritonAttentionDiffKVMetadataBuilder, -) -from vllm.v1.kv_cache_interface import FullAttentionSpec - - -# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and -# the 64-segment scratch holds every query token (e.g. multi-query verify). -@pytest.mark.parametrize( - "builder_cls", - [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder], -) -@pytest.mark.parametrize( - "capability,num_kv_heads,num_seqs,query_len,segments", - [ - ((12, 0), 1, 11, 1, 64), - ((12, 0), 1, 12, 1, 16), - ((12, 0), 8, 1, 1, 64), - ((12, 0), 8, 2, 1, 16), - ((9, 0), 1, 1, 1, 16), - ((12, 0), 1, 2, 4, 64), - ((12, 0), 1, 10, 4, 16), - ], -) -def test_split_k_segments_follow_sm_occupancy( - monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments -): - monkeypatch.setattr(current_platform, "is_cuda", lambda: True) - monkeypatch.setattr( - current_platform, "is_device_capability", lambda cap: cap == capability - ) - monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188) - config = SimpleNamespace( - model_config=SimpleNamespace( - get_num_attention_heads=lambda _: 8, - get_num_kv_heads=lambda _: num_kv_heads, - get_head_size=lambda: 128, - rswa_window=None, - ), - parallel_config=None, - scheduler_config=SimpleNamespace(max_num_seqs=16), - speculative_config=None, - compilation_config=SimpleNamespace( - cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={} - ), - ) - spec = FullAttentionSpec( - block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16 - ) - builder = builder_cls(spec, ["layer.0"], config, "cpu") - batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs) - metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu")) - assert metadata.num_par_softmax_segments == segments - assert metadata.softmax_segm_output.shape[2] == segments - assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len - # DiffKV re-allocates the output; it must stay sized like the base buffers. - assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0] - assert ( - metadata.softmax_segm_output.data_ptr() - == builder.softmax_segm_output.data_ptr() - ) diff --git a/tests/v1/worker/test_gpu_target_dependent_ar_speculator.py b/tests/v1/worker/test_gpu_target_dependent_ar_speculator.py index a3ec5f4eb7..490b84e778 100644 --- a/tests/v1/worker/test_gpu_target_dependent_ar_speculator.py +++ b/tests/v1/worker/test_gpu_target_dependent_ar_speculator.py @@ -609,3 +609,25 @@ def test_update_draft_decode_metadata_skips_without_scheduler_metadata(monkeypat assert not called assert metadata.scheduler_metadata is None + + +@pytest.mark.parametrize(("num_speculative_steps", "expected"), [(1, False), (3, True)]) +def test_decode_metadata_update_needs_draft_decode_steps( + num_speculative_steps, expected +): + """One speculative step runs no draft decodes, so the in-place metadata + update (and its fallback log) does not apply even on a capable backend.""" + speculator = object.__new__(_TestSpeculator) + speculator.num_speculative_steps = num_speculative_steps + speculator.attn_groups = [ + [ + SimpleNamespace( + supports_draft_decode_metadata_update=True, + backend=SimpleNamespace(get_name=lambda: "CAPABLE"), + ) + ] + ] + + speculator._configure_decode_metadata_update() + + assert speculator.supports_decode_metadata_update is expected diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 0e02b486a5..a5ab092839 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -206,24 +206,36 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui metadata = super().build( common_prefix_len, common_attn_metadata, fast_build, replay_start ) - num_tokens = metadata.num_decode_tokens - if not common_attn_metadata.causal and num_tokens > 0: - assert metadata.decode_swa_lens is not None - assert metadata.seq_lens is not None - assert metadata.token_to_req_indices is not None - topk_lens = self._decode_topk_lens[:num_tokens] - seq_lens = self._decode_seq_lens[:num_tokens] - torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens) - torch.index_select( - metadata.seq_lens, - 0, - metadata.token_to_req_indices[:num_tokens], - out=seq_lens, - ) - metadata.flashinfer_decode_topk_lens = topk_lens - metadata.flashinfer_decode_seq_lens = seq_lens + if not common_attn_metadata.causal and metadata.num_decode_tokens > 0: + self._fill_noncausal_decode_lens(metadata) return metadata + def update_draft_decode_metadata( + self, metadata: "DeepseekSparseSWAMetadata" + ) -> None: + super().update_draft_decode_metadata(metadata) + if metadata.flashinfer_decode_topk_lens is not None: + self._fill_noncausal_decode_lens(metadata) + + def _fill_noncausal_decode_lens( + self, metadata: "DeepseekSparseSWAMetadata" + ) -> None: + num_tokens = metadata.num_decode_tokens + assert metadata.decode_swa_lens is not None + assert metadata.seq_lens is not None + assert metadata.token_to_req_indices is not None + topk_lens = self._decode_topk_lens[:num_tokens] + seq_lens = self._decode_seq_lens[:num_tokens] + torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens) + torch.index_select( + metadata.seq_lens, + 0, + metadata.token_to_req_indices[:num_tokens], + out=seq_lens, + ) + metadata.flashinfer_decode_topk_lens = topk_lens + metadata.flashinfer_decode_seq_lens = seq_lens + class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend): @staticmethod diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index c5cbb4ef28..a691efe8bf 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -852,23 +852,44 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): num_tokens, ) - _COMPUTE_SWA_INDICES_AND_LENS_KERNEL( - metadata.decode_swa_indices, - metadata.decode_swa_lens, - metadata.decode_swa_indices.shape[-1], - metadata.decode_swa_indices.shape[-1], - metadata.decode_swa_lens, # unused (HAS_IMAGE=False) - metadata.decode_swa_lens, # unused (HAS_IMAGE=False) - metadata.query_start_loc, - metadata.seq_lens, - metadata.token_to_req_indices, - metadata.is_valid_token, - metadata.block_table, - self.block_size, - metadata.replay_start, - num_tokens=metadata.num_decode_tokens, - token_offset=0, - ) + # build() points non-causal (DSpark draft) metadata at its own buffer. + noncausal = self.decode_swa_indices_noncausal + if ( + noncausal is not None + and metadata.decode_swa_indices.data_ptr() == noncausal.data_ptr() + ): + _COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL( + metadata.decode_swa_indices, + metadata.decode_swa_lens, + self.window_size, + self.noncausal_index_width, + metadata.query_start_loc, + metadata.seq_lens, + metadata.token_to_req_indices, + metadata.is_valid_token, + metadata.block_table, + self.block_size, + num_tokens=num_tokens, + token_offset=0, + ) + else: + _COMPUTE_SWA_INDICES_AND_LENS_KERNEL( + metadata.decode_swa_indices, + metadata.decode_swa_lens, + metadata.decode_swa_indices.shape[-1], + metadata.decode_swa_indices.shape[-1], + metadata.decode_swa_lens, # unused (HAS_IMAGE=False) + metadata.decode_swa_lens, # unused (HAS_IMAGE=False) + metadata.query_start_loc, + metadata.seq_lens, + metadata.token_to_req_indices, + metadata.is_valid_token, + metadata.block_table, + self.block_size, + metadata.replay_start, + num_tokens=metadata.num_decode_tokens, + token_offset=0, + ) tile_sched = self.build_tile_scheduler(metadata.num_decode_tokens) metadata.tile_sched_swaonly = tile_sched[_LAYER_TYPE_SWAONLY] metadata.tile_sched_c4a = tile_sched[_LAYER_TYPE_C4A] diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index d8577b61f1..095163f144 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -193,6 +193,9 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet device=device, ) self.rswa_window = model_config.rswa_window + # R-SWA restages prefix lengths in build(), which the no-op + # update_draft_decode_metadata() does not refresh. + self.supports_draft_decode_metadata_update = self.rswa_window is None self.persistent_rswa_prefix_lens: torch.Tensor | None = None if self.rswa_window is not None: self.persistent_rswa_prefix_lens = torch.empty( diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py index e21ca9f870..2e189da0c9 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Callable, Mapping +from typing import Any import torch @@ -84,11 +85,11 @@ class DFlashCudaGraphManager(CudaGraphManager): kv_cache_config: KVCacheConfig, max_model_len: int, causal: bool | Mapping[int, bool], - precompute_context_kv: Callable[[int], None], + prepare_forward: Callable[[int, dict[str, Any] | None], None], progress_bar_desc: str = "Capturing CUDA graphs", ) -> None: - """``precompute_context_kv(num_reqs)`` is captured ahead of the query - forward.""" + """``prepare_forward(num_reqs, attn_metadata)`` is captured ahead of the + query forward.""" def create_forward_fn( desc: BatchExecutionDescriptor, @@ -115,7 +116,7 @@ class DFlashCudaGraphManager(CudaGraphManager): attn_metadata, slot_mappings = attn_state def forward(cg_mode: CUDAGraphMode) -> None: - precompute_context_kv(num_reqs) + prepare_forward(num_reqs, attn_metadata) forward_fn( num_reqs, num_tokens, diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index 884de5a264..5a13441ee8 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -166,12 +166,18 @@ class DFlashSpeculator(DraftModelSpeculator): self.kv_cache_config, self.max_model_len, causal=self._group_causal, - precompute_context_kv=lambda num_reqs: self._precompute_context_kv( - 0, self._num_graph_context_tokens(num_reqs) - ), + prepare_forward=self._prepare_graph_forward, progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs", ) + def _prepare_graph_forward( + self, num_reqs: int, attn_metadata: dict[str, Any] | None + ) -> None: + self._precompute_context_kv(0, self._num_graph_context_tokens(num_reqs)) + # Recorded in the FULL graph so replays skip the eager metadata build. + if attn_metadata is not None and self.supports_decode_metadata_update: + self._update_draft_decode_metadata(attn_metadata, num_reqs) + def load_draft_model( self, target_model: nn.Module, @@ -486,29 +492,32 @@ class DFlashSpeculator(DraftModelSpeculator): else: self._precompute_context_kv(0, num_target_tokens, dummy_run) - # Rebuild the draft attention metadata even when replaying the FULL - # graph so that any attention metadata builder state is updated. - draft_attn_metadata = self._build_uniform_attn_metadata( - num_reqs=num_reqs, - batch_desc=batch_desc, - num_query_per_req=self.num_query_per_req, - seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound, - step=self.num_query_per_req, - causal=self._group_causal, - ) - draft_slot_mappings_by_layer = build_slot_mappings_by_layer( - self.block_tables.slot_mappings[:, :num_tokens_padded], - self.kv_cache_config, - ) + is_full_cudagraph = batch_desc.cg_mode == CUDAGraphMode.FULL + # During FULL cudagraph with a model that supports decode metadata updates, + # the metadata update is captured in the graph, and we can skip the full + # build below. + if not (is_full_cudagraph and self.supports_decode_metadata_update): + draft_attn_metadata = self._build_uniform_attn_metadata( + num_reqs=num_reqs, + batch_desc=batch_desc, + num_query_per_req=self.num_query_per_req, + seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound, + step=self.num_query_per_req, + causal=self._group_causal, + ) # DFlash processes all speculative tokens in one forward pass, # so the real token count is num_query_tokens. self._prepare_eplb_forward(num_query_tokens) - if batch_desc.cg_mode == CUDAGraphMode.FULL: + if is_full_cudagraph: assert self.query_cudagraph_manager is not None self.query_cudagraph_manager.run_fullgraph(batch_desc) else: + draft_slot_mappings_by_layer = build_slot_mappings_by_layer( + self.block_tables.slot_mappings[:, :num_tokens_padded], + self.kv_cache_config, + ) self._generate_draft( num_reqs, num_tokens_padded, diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index c3e8dcbff4..d86a0b0351 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -207,6 +207,9 @@ class DraftModelSpeculator(BaseSpeculator): self.supports_mm_inputs = False self.pcp_manager: PCPManager | None = None + # Whether every draft attention builder can refresh its decode metadata + # in place, so the refresh can be recorded in the draft CUDA graph. + self.supports_decode_metadata_update = False @abstractmethod def load_draft_model( @@ -299,6 +302,24 @@ class DraftModelSpeculator(BaseSpeculator): # builders and buffers. self.target_input_buffers = target_input_buffers self.target_attn_groups = target_attn_groups + self._configure_decode_metadata_update() + + def _configure_decode_metadata_update(self) -> None: + unsupported_backends = sorted( + { + attn_group.backend.get_name() + for attn_groups in self.attn_groups + for attn_group in attn_groups + if not attn_group.supports_draft_decode_metadata_update + } + ) + self.supports_decode_metadata_update = not unsupported_backends + if unsupported_backends: + logger.info_once( + "Attention backend(s) %s cannot update draft decode metadata in " + "place; falling back to rebuilding attention metadata eagerly.", + ", ".join(unsupported_backends), + ) def _build_attn_metadata( self, diff --git a/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/speculator.py b/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/speculator.py index 02e28875b8..fdbedd78f6 100644 --- a/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/target_dependent_ar/speculator.py @@ -10,18 +10,15 @@ from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger from vllm.triton_utils import tl, triton -from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer -from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers -from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import ( SpeculatorCudaGraphManager, ) -from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count +from vllm.v1.worker.utils import get_uniform_decode_token_count logger = init_logger(__name__) @@ -51,7 +48,6 @@ class TargetDependentARSpeculator(DraftModelSpeculator): self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None - self.use_fused_multi_step_decode = False def load_model(self, target_model: nn.Module) -> None: super().load_model(target_model) @@ -83,44 +79,12 @@ class TargetDependentARSpeculator(DraftModelSpeculator): """ return True - def set_attn( - self, - model_state: ModelState, - kv_cache_config: KVCacheConfig, - block_tables: BlockTables, - target_input_buffers: InputBuffers, - target_attn_groups: list[list[AttentionGroup]], - ) -> None: - super().set_attn( - model_state, - kv_cache_config, - block_tables, - target_input_buffers, - target_attn_groups, - ) - self._configure_fused_multi_step_decode() - - def _configure_fused_multi_step_decode(self) -> None: + def _configure_decode_metadata_update(self) -> None: + # With one speculative step there are no draft decode steps to update. if self.num_speculative_steps == 1: - self.use_fused_multi_step_decode = False + self.supports_decode_metadata_update = False return - - unsupported_backends = sorted( - { - attn_group.backend.get_name() - for attn_groups in self.attn_groups - for attn_group in attn_groups - if not attn_group.supports_draft_decode_metadata_update - } - ) - self.use_fused_multi_step_decode = not unsupported_backends - if unsupported_backends: - logger.info_once( - "Fused multi-step draft decode is not supported by attention " - "backend(s) %s; falling back to rebuilding attention metadata " - "between draft steps.", - ", ".join(unsupported_backends), - ) + super()._configure_decode_metadata_update() def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: # Initialize cudagraph manager for draft prefill (draft position 0). @@ -181,7 +145,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator): assert self.decode_cudagraph_manager is not None decode_fn = ( self._generate_fused_drafts - if self.use_fused_multi_step_decode + if self.supports_decode_metadata_update else self._generate_draft ) self.decode_cudagraph_manager.capture( @@ -192,7 +156,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator): self.attn_groups, self.kv_cache_config, progress_bar_desc="Capturing decode CUDA graphs", - specialize_spec_tokens=self.use_fused_multi_step_decode + specialize_spec_tokens=self.supports_decode_metadata_update and self.speculative_config.uses_dynamic_speculative_decoding(), ) self.on_multi_step_decode_end(self.max_num_reqs) @@ -384,7 +348,7 @@ class TargetDependentARSpeculator(DraftModelSpeculator): # Generate the remaining num_speculative_steps - 1 draft tokens. decode_fn = ( self._fused_multi_step_decode - if self.use_fused_multi_step_decode + if self.supports_decode_metadata_update else self._multi_step_decode ) decode_fn(