From 87a4bf664f4b2344eced5a10add977b750d4b7d5 Mon Sep 17 00:00:00 2001 From: Luca Motz Date: Thu, 1 Oct 2026 08:53:06 +0200 Subject: [PATCH] [Perf][GDN] Slice pure spec-decode rows instead of a host-mask gather (#58763) Signed-off-by: Luca Motz Co-authored-by: Claude Opus 5.5 Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> --- .../v1/attention/test_gdn_metadata_builder.py | 18 ++++++++++++++++-- vllm/v1/attention/backends/gdn_attn.py | 16 +++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index 0256a13141..70a77f5929 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -65,6 +65,17 @@ GDN_BUILD_TEST_CASES = { expected_num_prefill_tokens=0, expected_num_spec_decodes=2, ), + # Padded (CUDA graph) sequences trail the spec decodes + "pure_spec_decode_with_padding": GDNBuildTestCase( + seq_lens=[50, 30, 16], + query_lens=[3, 3, 0], + num_decode_draft_tokens=[2, 2, -1], + num_speculative_tokens=2, + expected_num_decodes=0, + expected_num_prefills=0, + expected_num_prefill_tokens=0, + expected_num_spec_decodes=2, + ), # No speculative config at all — standard decode path "pure_regular_decode": GDNBuildTestCase( seq_lens=[40, 30, 20], @@ -143,8 +154,9 @@ def _create_gdn_builder( model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE, ) - if full_cuda_graph: - vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE + vllm_config.compilation_config.cudagraph_mode = ( + CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE + ) if num_speculative_tokens > 0: vllm_config.speculative_config = SpeculativeConfig( method="ngram", @@ -197,6 +209,8 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase): assert meta.num_prefills == test_case.expected_num_prefills assert meta.num_prefill_tokens == test_case.expected_num_prefill_tokens assert meta.num_spec_decodes == test_case.expected_num_spec_decodes + if meta.spec_state_indices_tensor is not None: + assert len(meta.spec_state_indices_tensor) == meta.num_spec_decodes @pytest.mark.parametrize("mamba_cache_mode", ["none", "align"]) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index eb616644f3..6b2c6b292b 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -361,9 +361,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] non_spec_token_indx = torch.empty( 0, dtype=torch.int32, device=query_start_loc.device ) - # Filter by spec_sequence_masks to exclude padded sequences + # Padded sequences trail the spec decodes, so slice them off + # rather than gather with the host mask (an H2D copy + a kernel). spec_state_indices_tensor = block_table_tensor[ - spec_sequence_masks_cpu, : self.num_spec + 1 + :num_spec_decodes, : self.num_spec + 1 ] non_spec_state_indices_tensor = None # Padded sequences are always at the back, so the first @@ -421,7 +422,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] ) assert num_accepted_tokens is not None - num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu] + if num_prefills == 0 and num_decodes == 0: + num_accepted_tokens = num_accepted_tokens[:num_spec_decodes] + else: + num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu] chunk_indices: torch.Tensor | None = None chunk_offsets: torch.Tensor | None = None @@ -626,10 +630,12 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] non_spec_indices = blk_table[:, 0] if m.num_prefills > 0: prefill_indices = non_spec_indices[m.num_decodes :] + elif m.num_prefills == 0: + # Same as build(): padded sequences trail the spec decodes. + spec_indices = blk_table[: m.num_spec_decodes, : self.num_spec + 1] else: spec_indices = blk_table[masks, : self.num_spec + 1] - if m.num_prefills > 0: - non_spec_indices = prefill_indices = blk_table[~masks, 0] + non_spec_indices = prefill_indices = blk_table[~masks, 0] if self._stage_spec_decode( m.num_prefills, m.num_decodes, m.num_spec_decodes, m.num_spec_decode_tokens