[Perf][GDN] Slice pure spec-decode rows instead of a host-mask gather (#58763)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
Luca Motz
2026-10-01 06:53:06 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Vadim Gimpelson
parent 4a9d6f9572
commit 87a4bf664f
2 changed files with 27 additions and 7 deletions
@@ -65,6 +65,17 @@ GDN_BUILD_TEST_CASES = {
expected_num_prefill_tokens=0,
expected_num_spec_decodes=2,
),
# Padded (CUDA graph) sequences trail the spec decodes
"pure_spec_decode_with_padding": GDNBuildTestCase(
seq_lens=[50, 30, 16],
query_lens=[3, 3, 0],
num_decode_draft_tokens=[2, 2, -1],
num_speculative_tokens=2,
expected_num_decodes=0,
expected_num_prefills=0,
expected_num_prefill_tokens=0,
expected_num_spec_decodes=2,
),
# No speculative config at all — standard decode path
"pure_regular_decode": GDNBuildTestCase(
seq_lens=[40, 30, 20],
@@ -143,8 +154,9 @@ def _create_gdn_builder(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
if full_cuda_graph:
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
vllm_config.compilation_config.cudagraph_mode = (
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
)
if num_speculative_tokens > 0:
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
@@ -197,6 +209,8 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
assert meta.num_prefills == test_case.expected_num_prefills
assert meta.num_prefill_tokens == test_case.expected_num_prefill_tokens
assert meta.num_spec_decodes == test_case.expected_num_spec_decodes
if meta.spec_state_indices_tensor is not None:
assert len(meta.spec_state_indices_tensor) == meta.num_spec_decodes
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
+11 -5
View File
@@ -361,9 +361,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
non_spec_token_indx = torch.empty(
0, dtype=torch.int32, device=query_start_loc.device
)
# Filter by spec_sequence_masks to exclude padded sequences
# Padded sequences trail the spec decodes, so slice them off
# rather than gather with the host mask (an H2D copy + a kernel).
spec_state_indices_tensor = block_table_tensor[
spec_sequence_masks_cpu, : self.num_spec + 1
:num_spec_decodes, : self.num_spec + 1
]
non_spec_state_indices_tensor = None
# Padded sequences are always at the back, so the first
@@ -421,7 +422,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
)
assert num_accepted_tokens is not None
num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu]
if num_prefills == 0 and num_decodes == 0:
num_accepted_tokens = num_accepted_tokens[:num_spec_decodes]
else:
num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu]
chunk_indices: torch.Tensor | None = None
chunk_offsets: torch.Tensor | None = None
@@ -626,10 +630,12 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
non_spec_indices = blk_table[:, 0]
if m.num_prefills > 0:
prefill_indices = non_spec_indices[m.num_decodes :]
elif m.num_prefills == 0:
# Same as build(): padded sequences trail the spec decodes.
spec_indices = blk_table[: m.num_spec_decodes, : self.num_spec + 1]
else:
spec_indices = blk_table[masks, : self.num_spec + 1]
if m.num_prefills > 0:
non_spec_indices = prefill_indices = blk_table[~masks, 0]
non_spec_indices = prefill_indices = blk_table[~masks, 0]
if self._stage_spec_decode(
m.num_prefills, m.num_decodes, m.num_spec_decodes, m.num_spec_decode_tokens