[Perf][GDN] Slice pure spec-decode rows instead of a host-mask gather (#58763)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Vadim Gimpelson
parent
4a9d6f9572
commit
87a4bf664f
@@ -65,6 +65,17 @@ GDN_BUILD_TEST_CASES = {
|
||||
expected_num_prefill_tokens=0,
|
||||
expected_num_spec_decodes=2,
|
||||
),
|
||||
# Padded (CUDA graph) sequences trail the spec decodes
|
||||
"pure_spec_decode_with_padding": GDNBuildTestCase(
|
||||
seq_lens=[50, 30, 16],
|
||||
query_lens=[3, 3, 0],
|
||||
num_decode_draft_tokens=[2, 2, -1],
|
||||
num_speculative_tokens=2,
|
||||
expected_num_decodes=0,
|
||||
expected_num_prefills=0,
|
||||
expected_num_prefill_tokens=0,
|
||||
expected_num_spec_decodes=2,
|
||||
),
|
||||
# No speculative config at all — standard decode path
|
||||
"pure_regular_decode": GDNBuildTestCase(
|
||||
seq_lens=[40, 30, 20],
|
||||
@@ -143,8 +154,9 @@ def _create_gdn_builder(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
if full_cuda_graph:
|
||||
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
vllm_config.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
|
||||
)
|
||||
if num_speculative_tokens > 0:
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
@@ -197,6 +209,8 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
|
||||
assert meta.num_prefills == test_case.expected_num_prefills
|
||||
assert meta.num_prefill_tokens == test_case.expected_num_prefill_tokens
|
||||
assert meta.num_spec_decodes == test_case.expected_num_spec_decodes
|
||||
if meta.spec_state_indices_tensor is not None:
|
||||
assert len(meta.spec_state_indices_tensor) == meta.num_spec_decodes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
|
||||
@@ -361,9 +361,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
non_spec_token_indx = torch.empty(
|
||||
0, dtype=torch.int32, device=query_start_loc.device
|
||||
)
|
||||
# Filter by spec_sequence_masks to exclude padded sequences
|
||||
# Padded sequences trail the spec decodes, so slice them off
|
||||
# rather than gather with the host mask (an H2D copy + a kernel).
|
||||
spec_state_indices_tensor = block_table_tensor[
|
||||
spec_sequence_masks_cpu, : self.num_spec + 1
|
||||
:num_spec_decodes, : self.num_spec + 1
|
||||
]
|
||||
non_spec_state_indices_tensor = None
|
||||
# Padded sequences are always at the back, so the first
|
||||
@@ -421,7 +422,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
)
|
||||
|
||||
assert num_accepted_tokens is not None
|
||||
num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu]
|
||||
if num_prefills == 0 and num_decodes == 0:
|
||||
num_accepted_tokens = num_accepted_tokens[:num_spec_decodes]
|
||||
else:
|
||||
num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu]
|
||||
|
||||
chunk_indices: torch.Tensor | None = None
|
||||
chunk_offsets: torch.Tensor | None = None
|
||||
@@ -626,10 +630,12 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
non_spec_indices = blk_table[:, 0]
|
||||
if m.num_prefills > 0:
|
||||
prefill_indices = non_spec_indices[m.num_decodes :]
|
||||
elif m.num_prefills == 0:
|
||||
# Same as build(): padded sequences trail the spec decodes.
|
||||
spec_indices = blk_table[: m.num_spec_decodes, : self.num_spec + 1]
|
||||
else:
|
||||
spec_indices = blk_table[masks, : self.num_spec + 1]
|
||||
if m.num_prefills > 0:
|
||||
non_spec_indices = prefill_indices = blk_table[~masks, 0]
|
||||
non_spec_indices = prefill_indices = blk_table[~masks, 0]
|
||||
|
||||
if self._stage_spec_decode(
|
||||
m.num_prefills, m.num_decodes, m.num_spec_decodes, m.num_spec_decode_tokens
|
||||
|
||||
Reference in New Issue
Block a user