[Perf][MRV2] Reuse Mamba/GDN metadata across KV cache groups (#58762)

Signed-off-by: Luca Motz <luca.motz@icloud.com>
Co-authored-by: Xin Yang <xyangx@amazon.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Luca Motz
2026-09-28 04:36:58 +00:00
committed by GitHub
co-authored by Xin Yang Claude Opus 5.5 Codex Lucas Wilkinson
parent 55de40a2fc
commit dd3bc9c72f
5 changed files with 189 additions and 21 deletions
@@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = {
"prefill_state_indices",
"prefill_has_initial_state",
"spec_sequence_masks",
"spec_sequence_masks_cpu",
"uniform_spec_sequence_length",
"flashinfer_prefill_query_start_loc",
"flashinfer_prefill_seq_order",
@@ -21,6 +21,7 @@ from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
@@ -75,6 +76,17 @@ GDN_BUILD_TEST_CASES = {
expected_num_prefill_tokens=0,
expected_num_spec_decodes=0,
),
# No speculative config, decode alongside prefill
"regular_decode_with_prefill": GDNBuildTestCase(
seq_lens=[40, 100],
query_lens=[1, 50],
num_decode_draft_tokens=None,
num_speculative_tokens=0,
expected_num_decodes=1,
expected_num_prefills=1,
expected_num_prefill_tokens=50,
expected_num_spec_decodes=0,
),
# Multi-token prefill alongside spec decode — no decode to reclassify
"spec_decode_with_real_prefill": GDNBuildTestCase(
seq_lens=[100, 20],
@@ -155,9 +167,12 @@ def _build(
builder: GDNAttentionMetadataBuilder,
batch_spec: BatchSpec,
num_decode_draft_tokens: list[int] | None = None,
block_table: torch.Tensor | None = None,
) -> GDNAttentionMetadata:
"""Build GDN attention metadata, optionally with spec-decode kwargs."""
common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE)
if block_table is not None:
common = common.replace(block_table_tensor=block_table)
kwargs: dict = {}
if num_decode_draft_tokens is not None:
kwargs["num_decode_draft_tokens_cpu"] = torch.tensor(
@@ -184,6 +199,54 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
assert meta.num_spec_decodes == test_case.expected_num_spec_decodes
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
@pytest.mark.parametrize("full_cuda_graph", [False, True])
@pytest.mark.parametrize(
"test_case", GDN_BUILD_TEST_CASES.values(), ids=GDN_BUILD_TEST_CASES.keys()
)
def test_update_block_table_matches_build(
test_case: GDNBuildTestCase, full_cuda_graph: bool, mamba_cache_mode: str
):
"""update_block_table() on another group's metadata matches build()."""
batch = BatchSpec(seq_lens=test_case.seq_lens, query_lens=test_case.query_lens)
src, dst, ref = (
_create_gdn_builder(test_case.num_speculative_tokens, full_cuda_graph)
for _ in range(3)
)
for builder in (src, dst, ref):
builder.vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE)
if mamba_cache_mode == "align":
# MRV2 precomputes these for every Mamba group each step.
dst.mamba_aligned_state_indices = ref.mamba_aligned_state_indices = (
mamba_get_block_table_tensor(
common.block_table_tensor, common.seq_lens, ref.kv_cache_spec, "align"
)
)
draft_tokens = test_case.num_decode_draft_tokens
expected = _build(ref, batch, draft_tokens, common.block_table_tensor)
source = _build(src, batch, draft_tokens)
fields = (
"spec_state_indices_tensor",
"non_spec_state_indices_tensor",
"prefill_state_indices",
)
source_indices = [getattr(source, f) for f in fields]
source_indices = [t if t is None else t.clone() for t in source_indices]
meta = dst.update_block_table(
source, common.block_table_tensor, common.slot_mapping
)
for field, source_index in zip(fields, source_indices):
actual = getattr(meta, field)
torch.testing.assert_close(actual, getattr(expected, field))
# The source group's indices are untouched.
torch.testing.assert_close(getattr(source, field), source_index)
# FULL graph state indices land in this group's own buffers.
if full_cuda_graph and meta.num_prefills == 0 and actual is not None:
assert actual.data_ptr() == getattr(dst, field).data_ptr()
def test_has_initial_state_after_reclassification():
"""After reclassification, num_prefills > 0 so the prefill kernel path
should compute has_initial_state. For the reclassified request with
@@ -98,7 +98,9 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
speculator.attn_groups = [
[
SimpleNamespace(
get_metadata_builder=lambda _: SimpleNamespace(build=build),
get_metadata_builder=lambda _: SimpleNamespace(
build=build, supports_update_block_table=False
),
layer_names=["draft"],
)
]