[Perf][MRV2] Reuse Mamba/GDN metadata across KV cache groups (#58762)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: Xin Yang <xyangx@amazon.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Xin Yang
Claude Opus 5.5
Codex
Lucas Wilkinson
parent
55de40a2fc
commit
dd3bc9c72f
@@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = {
|
||||
"prefill_state_indices",
|
||||
"prefill_has_initial_state",
|
||||
"spec_sequence_masks",
|
||||
"spec_sequence_masks_cpu",
|
||||
"uniform_spec_sequence_length",
|
||||
"flashinfer_prefill_query_start_loc",
|
||||
"flashinfer_prefill_seq_order",
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
@@ -75,6 +76,17 @@ GDN_BUILD_TEST_CASES = {
|
||||
expected_num_prefill_tokens=0,
|
||||
expected_num_spec_decodes=0,
|
||||
),
|
||||
# No speculative config, decode alongside prefill
|
||||
"regular_decode_with_prefill": GDNBuildTestCase(
|
||||
seq_lens=[40, 100],
|
||||
query_lens=[1, 50],
|
||||
num_decode_draft_tokens=None,
|
||||
num_speculative_tokens=0,
|
||||
expected_num_decodes=1,
|
||||
expected_num_prefills=1,
|
||||
expected_num_prefill_tokens=50,
|
||||
expected_num_spec_decodes=0,
|
||||
),
|
||||
# Multi-token prefill alongside spec decode — no decode to reclassify
|
||||
"spec_decode_with_real_prefill": GDNBuildTestCase(
|
||||
seq_lens=[100, 20],
|
||||
@@ -155,9 +167,12 @@ def _build(
|
||||
builder: GDNAttentionMetadataBuilder,
|
||||
batch_spec: BatchSpec,
|
||||
num_decode_draft_tokens: list[int] | None = None,
|
||||
block_table: torch.Tensor | None = None,
|
||||
) -> GDNAttentionMetadata:
|
||||
"""Build GDN attention metadata, optionally with spec-decode kwargs."""
|
||||
common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE)
|
||||
if block_table is not None:
|
||||
common = common.replace(block_table_tensor=block_table)
|
||||
kwargs: dict = {}
|
||||
if num_decode_draft_tokens is not None:
|
||||
kwargs["num_decode_draft_tokens_cpu"] = torch.tensor(
|
||||
@@ -184,6 +199,54 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
|
||||
assert meta.num_spec_decodes == test_case.expected_num_spec_decodes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
@pytest.mark.parametrize("full_cuda_graph", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"test_case", GDN_BUILD_TEST_CASES.values(), ids=GDN_BUILD_TEST_CASES.keys()
|
||||
)
|
||||
def test_update_block_table_matches_build(
|
||||
test_case: GDNBuildTestCase, full_cuda_graph: bool, mamba_cache_mode: str
|
||||
):
|
||||
"""update_block_table() on another group's metadata matches build()."""
|
||||
batch = BatchSpec(seq_lens=test_case.seq_lens, query_lens=test_case.query_lens)
|
||||
src, dst, ref = (
|
||||
_create_gdn_builder(test_case.num_speculative_tokens, full_cuda_graph)
|
||||
for _ in range(3)
|
||||
)
|
||||
for builder in (src, dst, ref):
|
||||
builder.vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE)
|
||||
if mamba_cache_mode == "align":
|
||||
# MRV2 precomputes these for every Mamba group each step.
|
||||
dst.mamba_aligned_state_indices = ref.mamba_aligned_state_indices = (
|
||||
mamba_get_block_table_tensor(
|
||||
common.block_table_tensor, common.seq_lens, ref.kv_cache_spec, "align"
|
||||
)
|
||||
)
|
||||
draft_tokens = test_case.num_decode_draft_tokens
|
||||
expected = _build(ref, batch, draft_tokens, common.block_table_tensor)
|
||||
source = _build(src, batch, draft_tokens)
|
||||
fields = (
|
||||
"spec_state_indices_tensor",
|
||||
"non_spec_state_indices_tensor",
|
||||
"prefill_state_indices",
|
||||
)
|
||||
source_indices = [getattr(source, f) for f in fields]
|
||||
source_indices = [t if t is None else t.clone() for t in source_indices]
|
||||
meta = dst.update_block_table(
|
||||
source, common.block_table_tensor, common.slot_mapping
|
||||
)
|
||||
|
||||
for field, source_index in zip(fields, source_indices):
|
||||
actual = getattr(meta, field)
|
||||
torch.testing.assert_close(actual, getattr(expected, field))
|
||||
# The source group's indices are untouched.
|
||||
torch.testing.assert_close(getattr(source, field), source_index)
|
||||
# FULL graph state indices land in this group's own buffers.
|
||||
if full_cuda_graph and meta.num_prefills == 0 and actual is not None:
|
||||
assert actual.data_ptr() == getattr(dst, field).data_ptr()
|
||||
|
||||
|
||||
def test_has_initial_state_after_reclassification():
|
||||
"""After reclassification, num_prefills > 0 so the prefill kernel path
|
||||
should compute has_initial_state. For the reclassified request with
|
||||
|
||||
@@ -98,7 +98,9 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
speculator.attn_groups = [
|
||||
[
|
||||
SimpleNamespace(
|
||||
get_metadata_builder=lambda _: SimpleNamespace(build=build),
|
||||
get_metadata_builder=lambda _: SimpleNamespace(
|
||||
build=build, supports_update_block_table=False
|
||||
),
|
||||
layer_names=["draft"],
|
||||
)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user