[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)

Signed-off-by: Johnny-Liou <a897111@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Johnny-Liou
2026-07-24 09:39:49 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> tomeras91 Cyrus Leung
parent d02df748bf
commit 866fea2b99
23 changed files with 2954 additions and 31 deletions
@@ -42,6 +42,8 @@ def _make_vllm_config(
cache_config=SimpleNamespace(
block_size=block_size,
mamba_cache_mode="all",
use_replayssm=False,
replayssm_buffer_len=16,
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.FULL,
@@ -0,0 +1,256 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mamba2 ReplaySSM decode write-position derivation in
BaseMambaAttentionMetadataBuilder: write_pos and is_flush computed from the
per-request ring origin (replayssm_decode_base) and num_computed.
"""
from dataclasses import dataclass
import pytest
import torch
from tests.v1.attention.utils import (
BatchSpec,
MockMambaBuilder,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
DEVICE = torch.device("cpu")
@dataclass
class ReplaySSMBuildCase:
"""A decode batch and its expected per-row write_pos / is_flush.
num_computed = seq_len - query_len; write_pos =
(num_computed - decode_base) % buffer_len; is_flush = write_pos ==
buffer_len - 1 (or a forced one-token flush when num_computed < decode_base).
"""
seq_lens: list[int]
query_lens: list[int]
is_prefilling: list[bool]
decode_base: list[int]
buffer_len: int
expected_write_pos: list[int]
expected_is_flush: list[int]
mamba_cache_mode: str = "none"
REPLAYSSM_BUILD_CASES = {
# decode_base == num_prompt (fresh request).
"fresh_decode": ReplaySSMBuildCase(
seq_lens=[106],
query_lens=[1],
is_prefilling=[False],
decode_base=[100],
buffer_len=16,
expected_write_pos=[5],
expected_is_flush=[0],
),
# decode_base > num_prompt anchors write_pos at the resume point.
"resumed_reanchors_to_zero": ReplaySSMBuildCase(
seq_lens=[106],
query_lens=[1],
is_prefilling=[False],
decode_base=[105],
buffer_len=16,
expected_write_pos=[0],
expected_is_flush=[0],
),
# write_pos == buffer_len - 1 flushes.
"flush_boundary": ReplaySSMBuildCase(
seq_lens=[116],
query_lens=[1],
is_prefilling=[False],
decode_base=[100],
buffer_len=16,
expected_write_pos=[15],
expected_is_flush=[1],
),
# Resumed request landing on a flush boundary.
"resumed_flush_boundary": ReplaySSMBuildCase(
seq_lens=[121],
query_lens=[1],
is_prefilling=[False],
decode_base=[105],
buffer_len=16,
expected_write_pos=[15],
expected_is_flush=[1],
),
# Per-row write_pos / is_flush are independent.
"mixed_rows": ReplaySSMBuildCase(
seq_lens=[104, 106, 216],
query_lens=[1, 1, 1],
is_prefilling=[False, False, False],
decode_base=[100, 105, 200],
buffer_len=16,
expected_write_pos=[3, 0, 15],
expected_is_flush=[0, 0, 1],
),
# write_pos wraps within the buffer (6 % 4 == 2).
"small_buffer_wrap": ReplaySSMBuildCase(
seq_lens=[112],
query_lens=[1],
is_prefilling=[False],
decode_base=[105],
buffer_len=4,
expected_write_pos=[2],
expected_is_flush=[0],
),
# Single-token prefill-as-decode still in the prompt (num_computed <
# decode_base): forced one-token flush.
"leftover_prompt_one_token_flush": ReplaySSMBuildCase(
seq_lens=[100],
query_lens=[1],
is_prefilling=[True],
decode_base=[100],
buffer_len=16,
expected_write_pos=[0],
expected_is_flush=[1],
),
# Align mode (block_size 16). Past the first boundary the ring re-anchors at
# the block start: num_computed 117 -> block_start 112, write_pos 5 (vs 1 in
# none mode).
"align_reanchor_past_boundary": ReplaySSMBuildCase(
seq_lens=[118],
query_lens=[1],
is_prefilling=[False],
decode_base=[100],
buffer_len=16,
expected_write_pos=[5],
expected_is_flush=[0],
mamba_cache_mode="align",
),
# First-block boundary (num_computed+1 == 112) forces a flush even though
# write_pos (11) != buffer_len - 1.
"align_first_block_boundary_flush": ReplaySSMBuildCase(
seq_lens=[112],
query_lens=[1],
is_prefilling=[False],
decode_base=[100],
buffer_len=16,
expected_write_pos=[11],
expected_is_flush=[1],
mamba_cache_mode="align",
),
# First step of a new block re-anchors write_pos to 0.
"align_new_block_start_zero": ReplaySSMBuildCase(
seq_lens=[113],
query_lens=[1],
is_prefilling=[False],
decode_base=[100],
buffer_len=16,
expected_write_pos=[0],
expected_is_flush=[0],
mamba_cache_mode="align",
),
# block_size % buffer_len == 0: a later boundary lands on write_pos ==
# buffer_len - 1, so the boundary flush coincides with the natural flush.
"align_boundary_coincides_natural_flush": ReplaySSMBuildCase(
seq_lens=[128],
query_lens=[1],
is_prefilling=[False],
decode_base=[100],
buffer_len=16,
expected_write_pos=[15],
expected_is_flush=[1],
mamba_cache_mode="align",
),
# block_size % buffer_len != 0 (buffer_len 6): the boundary step still flushes
# although write_pos (3) != buffer_len - 1.
"align_unaligned_buffer_forces_flush": ReplaySSMBuildCase(
seq_lens=[128],
query_lens=[1],
is_prefilling=[False],
decode_base=[100],
buffer_len=6,
expected_write_pos=[3],
expected_is_flush=[1],
mamba_cache_mode="align",
),
# Per-row independence in align mode: partial-block / new-block / boundary.
"align_mixed_rows": ReplaySSMBuildCase(
seq_lens=[105, 113, 112],
query_lens=[1, 1, 1],
is_prefilling=[False, False, False],
decode_base=[100, 100, 100],
buffer_len=16,
expected_write_pos=[4, 0, 11],
expected_is_flush=[0, 0, 1],
mamba_cache_mode="align",
),
}
def _make_mamba_spec(buffer_len: int) -> MambaSpec:
# Five-tensor ReplaySSM page; the builder only reads shapes[4][0] (bc groups).
return MambaSpec(
block_size=BLOCK_SIZE,
shapes=(
(1, 1),
(1, 1, 1),
(1, buffer_len, 1),
(1, buffer_len),
(1, buffer_len, 1),
),
dtypes=(torch.float32,),
)
def _create_replayssm_builder(
buffer_len: int, mamba_cache_mode: str = "none"
) -> MockMambaBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE
)
# Set the flags after construction to skip validate_mamba_cached_kernel
# (it requires a Triton backend) on the mock model.
vllm_config.cache_config.use_replayssm = True
vllm_config.cache_config.replayssm_buffer_len = buffer_len
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
return MockMambaBuilder(
_make_mamba_spec(buffer_len), ["layer0"], vllm_config, DEVICE
)
def _build(builder: MockMambaBuilder, case: ReplaySSMBuildCase):
batch = BatchSpec(seq_lens=case.seq_lens, query_lens=case.query_lens)
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
is_prefilling=torch.tensor(case.is_prefilling, dtype=torch.bool),
replayssm_decode_base_cpu=torch.tensor(case.decode_base, dtype=torch.int32),
)
return builder.build(0, common)
@pytest.mark.parametrize(
"case", REPLAYSSM_BUILD_CASES.values(), ids=REPLAYSSM_BUILD_CASES.keys()
)
def test_replayssm_write_pos(case: ReplaySSMBuildCase):
builder = _create_replayssm_builder(case.buffer_len, case.mamba_cache_mode)
meta = _build(builder, case)
assert meta.write_pos_d is not None
assert meta.is_flush_d is not None
n = len(case.expected_write_pos)
assert meta.write_pos_d[:n].tolist() == case.expected_write_pos
assert meta.is_flush_d[:n].tolist() == case.expected_is_flush
def test_resumed_request_differs_from_fresh():
"""Same token count, different decode_base: fresh (base 100) -> write_pos 5,
resumed (base 105) -> write_pos 0."""
builder = _create_replayssm_builder(16)
batch = BatchSpec(seq_lens=[106, 106], query_lens=[1, 1])
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
is_prefilling=torch.tensor([False, False]),
replayssm_decode_base_cpu=torch.tensor([100, 105], dtype=torch.int32),
)
meta = builder.build(0, common)
assert meta.write_pos_d.tolist()[:2] == [5, 0]
assert meta.is_flush_d.tolist()[:2] == [0, 0]
+138
View File
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Engine-level parity: ReplaySSM standard decode vs the baseline SSM kernel."""
import pytest
from vllm.v1.metrics.reader import Counter
from ...models.utils import check_logprobs_close
from ...utils import large_gpu_mark, multi_gpu_test
# Mamba2 (Nemotron-3) hybrid.
MAMBA2_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
MODELS = [
pytest.param(MAMBA2_MODEL, marks=large_gpu_mark(min_gb=40)),
]
PROMPTS = [
"The capital of France is",
"Once upon a time, in a small village,",
]
def _check_replayssm_parity(vllm_runner, model_name, *, tensor_parallel_size=1):
# Compare logprobs, not greedy ids: ReplaySSM's fp arithmetic can flip a
# near-tie. Baseline and ReplaySSM run at the same TP, so TP numerics are
# common-mode and only ReplaySSM varies.
common = dict(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
tensor_parallel_size=tensor_parallel_size,
)
with vllm_runner(model_name, **common) as llm:
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
with vllm_runner(
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
) as llm:
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
check_logprobs_close(
outputs_0_lst=baseline,
outputs_1_lst=replay,
name_0="baseline",
name_1="replayssm",
)
@pytest.mark.parametrize("model_name", MODELS)
def test_replayssm_decode_matches_baseline(vllm_runner, model_name):
_check_replayssm_parity(vllm_runner, model_name)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("model_name", [MAMBA2_MODEL])
def test_replayssm_decode_matches_baseline_tp2(vllm_runner, model_name):
# Tensor-parallel correctness: ReplaySSM's caches and checkpoint state are
# sharded per rank, so TP2 decode must still match the baseline at TP2.
_check_replayssm_parity(vllm_runner, model_name, tensor_parallel_size=2)
# Prefix spans several mamba blocks; prefix caching only reuses full blocks.
_PC_SENTENCE = (
"In a detailed survey of state space models, the authors compared many "
"architectures across a wide range of long-context language tasks and "
"measured their throughput, memory use, and accuracy in careful detail. "
)
_PC_PREFIX = _PC_SENTENCE * 120
PREFIX_CACHING_PROMPTS = [
_PC_PREFIX + "The most important conclusion was that",
_PC_PREFIX + "Surprisingly, the experiments showed that",
_PC_PREFIX + "The most important conclusion was that",
]
def _prefix_cache_hits(llm) -> int:
return sum(
m.value
for m in llm.llm.get_metrics()
if isinstance(m, Counter) and m.name == "vllm:prefix_cache_hits"
)
def _check_replayssm_prefix_caching_parity(
vllm_runner, model_name, *, tensor_parallel_size=1
):
# align mode materializes the exact SSM state at each block boundary, so
# ReplaySSM's cached prefixes must match the always-materialized baseline.
common = dict(
max_model_len=8192,
trust_remote_code=True,
enable_prefix_caching=True,
enable_chunked_prefill=True,
mamba_cache_mode="align",
disable_log_stats=False, # required for llm.get_metrics()
tensor_parallel_size=tensor_parallel_size,
)
with vllm_runner(model_name, **common) as llm:
baseline = llm.generate_greedy_logprobs(
PREFIX_CACHING_PROMPTS, max_tokens=32, num_logprobs=5
)
with vllm_runner(
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
) as llm:
# Prime the cache, then measure, so cache hits are deterministic.
llm.generate_greedy_logprobs(
PREFIX_CACHING_PROMPTS, max_tokens=32, num_logprobs=5
)
replay = llm.generate_greedy_logprobs(
PREFIX_CACHING_PROMPTS, max_tokens=32, num_logprobs=5
)
replay_hits = _prefix_cache_hits(llm)
# Without real cache hits the cached path is never exercised.
assert replay_hits > 0, (
"ReplaySSM align-mode run produced no prefix-cache hits; the shared "
"prefix may be shorter than one mamba block, so prefix caching is inert"
)
check_logprobs_close(
outputs_0_lst=baseline,
outputs_1_lst=replay,
name_0="baseline_align_pc",
name_1="replayssm_align_pc",
)
@pytest.mark.parametrize("model_name", MODELS)
def test_replayssm_prefix_caching_matches_baseline(vllm_runner, model_name):
_check_replayssm_prefix_caching_parity(vllm_runner, model_name)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("model_name", [MAMBA2_MODEL])
def test_replayssm_prefix_caching_matches_baseline_tp2(vllm_runner, model_name):
_check_replayssm_prefix_caching_parity(
vllm_runner, model_name, tensor_parallel_size=2
)