[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-30 21:41:17 +00:00
committed by GitHub
co-authored by Andrii Skliar
parent 7e583e615c
commit deb148e730
18 changed files with 1029 additions and 113 deletions
@@ -5,7 +5,9 @@
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
MiniMaxText01LinearAttention,
)
@@ -19,6 +21,32 @@ from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend
@pytest.mark.parametrize(("use_replayssm", "expected_blocks"), [(False, 3), (True, 0)])
def test_replayssm_does_not_reserve_speculative_state_blocks(
use_replayssm, expected_blocks
):
layer = SimpleNamespace(
get_state_shape=lambda: ((2,),),
get_state_dtype=lambda: (torch.float32,),
mamba_type=MambaAttentionBackendEnum.MAMBA2,
is_kv_cache_tp_replicated=False,
)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
mamba_block_size=1,
mamba_page_size_padded=None,
mamba_cache_mode="none",
use_replayssm=use_replayssm,
),
num_speculative_tokens=3,
)
spec = MambaBase.get_kv_cache_spec(layer, vllm_config)
assert spec is not None
assert spec.num_speculative_blocks == expected_blocks
@pytest.mark.parametrize(
"layer_class, init_kwargs, expected_backend, expected_mamba_type",
[
@@ -16,6 +16,8 @@ from tests.v1.attention.utils import (
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.mamba import MambaBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
@@ -191,9 +193,12 @@ REPLAYSSM_BUILD_CASES = {
def _make_mamba_spec(
buffer_len: int,
mamba_backend: MambaBackendEnum,
num_speculative_tokens: int = 0,
) -> MambaSpec:
ring_buffer_len = buffer_len + (
1 if mamba_backend == MambaBackendEnum.FLASHINFER else 0
1 + num_speculative_tokens
if mamba_backend == MambaBackendEnum.FLASHINFER
else 0
)
shapes = (
(1, 1),
@@ -214,6 +219,7 @@ def _create_replayssm_builder(
mamba_cache_mode: str = "none",
*,
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON,
num_speculative_tokens: int = 0,
) -> MockMambaBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE
@@ -224,21 +230,32 @@ def _create_replayssm_builder(
vllm_config.cache_config.replayssm_buffer_len = buffer_len
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
vllm_config.mamba_config.backend = mamba_backend
if num_speculative_tokens > 0:
# Triton ReplaySSM requires exact synchronous CPU metadata.
vllm_config.scheduler_config.async_scheduling = False
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=num_speculative_tokens,
)
return MockMambaBuilder(
_make_mamba_spec(buffer_len, mamba_backend),
_make_mamba_spec(buffer_len, mamba_backend, num_speculative_tokens),
["layer0"],
vllm_config,
DEVICE,
)
def _build(builder: MockMambaBuilder, case: ReplaySSMBuildCase):
def _build(
builder: MockMambaBuilder,
case: ReplaySSMBuildCase,
num_accepted_tokens: torch.Tensor | None = None,
):
batch = BatchSpec(seq_lens=case.seq_lens, query_lens=case.query_lens)
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
is_prefilling=torch.tensor(case.is_prefilling, dtype=torch.bool),
replayssm_decode_base_cpu=torch.tensor(case.decode_base, dtype=torch.int32),
)
return builder.build(0, common)
return builder.build(0, common, num_accepted_tokens=num_accepted_tokens)
@pytest.mark.parametrize(
@@ -270,21 +287,60 @@ def test_resumed_request_differs_from_fresh():
assert meta.is_flush_d.tolist()[:2] == [0, 0]
def test_flashinfer_replayssm_scratch_metadata_fresh_decode():
checkpointing_ssu = pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
if not hasattr(checkpointing_ssu, "allocate_checkpointing_ssu_scratch"):
pytest.skip("FlashInfer does not expose ReplaySSM scratch allocation")
def test_spec_decode_single_token_chunk_synthesizes_acceptance_metadata():
builder = _create_replayssm_builder(16, num_speculative_tokens=3)
case = REPLAYSSM_BUILD_CASES["leftover_prompt_one_token_flush"]
builder = _create_replayssm_builder(16, mamba_backend=MambaBackendEnum.FLASHINFER)
case = REPLAYSSM_BUILD_CASES["fresh_decode"]
meta = _build(builder, case)
assert meta.write_pos_d is None
assert meta.is_flush_d is None
assert meta.bc_pre_scratch is None
assert meta.replayssm_scratch is not None
assert [tensor.shape for tensor in meta.replayssm_scratch] == [
(1, 1, 32, 8),
(1, 1, 16),
(1, 1, 32, 8),
]
assert meta.query_start_loc_d is not None
assert meta.query_start_loc_d.tolist() == [0, 1]
assert meta.num_accepted_tokens is not None
assert meta.num_accepted_tokens.tolist() == [1]
def test_flashinfer_replayssm_state_indices_are_stable_for_full_cudagraph():
checkpointing_ssu = pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
if not hasattr(checkpointing_ssu, "allocate_checkpointing_ssu_scratch"):
pytest.skip("requires FlashInfer ReplaySSM autotuning support")
builder = _create_replayssm_builder(
16,
mamba_backend=MambaBackendEnum.FLASHINFER,
num_speculative_tokens=3,
)
builder.compilation_config.cudagraph_mode = CUDAGraphMode.FULL
first = _build(
builder,
ReplaySSMBuildCase(
seq_lens=[106, 106],
query_lens=[1, 1],
is_prefilling=[False, False],
decode_base=[100, 100],
buffer_len=16,
expected_write_pos=[],
expected_is_flush=[],
),
)
first_indices = first.replayssm_state_indices_d
assert first_indices is not None
assert first_indices.is_contiguous()
first_ptr = first_indices.data_ptr()
second = _build(
builder,
ReplaySSMBuildCase(
seq_lens=[122, 122],
query_lens=[1, 1],
is_prefilling=[False, False],
decode_base=[116, 116],
buffer_len=16,
expected_write_pos=[],
expected_is_flush=[],
),
)
second_indices = second.replayssm_state_indices_d
assert second_indices is not None
assert second_indices.data_ptr() == first_ptr
assert torch.equal(second_indices, second.state_indices_tensor_d[:, 0])
+120 -26
View File
@@ -10,11 +10,16 @@ from vllm.v1.metrics.reader import Counter
from ...models.utils import check_logprobs_close
from ...utils import large_gpu_mark, multi_gpu_test
# Mamba2 (Nemotron-3) hybrid.
# Mamba2 (Nemotron-3) hybrids.
MAMBA2_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
# FlashInfer 0.7.0's standard SSU cannot compile Nano's DIM=80 configuration.
MAMBA2_FLASHINFER_MODEL = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4"
MODELS = [
pytest.param(MAMBA2_MODEL, marks=large_gpu_mark(min_gb=40)),
]
FLASHINFER_MODELS = [
pytest.param(MAMBA2_FLASHINFER_MODEL, marks=large_gpu_mark(min_gb=40)),
]
PROMPTS = [
"The capital of France is",
@@ -22,6 +27,16 @@ PROMPTS = [
]
@pytest.fixture(autouse=True)
def _use_v1_model_runner_by_default(monkeypatch):
# Triton ReplaySSM is V1-only. FlashInfer V2 tests override this locally.
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
envs.disable_envs_cache()
yield
envs.disable_envs_cache()
def _check_replayssm_parity(
vllm_runner,
model_name,
@@ -29,17 +44,11 @@ def _check_replayssm_parity(
tensor_parallel_size=1,
mamba_backend: str = "triton",
name_1: str = "replayssm",
require_v2: bool = False,
monkeypatch: pytest.MonkeyPatch | None = None,
expected_v2: bool | None = None,
):
# Compare logprobs, not greedy ids: ReplaySSM's fp arithmetic can flip a
# near-tie. Baseline and ReplaySSM run at the same TP, so TP numerics are
# common-mode and only ReplaySSM varies.
if require_v2:
assert monkeypatch is not None
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
envs.disable_envs_cache()
# near-tie. Baseline and ReplaySSM use the same backend so only ReplaySSM
# behavior varies.
common = dict(
max_model_len=1024,
trust_remote_code=True,
@@ -49,14 +58,17 @@ def _check_replayssm_parity(
mamba_backend=mamba_backend,
)
with vllm_runner(model_name, **common) as llm:
if require_v2:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
if expected_v2 is not None:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
with vllm_runner(
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
**common,
) as llm:
if require_v2:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
if expected_v2 is not None:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
check_logprobs_close(
@@ -80,19 +92,101 @@ def test_replayssm_decode_matches_baseline_tp2(vllm_runner, model_name):
_check_replayssm_parity(vllm_runner, model_name, tensor_parallel_size=2)
@pytest.mark.parametrize("model_name", MODELS)
def test_replayssm_flashinfer_decode_matches_baseline_v2(
vllm_runner, model_name, monkeypatch
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_replayssm_flashinfer_decode_matches_baseline(
vllm_runner, model_name, monkeypatch, use_v2_model_runner
):
pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
_check_replayssm_parity(
vllm_runner,
model_name,
mamba_backend="flashinfer",
name_1="replayssm_flashinfer_v2",
require_v2=True,
monkeypatch=monkeypatch,
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
_check_replayssm_parity(
vllm_runner,
model_name,
mamba_backend="flashinfer",
name_1="replayssm_flashinfer",
expected_v2=use_v2_model_runner,
)
finally:
# The context restores the environment before the final cache reset.
envs.disable_envs_cache()
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
common = dict(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
speculative_config={
"method": "ngram",
"num_speculative_tokens": 3,
"prompt_lookup_max": 3,
},
)
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
with vllm_runner(
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
mamba_backend="flashinfer",
**common,
) as llm:
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
check_logprobs_close(
outputs_0_lst=baseline,
outputs_1_lst=replay,
name_0="baseline_spec",
name_1="replayssm_flashinfer_spec",
)
@multi_gpu_test(num_gpus=2)
@large_gpu_mark(min_gb=40)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner):
common = dict(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
mamba_backend="flashinfer",
tensor_parallel_size=2,
disable_log_stats=False,
speculative_config={"method": "mtp", "num_speculative_tokens": 3},
)
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
with vllm_runner(
MAMBA2_FLASHINFER_MODEL,
use_replayssm=True,
replayssm_buffer_len=16,
**common,
) as llm:
assert (
llm.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
outputs = llm.generate_greedy(PROMPTS, max_tokens=32)
draft_count = sum(
metric.value
for metric in llm.llm.get_metrics()
if isinstance(metric, Counter)
and metric.name == "vllm:spec_decode_num_drafts"
)
finally:
envs.disable_envs_cache()
# At least one request must run past the 16-token replay window; another
# may legitimately stop early on EOS.
assert any(len(token_ids) > 16 for token_ids, _ in outputs)
assert draft_count > 0
# Prefix spans several mamba blocks; prefix caching only reuses full blocks.
+14 -12
View File
@@ -1260,6 +1260,8 @@ class _TestReplaySSMMixer(MambaMixer2):
self.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32)
self._commits_replayssm_trackers = True
self._updates_replayssm_trackers = True
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
@@ -1294,19 +1296,19 @@ def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group(layers_only):
else:
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
assert (
mixers[0]._replayssm_ring_start.data_ptr()
== mixers[2]._replayssm_ring_start.data_ptr()
tracker_names = (
"_replayssm_ring_start",
"_replayssm_prev_num_accepted",
"_replayssm_prev_query_len",
)
assert (
mixers[0]._replayssm_prev_num_accepted.data_ptr()
== mixers[2]._replayssm_prev_num_accepted.data_ptr()
)
assert (
mixers[1]._replayssm_ring_start.data_ptr()
!= mixers[0]._replayssm_ring_start.data_ptr()
)
# Group {0, 2} shares trackers; layer 2 (not 0) updates after both run.
for tracker_name in tracker_names:
group_tracker = getattr(mixers[0], tracker_name)
assert group_tracker.data_ptr() == getattr(mixers[2], tracker_name).data_ptr()
assert group_tracker.data_ptr() != getattr(mixers[1], tracker_name).data_ptr()
assert group_tracker.shape == (4,)
assert torch.count_nonzero(group_tracker) == 0
assert [m._commits_replayssm_trackers for m in mixers] == [True, True, False]
assert [m._updates_replayssm_trackers for m in mixers] == [False, True, True]