[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-30 21:41:17 +00:00
committed by GitHub
co-authored by Andrii Skliar
parent 7e583e615c
commit deb148e730
18 changed files with 1029 additions and 113 deletions
+120 -26
View File
@@ -10,11 +10,16 @@ from vllm.v1.metrics.reader import Counter
from ...models.utils import check_logprobs_close
from ...utils import large_gpu_mark, multi_gpu_test
# Mamba2 (Nemotron-3) hybrid.
# Mamba2 (Nemotron-3) hybrids.
MAMBA2_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
# FlashInfer 0.7.0's standard SSU cannot compile Nano's DIM=80 configuration.
MAMBA2_FLASHINFER_MODEL = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4"
MODELS = [
pytest.param(MAMBA2_MODEL, marks=large_gpu_mark(min_gb=40)),
]
FLASHINFER_MODELS = [
pytest.param(MAMBA2_FLASHINFER_MODEL, marks=large_gpu_mark(min_gb=40)),
]
PROMPTS = [
"The capital of France is",
@@ -22,6 +27,16 @@ PROMPTS = [
]
@pytest.fixture(autouse=True)
def _use_v1_model_runner_by_default(monkeypatch):
# Triton ReplaySSM is V1-only. FlashInfer V2 tests override this locally.
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
envs.disable_envs_cache()
yield
envs.disable_envs_cache()
def _check_replayssm_parity(
vllm_runner,
model_name,
@@ -29,17 +44,11 @@ def _check_replayssm_parity(
tensor_parallel_size=1,
mamba_backend: str = "triton",
name_1: str = "replayssm",
require_v2: bool = False,
monkeypatch: pytest.MonkeyPatch | None = None,
expected_v2: bool | None = None,
):
# Compare logprobs, not greedy ids: ReplaySSM's fp arithmetic can flip a
# near-tie. Baseline and ReplaySSM run at the same TP, so TP numerics are
# common-mode and only ReplaySSM varies.
if require_v2:
assert monkeypatch is not None
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
envs.disable_envs_cache()
# near-tie. Baseline and ReplaySSM use the same backend so only ReplaySSM
# behavior varies.
common = dict(
max_model_len=1024,
trust_remote_code=True,
@@ -49,14 +58,17 @@ def _check_replayssm_parity(
mamba_backend=mamba_backend,
)
with vllm_runner(model_name, **common) as llm:
if require_v2:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
if expected_v2 is not None:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
with vllm_runner(
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
**common,
) as llm:
if require_v2:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
if expected_v2 is not None:
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
check_logprobs_close(
@@ -80,19 +92,101 @@ def test_replayssm_decode_matches_baseline_tp2(vllm_runner, model_name):
_check_replayssm_parity(vllm_runner, model_name, tensor_parallel_size=2)
@pytest.mark.parametrize("model_name", MODELS)
def test_replayssm_flashinfer_decode_matches_baseline_v2(
vllm_runner, model_name, monkeypatch
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_replayssm_flashinfer_decode_matches_baseline(
vllm_runner, model_name, monkeypatch, use_v2_model_runner
):
pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
_check_replayssm_parity(
vllm_runner,
model_name,
mamba_backend="flashinfer",
name_1="replayssm_flashinfer_v2",
require_v2=True,
monkeypatch=monkeypatch,
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
_check_replayssm_parity(
vllm_runner,
model_name,
mamba_backend="flashinfer",
name_1="replayssm_flashinfer",
expected_v2=use_v2_model_runner,
)
finally:
# The context restores the environment before the final cache reset.
envs.disable_envs_cache()
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
common = dict(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
speculative_config={
"method": "ngram",
"num_speculative_tokens": 3,
"prompt_lookup_max": 3,
},
)
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
with vllm_runner(
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
mamba_backend="flashinfer",
**common,
) as llm:
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
check_logprobs_close(
outputs_0_lst=baseline,
outputs_1_lst=replay,
name_0="baseline_spec",
name_1="replayssm_flashinfer_spec",
)
@multi_gpu_test(num_gpus=2)
@large_gpu_mark(min_gb=40)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner):
common = dict(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
mamba_backend="flashinfer",
tensor_parallel_size=2,
disable_log_stats=False,
speculative_config={"method": "mtp", "num_speculative_tokens": 3},
)
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
with vllm_runner(
MAMBA2_FLASHINFER_MODEL,
use_replayssm=True,
replayssm_buffer_len=16,
**common,
) as llm:
assert (
llm.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
outputs = llm.generate_greedy(PROMPTS, max_tokens=32)
draft_count = sum(
metric.value
for metric in llm.llm.get_metrics()
if isinstance(metric, Counter)
and metric.name == "vllm:spec_decode_num_drafts"
)
finally:
envs.disable_envs_cache()
# At least one request must run past the 16-token replay window; another
# may legitimately stop early on EOS.
assert any(len(token_ids) > 16 for token_ids, _ in outputs)
assert draft_count > 0
# Prefix spans several mamba blocks; prefix caching only reuses full blocks.