[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
7e583e615c
commit
deb148e730
@@ -10,11 +10,16 @@ from vllm.v1.metrics.reader import Counter
|
||||
from ...models.utils import check_logprobs_close
|
||||
from ...utils import large_gpu_mark, multi_gpu_test
|
||||
|
||||
# Mamba2 (Nemotron-3) hybrid.
|
||||
# Mamba2 (Nemotron-3) hybrids.
|
||||
MAMBA2_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
|
||||
# FlashInfer 0.7.0's standard SSU cannot compile Nano's DIM=80 configuration.
|
||||
MAMBA2_FLASHINFER_MODEL = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
||||
MODELS = [
|
||||
pytest.param(MAMBA2_MODEL, marks=large_gpu_mark(min_gb=40)),
|
||||
]
|
||||
FLASHINFER_MODELS = [
|
||||
pytest.param(MAMBA2_FLASHINFER_MODEL, marks=large_gpu_mark(min_gb=40)),
|
||||
]
|
||||
|
||||
PROMPTS = [
|
||||
"The capital of France is",
|
||||
@@ -22,6 +27,16 @@ PROMPTS = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _use_v1_model_runner_by_default(monkeypatch):
|
||||
# Triton ReplaySSM is V1-only. FlashInfer V2 tests override this locally.
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
envs.disable_envs_cache()
|
||||
yield
|
||||
envs.disable_envs_cache()
|
||||
|
||||
|
||||
def _check_replayssm_parity(
|
||||
vllm_runner,
|
||||
model_name,
|
||||
@@ -29,17 +44,11 @@ def _check_replayssm_parity(
|
||||
tensor_parallel_size=1,
|
||||
mamba_backend: str = "triton",
|
||||
name_1: str = "replayssm",
|
||||
require_v2: bool = False,
|
||||
monkeypatch: pytest.MonkeyPatch | None = None,
|
||||
expected_v2: bool | None = None,
|
||||
):
|
||||
# Compare logprobs, not greedy ids: ReplaySSM's fp arithmetic can flip a
|
||||
# near-tie. Baseline and ReplaySSM run at the same TP, so TP numerics are
|
||||
# common-mode and only ReplaySSM varies.
|
||||
if require_v2:
|
||||
assert monkeypatch is not None
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
envs.disable_envs_cache()
|
||||
|
||||
# near-tie. Baseline and ReplaySSM use the same backend so only ReplaySSM
|
||||
# behavior varies.
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
@@ -49,14 +58,17 @@ def _check_replayssm_parity(
|
||||
mamba_backend=mamba_backend,
|
||||
)
|
||||
with vllm_runner(model_name, **common) as llm:
|
||||
if require_v2:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
if expected_v2 is not None:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
|
||||
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
with vllm_runner(
|
||||
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
**common,
|
||||
) as llm:
|
||||
if require_v2:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
if expected_v2 is not None:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
|
||||
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
|
||||
check_logprobs_close(
|
||||
@@ -80,19 +92,101 @@ def test_replayssm_decode_matches_baseline_tp2(vllm_runner, model_name):
|
||||
_check_replayssm_parity(vllm_runner, model_name, tensor_parallel_size=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_replayssm_flashinfer_decode_matches_baseline_v2(
|
||||
vllm_runner, model_name, monkeypatch
|
||||
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_replayssm_flashinfer_decode_matches_baseline(
|
||||
vllm_runner, model_name, monkeypatch, use_v2_model_runner
|
||||
):
|
||||
pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
|
||||
_check_replayssm_parity(
|
||||
vllm_runner,
|
||||
model_name,
|
||||
mamba_backend="flashinfer",
|
||||
name_1="replayssm_flashinfer_v2",
|
||||
require_v2=True,
|
||||
monkeypatch=monkeypatch,
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
envs.disable_envs_cache()
|
||||
_check_replayssm_parity(
|
||||
vllm_runner,
|
||||
model_name,
|
||||
mamba_backend="flashinfer",
|
||||
name_1="replayssm_flashinfer",
|
||||
expected_v2=use_v2_model_runner,
|
||||
)
|
||||
finally:
|
||||
# The context restores the environment before the final cache reset.
|
||||
envs.disable_envs_cache()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
|
||||
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
speculative_config={
|
||||
"method": "ngram",
|
||||
"num_speculative_tokens": 3,
|
||||
"prompt_lookup_max": 3,
|
||||
},
|
||||
)
|
||||
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
|
||||
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
mamba_backend="flashinfer",
|
||||
**common,
|
||||
) as llm:
|
||||
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline,
|
||||
outputs_1_lst=replay,
|
||||
name_0="baseline_spec",
|
||||
name_1="replayssm_flashinfer_spec",
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@large_gpu_mark(min_gb=40)
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner):
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
mamba_backend="flashinfer",
|
||||
tensor_parallel_size=2,
|
||||
disable_log_stats=False,
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 3},
|
||||
)
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
envs.disable_envs_cache()
|
||||
with vllm_runner(
|
||||
MAMBA2_FLASHINFER_MODEL,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
**common,
|
||||
) as llm:
|
||||
assert (
|
||||
llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
outputs = llm.generate_greedy(PROMPTS, max_tokens=32)
|
||||
draft_count = sum(
|
||||
metric.value
|
||||
for metric in llm.llm.get_metrics()
|
||||
if isinstance(metric, Counter)
|
||||
and metric.name == "vllm:spec_decode_num_drafts"
|
||||
)
|
||||
finally:
|
||||
envs.disable_envs_cache()
|
||||
|
||||
# At least one request must run past the 16-token replay window; another
|
||||
# may legitimately stop early on EOS.
|
||||
assert any(len(token_ids) > 16 for token_ids, _ in outputs)
|
||||
assert draft_count > 0
|
||||
|
||||
|
||||
# Prefix spans several mamba blocks; prefix caching only reuses full blocks.
|
||||
|
||||
Reference in New Issue
Block a user