[Mamba] Add FlashInfer ReplaySSM backend (#52506)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-01 10:26:43 -04:00
committed by GitHub
co-authored by Andrii Skliar
parent 25efcfa788
commit adebc41b7e
19 changed files with 1029 additions and 105 deletions
+70 -2
View File
@@ -7,12 +7,15 @@ import pytest
import torch
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
FlashInferSSUBackend,
TritonSSUBackend,
get_mamba_ssu_backend,
initialize_mamba_ssu_backend,
reset_replayssm_ring_trackers,
selective_state_update,
update_replayssm_ring_trackers,
)
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
@@ -30,6 +33,47 @@ except ImportError:
HAS_FLASHINFER = False
@pytest.fixture(autouse=True)
def restore_backend_state():
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
old_backend = mod._mamba_ssu_backend
old_replayssm_kernel = mod._flashinfer_replayssm_kernel
yield
mod._mamba_ssu_backend = old_backend
mod._flashinfer_replayssm_kernel = old_replayssm_kernel
def test_flashinfer_replayssm_ring_tracker_lifecycle():
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
observed = []
for _ in range(33):
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_batch_indices,
logical_window=16,
ring_buffer_len=17,
)
observed.append((int(ring_start[1]), int(prev_num_accepted[1])))
assert observed[4] == (0, 5)
assert observed[15] == (0, 16)
assert observed[16] == (16, 1)
assert observed[31] == (16, 16)
assert observed[32] == (15, 1)
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_batch_indices,
)
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
def _kv_cache_config_with_ssu(
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2,
) -> KVCacheConfig:
@@ -116,11 +160,10 @@ def test_flashinfer_forwards_ssu_algorithm(
def test_uninitialized_backend_raises():
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
old = mod._mamba_ssu_backend
# restore_backend_state (autouse) puts the global back afterwards.
mod._mamba_ssu_backend = None
with pytest.raises(RuntimeError, match="not been initialized"):
get_mamba_ssu_backend()
mod._mamba_ssu_backend = old
@pytest.mark.parametrize(
@@ -186,3 +229,28 @@ def test_triton_basic_call():
out=out,
)
assert not torch.isnan(out).any()
@pytest.mark.parametrize(
("backend", "expected_ring_len"),
[
(MambaBackendEnum.TRITON, 16),
(MambaBackendEnum.FLASHINFER, 17),
],
)
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
base_shapes = ((64, 3), (8, 4, 16))
shapes = MambaStateShapeCalculator.append_replayssm_ring(
base_shapes,
n_groups=4,
tp_world_size=2,
logical_window=16,
backend=backend,
)
assert shapes[2:] == (
(8, expected_ring_len, 4),
(8, expected_ring_len),
(2, expected_ring_len, 16),
)