[Mamba] Add FlashInfer ReplaySSM backend (#52506)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-01 10:26:43 -04:00
committed by GitHub
co-authored by Andrii Skliar
parent 25efcfa788
commit adebc41b7e
19 changed files with 1029 additions and 105 deletions
@@ -10,11 +10,14 @@ the backend defaults to 'cpu'.
"""
from abc import ABC, abstractmethod
from collections.abc import Callable
from functools import cache
import torch
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
@@ -22,6 +25,99 @@ from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
logger = init_logger(__name__)
@triton.jit(
do_not_specialize=["n_slots", "state_batch_indices_stride"],
do_not_specialize_on_alignment=["state_batch_indices"],
)
def _update_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
state_batch_indices,
state_batch_indices_stride,
n_slots,
num_states,
logical_window: tl.constexpr,
ring_buffer_len: tl.constexpr,
pad_slot_id: tl.constexpr,
RESET: tl.constexpr,
BLOCK: tl.constexpr,
) -> None:
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n_slots
slots = tl.load(
state_batch_indices + offsets * state_batch_indices_stride,
mask=mask,
other=pad_slot_id,
)
valid = mask & (slots != pad_slot_id) & (slots >= 0) & (slots < num_states)
if RESET:
tl.store(ring_start + slots, 0, mask=valid)
tl.store(prev_num_accepted + slots, 0, mask=valid)
else:
prev = tl.load(prev_num_accepted + slots, mask=valid, other=0)
start = tl.load(ring_start + slots, mask=valid, other=0)
must_checkpoint = prev + 1 > logical_window
next_start = tl.where(
must_checkpoint,
(start + prev) % ring_buffer_len,
start,
)
next_prev = tl.where(must_checkpoint, 1, prev + 1)
tl.store(ring_start + slots, next_start, mask=valid)
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
def update_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
state_batch_indices: torch.Tensor,
logical_window: int | None = None,
ring_buffer_len: int | None = None,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Reset selected trackers, or advance them when a window is provided."""
if state_batch_indices.dim() > 1:
state_batch_indices = state_batch_indices[:, 0]
n_slots = state_batch_indices.numel()
if n_slots == 0:
return
reset = logical_window is None
if reset:
logical_window = 0
ring_buffer_len = 1
else:
assert ring_buffer_len is not None
block = 128
_update_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
state_batch_indices,
state_batch_indices.stride(0),
n_slots,
min(ring_start.numel(), prev_num_accepted.numel()),
logical_window,
ring_buffer_len,
pad_slot_id,
RESET=reset,
BLOCK=block,
)
def reset_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
state_batch_indices: torch.Tensor,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Reset selected ReplaySSM ring trackers."""
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_batch_indices,
pad_slot_id=pad_slot_id,
)
class MambaSSUBackend(ABC):
"""Abstract base class for Mamba SSU backends."""
@@ -262,15 +358,115 @@ _BACKEND_REGISTRY: dict[MambaBackendEnum, type[MambaSSUBackend]] = {
_mamba_ssu_backend: MambaSSUBackend | None = None
_flashinfer_replayssm_kernel: Callable[..., torch.Tensor] | None = None
@cache
def flashinfer_replayssm_autotune_supported() -> bool:
"""Return True when FlashInfer exposes ReplaySSM autotuning."""
try:
from flashinfer.mamba.checkpointing_ssu import ( # noqa: F401
CheckpointingSSURunner,
)
except ImportError:
return False
return True
def selective_state_update_replayssm_flashinfer(
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
out: torch.Tensor,
x_cache: torch.Tensor,
B_cache: torch.Tensor,
dt_cache: torch.Tensor,
ring_start: torch.Tensor,
prev_num_accepted_tokens: torch.Tensor,
logical_window: int,
D: torch.Tensor | None = None,
dt_bias: torch.Tensor | None = None,
dt_softplus: bool = False,
state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
scratch: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
update_trackers: bool = True,
enable_stochastic_rounding: bool = False,
stochastic_rounding_philox_rounds: int = 0,
) -> torch.Tensor:
"""Run FlashInfer checkpointing SSU and optionally advance shared trackers."""
if _flashinfer_replayssm_kernel is None:
raise RuntimeError(
"FlashInfer ReplaySSM has not been initialized. "
"Call initialize_mamba_ssu_backend() with use_replayssm=True."
)
if x.dim() == 3:
x = x.unsqueeze(1)
dt = dt.unsqueeze(1)
B = B.unsqueeze(1)
C = C.unsqueeze(1)
out = out.unsqueeze(1)
indices = state_batch_indices
if indices is not None and indices.dim() > 1:
indices = indices[:, 0]
cb_scaled = cumAdt_vec = cb_old = None
if scratch is not None:
cb_scaled, cumAdt_vec, cb_old = scratch
rand_seed = (
torch.randint(0, 2**32, (1,), device=state.device, dtype=torch.int64)
if enable_stochastic_rounding
else None
)
result = _flashinfer_replayssm_kernel(
state,
x_cache,
B_cache,
dt_cache,
ring_start,
prev_num_accepted_tokens,
x,
dt,
A,
B,
C,
out,
D=D,
dt_bias=dt_bias,
dt_softplus=dt_softplus,
state_batch_indices=indices,
pad_slot_id=null_block_id,
rand_seed=rand_seed,
philox_rounds=stochastic_rounding_philox_rounds or 10,
cb_scaled=cb_scaled,
cumAdt_vec=cumAdt_vec,
cb_old=cb_old,
)
if update_trackers and indices is not None:
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted_tokens,
indices,
logical_window=logical_window,
ring_buffer_len=x_cache.size(2),
pad_slot_id=null_block_id,
)
return result
def initialize_mamba_ssu_backend(
mamba_config: MambaConfig,
kv_cache_config: KVCacheConfig,
*,
use_replayssm: bool = False,
) -> None:
"""Initialize the global Mamba SSU backend.
No-op if `kv_cache_config` contains no specs that call
selective_state_update.
"""
"""Initialize the Mamba SSU backend and optional FlashInfer ReplaySSM."""
if not any(
isinstance(g.kv_cache_spec, MambaSpec)
and g.kv_cache_spec.mamba_type
@@ -279,13 +475,9 @@ def initialize_mamba_ssu_backend(
):
return
global _mamba_ssu_backend
global _flashinfer_replayssm_kernel, _mamba_ssu_backend
backend = mamba_config.backend
# On CPU-only platforms (PowerPC, x86 without CUDA) Triton JIT is
# unstable or unavailable. Silently fall back to the CPU
# backend unless the user explicitly chose something other than "triton".
if backend == MambaBackendEnum.TRITON:
from vllm.platforms import current_platform
@@ -301,13 +493,28 @@ def initialize_mamba_ssu_backend(
f"Unknown Mamba SSU backend: {backend}. "
f"Valid options: {list(_BACKEND_REGISTRY.keys())}"
)
if use_replayssm and backend not in (
MambaBackendEnum.TRITON,
MambaBackendEnum.FLASHINFER,
):
raise ValueError(f"ReplaySSM does not support mamba backend {backend.value!r}")
backend_cls = _BACKEND_REGISTRY[backend]
if isinstance(_mamba_ssu_backend, backend_cls):
return
if not isinstance(_mamba_ssu_backend, backend_cls):
_mamba_ssu_backend = backend_cls(mamba_config)
logger.info("Using %s Mamba SSU backend.", _mamba_ssu_backend.name)
_mamba_ssu_backend = backend_cls(mamba_config)
logger.info("Using %s Mamba SSU backend.", _mamba_ssu_backend.name)
_flashinfer_replayssm_kernel = None
if use_replayssm and backend == MambaBackendEnum.FLASHINFER:
try:
from flashinfer.mamba.checkpointing_ssu import checkpointing_ssu
except ImportError as e:
raise ImportError(
"FlashInfer ReplaySSM requires a compatible flashinfer-python package"
) from e
_flashinfer_replayssm_kernel = checkpointing_ssu
if use_replayssm:
logger.info("Using %s ReplaySSM backend.", backend.value)
def get_mamba_ssu_backend() -> MambaSSUBackend: