[Mamba] Add FlashInfer ReplaySSM backend (#52506)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-01 10:26:43 -04:00
committed by GitHub
co-authored by Andrii Skliar
parent 25efcfa788
commit adebc41b7e
19 changed files with 1029 additions and 105 deletions
@@ -10,6 +10,7 @@ import torch
import vllm.envs as envs
from vllm.config.cache import MambaDType
from vllm.config.mamba import MambaBackendEnum
from vllm.config.model import ModelDType
from vllm.distributed import divide
from vllm.logger import init_logger
@@ -214,20 +215,25 @@ class MambaStateShapeCalculator:
base_shapes: tuple[tuple[int, ...], ...],
n_groups: int,
tp_world_size: int,
replayssm_buffer_len: int,
logical_window: int,
backend: MambaBackendEnum,
) -> tuple[tuple[int, ...], ...]:
"""Append the ReplaySSM ring shapes (x_cache, dt_cache, B_cache) to a
base ``(conv, ssm)`` tuple. ``base_shapes[1]`` is the ssm shape
``(nheads // tp, head_dim, state_size)``; B_cache uses the un-extended
``n_groups``.
"""Append the physical ReplaySSM ring shapes.
``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``;
B_cache uses the un-extended ``n_groups``.
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and appended token together.
ring_buffer_len += 1
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
*base_shapes,
(local_nheads, replayssm_buffer_len, head_dim),
(local_nheads, replayssm_buffer_len),
(local_ngroups, replayssm_buffer_len, state_size),
(local_nheads, ring_buffer_len, head_dim),
(local_nheads, ring_buffer_len),
(local_ngroups, ring_buffer_len, state_size),
)
@classmethod