[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)

Signed-off-by: Johnny-Liou <a897111@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Johnny-Liou
2026-07-24 09:39:49 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> tomeras91 Cyrus Leung
parent d02df748bf
commit 866fea2b99
23 changed files with 2954 additions and 31 deletions
@@ -80,6 +80,18 @@ class MambaStateDtypeCalculator:
model_dtype, mamba_cache_dtype, mamba_ssm_cache_dtype
)
@classmethod
def append_replayssm_ring(
cls,
base_dtypes: tuple[torch.dtype, ...],
model_dtype: ModelDType | torch.dtype,
) -> tuple[torch.dtype, ...]:
"""Append the ReplaySSM ring dtypes to a base ``(conv, ssm)`` tuple:
``(x_cache, dt_cache, B_cache)`` = ``(activation, fp32, activation)``.
"""
activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype)
return (*base_dtypes, activation_dtype, torch.float32, activation_dtype)
@classmethod
def _mamba_state_dtype(
cls,
@@ -186,6 +198,28 @@ class MambaStateShapeCalculator:
temporal_state_shape = (divide(num_heads, tp_world_size), head_dim, state_size)
return conv_state_shape, temporal_state_shape
@classmethod
def append_replayssm_ring(
cls,
base_shapes: tuple[tuple[int, ...], ...],
n_groups: int,
tp_world_size: int,
replayssm_buffer_len: int,
) -> tuple[tuple[int, ...], ...]:
"""Append the ReplaySSM ring shapes (x_cache, dt_cache, B_cache) to a
base ``(conv, ssm)`` tuple. ``base_shapes[1]`` is the ssm shape
``(nheads // tp, head_dim, state_size)``; B_cache uses the un-extended
``n_groups``.
"""
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
*base_shapes,
(local_nheads, replayssm_buffer_len, head_dim),
(local_nheads, replayssm_buffer_len),
(local_ngroups, replayssm_buffer_len, state_size),
)
@classmethod
def short_conv_state_shape(
cls,