[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)
Signed-off-by: Johnny-Liou <a897111@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 4.8
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
tomeras91
Cyrus Leung
parent
d02df748bf
commit
866fea2b99
@@ -80,6 +80,18 @@ class MambaStateDtypeCalculator:
|
||||
model_dtype, mamba_cache_dtype, mamba_ssm_cache_dtype
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def append_replayssm_ring(
|
||||
cls,
|
||||
base_dtypes: tuple[torch.dtype, ...],
|
||||
model_dtype: ModelDType | torch.dtype,
|
||||
) -> tuple[torch.dtype, ...]:
|
||||
"""Append the ReplaySSM ring dtypes to a base ``(conv, ssm)`` tuple:
|
||||
``(x_cache, dt_cache, B_cache)`` = ``(activation, fp32, activation)``.
|
||||
"""
|
||||
activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype)
|
||||
return (*base_dtypes, activation_dtype, torch.float32, activation_dtype)
|
||||
|
||||
@classmethod
|
||||
def _mamba_state_dtype(
|
||||
cls,
|
||||
@@ -186,6 +198,28 @@ class MambaStateShapeCalculator:
|
||||
temporal_state_shape = (divide(num_heads, tp_world_size), head_dim, state_size)
|
||||
return conv_state_shape, temporal_state_shape
|
||||
|
||||
@classmethod
|
||||
def append_replayssm_ring(
|
||||
cls,
|
||||
base_shapes: tuple[tuple[int, ...], ...],
|
||||
n_groups: int,
|
||||
tp_world_size: int,
|
||||
replayssm_buffer_len: int,
|
||||
) -> tuple[tuple[int, ...], ...]:
|
||||
"""Append the ReplaySSM ring shapes (x_cache, dt_cache, B_cache) to a
|
||||
base ``(conv, ssm)`` tuple. ``base_shapes[1]`` is the ssm shape
|
||||
``(nheads // tp, head_dim, state_size)``; B_cache uses the un-extended
|
||||
``n_groups``.
|
||||
"""
|
||||
local_nheads, head_dim, state_size = base_shapes[1]
|
||||
local_ngroups = divide(n_groups, tp_world_size)
|
||||
return (
|
||||
*base_shapes,
|
||||
(local_nheads, replayssm_buffer_len, head_dim),
|
||||
(local_nheads, replayssm_buffer_len),
|
||||
(local_ngroups, replayssm_buffer_len, state_size),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def short_conv_state_shape(
|
||||
cls,
|
||||
|
||||
Reference in New Issue
Block a user