[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)

Signed-off-by: Johnny-Liou <a897111@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Johnny-Liou
2026-07-24 09:39:49 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> tomeras91 Cyrus Leung
parent d02df748bf
commit 866fea2b99
23 changed files with 2954 additions and 31 deletions
@@ -32,6 +32,9 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_update,
)
from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
selective_state_update_replayssm_output_only,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
@@ -494,12 +497,27 @@ class MambaMixer2(MambaBase, PluggableLayer):
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
# The tuple is (conv_state, ssm_state)
self.kv_cache = (torch.tensor([]), torch.tensor([]))
self.model_config = model_config
self.cache_config = cache_config
self.prefix = prefix
self.use_replayssm = (
cache_config.use_replayssm if cache_config is not None else False
)
self.replayssm_buffer_len = (
cache_config.replayssm_buffer_len
if cache_config is not None and cache_config.use_replayssm
else None
)
self.mamba_config = vllm_config.mamba_config
if self.use_replayssm and self.num_heads % self.tp_size != 0:
raise ValueError(
"--use-replayssm requires tensor-parallel heads to divide evenly"
)
# The tuple is (conv_state, ssm_state); with the cached (ReplaySSM) decode
# kernel enabled it is (conv_state, ssm_state, x_cache, dt_cache, B_cache).
_n_state = 5 if self.use_replayssm else 2
self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state))
self.num_spec = vllm_config.num_speculative_tokens
if self.num_spec > 0:
@@ -591,7 +609,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
# Triton's autotuner includes tensor dtypes in its cache key,
# so state_dtype must match what real inference uses.
_, ssm_state_dtype = self.get_state_dtype()
ssm_state_dtype = self.get_state_dtype()[1]
# SSD kernel autotune keys depend on dtype and head dimensions,
# not on sequence length or batch size, so a single shape suffices.
@@ -702,6 +720,10 @@ class MambaMixer2(MambaBase, PluggableLayer):
else self.kv_cache[0].transpose(-1, -2)
)
ssm_state = self.kv_cache[1]
if self.use_replayssm:
x_cache, dt_cache, B_cache = self.kv_cache[2:]
else:
x_cache = dt_cache = B_cache = None
has_initial_states_p = attn_metadata.has_initial_states_p
prep_initial_states = attn_metadata.prep_initial_states
chunk_size = attn_metadata.chunk_size
@@ -1027,37 +1049,78 @@ class MambaMixer2(MambaBase, PluggableLayer):
# - mamba_cache_params.ssm_state's slots will be selected
# using state_indices_tensor_d
# NOTE: final output is an in-place update of out tensor
selective_state_update(
ssm_state,
hidden_states_d,
dt_d,
A_d,
B_d,
C_d,
D_d,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
out=preallocated_ssm_out_d.view(num_decode_tokens, -1, self.head_dim),
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
is_blackwell=self.is_blackwell,
preallocated_ssm_out_d = preallocated_ssm_out_d.view(
num_decode_tokens, -1, self.head_dim
)
if self.use_replayssm:
assert self.replayssm_buffer_len is not None
selective_state_update_replayssm_output_only(
ssm_state,
hidden_states_d,
dt_d,
A_d,
B_d,
C_d,
D_d,
dt_bias,
dt_softplus=True,
x_cache=x_cache,
dt_cache=dt_cache,
B_cache=B_cache,
bc_pre=attn_metadata.bc_pre_scratch,
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
out=preallocated_ssm_out_d,
# Stochastic Rounding for the vanilla decode path is read
# from mamba_config inside ssu_dispatch; the replay kernel
# isn't a dispatch backend, so pass it here.
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
cache_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
)
else:
selective_state_update(
ssm_state,
hidden_states_d,
dt_d,
A_d,
B_d,
C_d,
D_d,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
out=preallocated_ssm_out_d,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
is_blackwell=self.is_blackwell,
)
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.mamba2_state_dtype(
base_dtype = MambaStateDtypeCalculator.mamba2_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
self.cache_config.mamba_ssm_cache_dtype,
)
if self.use_replayssm:
return MambaStateDtypeCalculator.append_replayssm_ring(
base_dtype, self.model_config.dtype
)
return base_dtype
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.mamba2_state_shape(
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
tp_world_size = get_tensor_model_parallel_world_size()
base_shape = MambaStateShapeCalculator.mamba2_state_shape(
intermediate_size=self.intermediate_size,
tp_world_size=get_tensor_model_parallel_world_size(),
tp_world_size=tp_world_size,
n_groups=self.n_groups,
num_heads=self.num_heads,
head_dim=self.head_dim,
@@ -1065,6 +1128,15 @@ class MambaMixer2(MambaBase, PluggableLayer):
conv_kernel=self.conv_kernel_size,
num_spec=self.num_spec,
)
if self.use_replayssm:
assert self.replayssm_buffer_len is not None
return MambaStateShapeCalculator.append_replayssm_ring(
base_shape,
self.n_groups,
tp_world_size,
self.replayssm_buffer_len,
)
return base_shape
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
@@ -80,6 +80,18 @@ class MambaStateDtypeCalculator:
model_dtype, mamba_cache_dtype, mamba_ssm_cache_dtype
)
@classmethod
def append_replayssm_ring(
cls,
base_dtypes: tuple[torch.dtype, ...],
model_dtype: ModelDType | torch.dtype,
) -> tuple[torch.dtype, ...]:
"""Append the ReplaySSM ring dtypes to a base ``(conv, ssm)`` tuple:
``(x_cache, dt_cache, B_cache)`` = ``(activation, fp32, activation)``.
"""
activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype)
return (*base_dtypes, activation_dtype, torch.float32, activation_dtype)
@classmethod
def _mamba_state_dtype(
cls,
@@ -186,6 +198,28 @@ class MambaStateShapeCalculator:
temporal_state_shape = (divide(num_heads, tp_world_size), head_dim, state_size)
return conv_state_shape, temporal_state_shape
@classmethod
def append_replayssm_ring(
cls,
base_shapes: tuple[tuple[int, ...], ...],
n_groups: int,
tp_world_size: int,
replayssm_buffer_len: int,
) -> tuple[tuple[int, ...], ...]:
"""Append the ReplaySSM ring shapes (x_cache, dt_cache, B_cache) to a
base ``(conv, ssm)`` tuple. ``base_shapes[1]`` is the ssm shape
``(nheads // tp, head_dim, state_size)``; B_cache uses the un-extended
``n_groups``.
"""
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
*base_shapes,
(local_nheads, replayssm_buffer_len, head_dim),
(local_nheads, replayssm_buffer_len),
(local_ngroups, replayssm_buffer_len, state_size),
)
@classmethod
def short_conv_state_shape(
cls,
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Launch-config selection for the ReplaySSM Mamba2 output_only decode kernel.
Mirrors ``mamba_ssm.py``: a hard-coded heuristic per kernel, plus an
``override`` context manager for benchmarks/tests/config sweeps. Hardware is
auto-detected (Blackwell vs not) so call sites need not thread it through.
"""
import functools
from contextlib import contextmanager
from vllm.platforms import current_platform
from vllm.triton_utils import triton
@functools.cache
def _is_blackwell() -> bool:
try:
return current_platform.is_device_capability_family(100)
except Exception:
return False
# Per-kernel overrides keyed by the kernel name passed to get_replayssm_config.
_overrides: dict[str, tuple] = {}
@contextmanager
def override_replayssm_config(kernel: str, config: tuple):
"""Pin ``kernel``'s launch config for the duration of the context."""
prev = _overrides.get(kernel)
_overrides[kernel] = config
try:
yield
finally:
if prev is None:
_overrides.pop(kernel, None)
else:
_overrides[kernel] = prev
def _dstate_tile(dstate: int, tile: int) -> int:
return max(16, min(tile, triton.next_power_of_2(dstate)))
def _mamba2_output_only(dstate, L, is_blackwell):
# (block_size_m, num_warps, nf_dstate_tile, fl_dstate_tile, num_stages);
# decoupled dstate tiling, serving-batch optimum, dtype-independent per device.
if is_blackwell:
return 64, 1, _dstate_tile(dstate, 32), _dstate_tile(dstate, 64), 2
return 16, 1, _dstate_tile(dstate, 64), _dstate_tile(dstate, 128), 2
def get_replayssm_config(kernel: str, **shape) -> tuple:
"""Return the launch config for ``kernel`` (override > tuned default).
kernel: "mamba2_output_only". ``shape`` carries the keying dims (dstate;
``L`` for the buffer length, default 16); hardware is auto-detected.
"""
if kernel in _overrides:
return _overrides[kernel]
bw = _is_blackwell()
if kernel == "mamba2_output_only":
return _mamba2_output_only(shape["dstate"], shape.get("L", 16), bw)
raise ValueError(f"unknown ReplaySSM kernel config key: {kernel}")
@@ -0,0 +1,709 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import torch
from vllm.model_executor.layers.mamba.ops.mamba_ssm import convert_rs_fp16x2, softplus
from vllm.model_executor.layers.mamba.ops.replayssm_config import get_replayssm_config
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
@triton.heuristics(
{
"HAS_STATE_BATCH_INDICES": lambda args: args["state_batch_indices_ptr"]
is not None
}
)
@triton.heuristics(
{"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])}
)
@triton.jit
def _replayssm_output_only_precompute_kernel(
B_ptr,
C_ptr,
B_cache_ptr,
write_pos_ptr,
is_flush_ptr,
bc_pre_ptr,
state_batch_indices_ptr,
null_block_id,
# Matrix dimensions
batch,
ngroups,
dstate,
# Input strides
stride_B_batch,
stride_B_group,
stride_B_dstate,
stride_C_batch,
stride_C_group,
stride_C_dstate,
# Cache strides
stride_B_cache_batch,
stride_B_cache_group,
stride_B_cache_pos,
stride_B_cache_dstate,
stride_bc_pre_batch,
stride_bc_pre_group,
stride_bc_pre_pos,
stride_state_indices_batch,
stride_state_indices_T,
# Meta-parameters
MAX_CACHE_LEN: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
# heuristic-computed
BLOCK_SIZE_DSTATE: tl.constexpr,
HAS_STATE_BATCH_INDICES: tl.constexpr,
):
pid_b = tl.program_id(axis=0)
pid_g = tl.program_id(axis=1)
# On flush steps the main kernel does not read bc_pre, so skip the work.
is_flush = tl.load(is_flush_ptr + pid_b) != 0
if is_flush:
return
if HAS_STATE_BATCH_INDICES:
state_batch_idx = tl.load(
state_batch_indices_ptr
+ pid_b * stride_state_indices_batch
+ 0 * stride_state_indices_T
).to(tl.int64)
if state_batch_idx == null_block_id:
return
else:
state_batch_idx = pid_b
offs_k = tl.arange(0, BLOCK_SIZE_K)
offs_n = tl.arange(0, BLOCK_SIZE_DSTATE)
write_pos = tl.load(write_pos_ptr + pid_b).to(tl.int64)
B_ptr += pid_b * stride_B_batch + pid_g * stride_B_group
C_ptr += pid_b * stride_C_batch + pid_g * stride_C_group
B_cache_ptr += state_batch_idx * stride_B_cache_batch + pid_g * stride_B_cache_group
bc_pre_ptr += pid_b * stride_bc_pre_batch + pid_g * stride_bc_pre_group
B_cur = tl.load(
B_ptr + offs_n * stride_B_dstate,
mask=offs_n < dstate,
other=0.0,
)
C = tl.load(
C_ptr + offs_n * stride_C_dstate,
mask=offs_n < dstate,
other=0.0,
)
B_cache_ptrs = (
B_cache_ptr
+ offs_k[:, None] * stride_B_cache_pos
+ offs_n[None, :] * stride_B_cache_dstate
)
B_cache = tl.load(
B_cache_ptrs,
mask=(offs_k[:, None] < write_pos) & (offs_n[None, :] < dstate),
other=0.0,
)
B_all = tl.where(offs_k[:, None] == write_pos, B_cur[None, :], B_cache)
bc = tl.sum(B_all.to(tl.float32) * C[None, :].to(tl.float32), axis=1)
tl.store(
bc_pre_ptr + offs_k * stride_bc_pre_pos,
bc,
mask=(offs_k <= write_pos) & (offs_k < MAX_CACHE_LEN),
)
@triton.heuristics({"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None})
@triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None})
@triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None})
@triton.heuristics(
{
"HAS_STATE_BATCH_INDICES": lambda args: args["state_batch_indices_ptr"]
is not None
}
)
@triton.heuristics(
{"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])}
)
@triton.jit
def _replayssm_output_only_kernel(
# Pointers to matrices
state_ptr,
rand_seed_ptr,
x_ptr,
dt_ptr,
dt_bias_ptr,
A_ptr,
B_ptr,
C_ptr,
D_ptr,
z_ptr,
out_ptr,
x_cache_ptr,
dt_cache_ptr,
B_cache_ptr,
bc_pre_ptr,
write_pos_ptr,
is_flush_ptr,
state_batch_indices_ptr,
null_block_id,
# Matrix dimensions
batch,
nheads,
dim,
dstate,
nheads_ngroups_ratio,
# State strides
stride_state_batch,
stride_state_head,
stride_state_dim,
stride_state_dstate,
# Input strides
stride_x_batch,
stride_x_head,
stride_x_dim,
stride_dt_batch,
stride_dt_head,
stride_dt_bias_head,
stride_A_head,
stride_B_batch,
stride_B_group,
stride_B_dstate,
stride_C_batch,
stride_C_group,
stride_C_dstate,
stride_D_head,
stride_D_dim,
stride_z_batch,
stride_z_head,
stride_z_dim,
stride_out_batch,
stride_out_head,
stride_out_dim,
# Cache strides
stride_x_cache_batch,
stride_x_cache_head,
stride_x_cache_dim,
stride_x_cache_pos,
stride_dt_cache_batch,
stride_dt_cache_head,
stride_dt_cache_pos,
stride_B_cache_batch,
stride_B_cache_group,
stride_B_cache_pos,
stride_B_cache_dstate,
stride_bc_pre_batch,
stride_bc_pre_group,
stride_bc_pre_pos,
stride_state_indices_batch,
stride_state_indices_T,
# Meta-parameters
DT_SOFTPLUS: tl.constexpr,
MAX_CACHE_LEN: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_K_CACHE: tl.constexpr,
BLOCK_SIZE_K_DOT: tl.constexpr,
NF_DSTATE_TILE: tl.constexpr,
NF_NDS: tl.constexpr,
FL_DSTATE_TILE: tl.constexpr,
FL_NDS: tl.constexpr,
USE_RS_ROUNDING: tl.constexpr,
PHILOX_ROUNDS: tl.constexpr,
# heuristic-computed
BLOCK_SIZE_DSTATE: tl.constexpr,
HAS_DT_BIAS: tl.constexpr,
HAS_D: tl.constexpr,
HAS_Z: tl.constexpr,
HAS_STATE_BATCH_INDICES: tl.constexpr,
):
pid_m = tl.program_id(axis=0)
pid_b = tl.program_id(axis=1)
pid_h = tl.program_id(axis=2)
# Resolve the physical state slot for this decode row; skip padded rows.
if HAS_STATE_BATCH_INDICES:
state_batch_idx = tl.load(
state_batch_indices_ptr
+ pid_b * stride_state_indices_batch
+ 0 * stride_state_indices_T
).to(tl.int64)
if state_batch_idx == null_block_id:
return
else:
state_batch_idx = pid_b
offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
offs_n_full = tl.arange(0, BLOCK_SIZE_DSTATE)
# Buffer cursor (number of cached tokens so far) and the flush flag.
write_pos = tl.load(write_pos_ptr + pid_b).to(tl.int64)
is_flush = tl.load(is_flush_ptr + pid_b) != 0
# Advance every pointer to this (row, head, group).
state_ptr += state_batch_idx * stride_state_batch + pid_h * stride_state_head
x_ptr += pid_b * stride_x_batch + pid_h * stride_x_head
dt_ptr += pid_b * stride_dt_batch + pid_h * stride_dt_head
B_ptr += pid_b * stride_B_batch + (pid_h // nheads_ngroups_ratio) * stride_B_group
C_ptr += pid_b * stride_C_batch + (pid_h // nheads_ngroups_ratio) * stride_C_group
out_ptr += pid_b * stride_out_batch + pid_h * stride_out_head
x_cache_ptr += state_batch_idx * stride_x_cache_batch + pid_h * stride_x_cache_head
dt_cache_ptr += (
state_batch_idx * stride_dt_cache_batch + pid_h * stride_dt_cache_head
)
B_cache_ptr += (
state_batch_idx * stride_B_cache_batch
+ (pid_h // nheads_ngroups_ratio) * stride_B_cache_group
)
bc_pre_ptr += (
pid_b * stride_bc_pre_batch
+ (pid_h // nheads_ngroups_ratio) * stride_bc_pre_group
)
# Current-token dt (+ bias, softplus), scalar A, and current x. C, the
# checkpoint state S_0, and current-token B are read per dstate tile below.
dt_cur = tl.load(dt_ptr).to(tl.float32)
if HAS_DT_BIAS:
dt_cur += tl.load(dt_bias_ptr + pid_h * stride_dt_bias_head).to(tl.float32)
if DT_SOFTPLUS:
dt_cur = tl.where(dt_cur <= 20.0, softplus(dt_cur), dt_cur)
A = tl.load(A_ptr + pid_h * stride_A_head).to(tl.float32)
x_cur = tl.load(x_ptr + offs_m * stride_x_dim, mask=offs_m < dim, other=0.0)
if not is_flush:
# Output-only route: read y without materializing the state, using the
# precomputed k^T q products (`bc`):
# y = total_decay * (S_0 q) + sum_j s_j (k_j^T q) v_j.
# Then append the current token to the buffer.
offs_k_cache = tl.arange(0, BLOCK_SIZE_K_CACHE)
# dt over the window (history + current token), then the decay weights.
dt_all_cache = tl.load(
dt_cache_ptr + offs_k_cache * stride_dt_cache_pos,
mask=offs_k_cache < write_pos,
other=0.0,
).to(tl.float32)
dt_all_cache = tl.where(offs_k_cache == write_pos, dt_cur, dt_all_cache)
dA_cumsum_cache = A * tl.cumsum(dt_all_cache, axis=0)
dA_total_cache = A * tl.sum(dt_all_cache, axis=0)
total_decay_cache = tl.exp(dA_total_cache)
scale_cache = dt_all_cache * tl.exp(dA_total_cache - dA_cumsum_cache)
scale_cache = tl.where(offs_k_cache <= write_pos, scale_cache, 0.0)
# Gather buffered x over the window (history + current token).
x_all_cache_ptrs = (
x_cache_ptr
+ offs_m[:, None] * stride_x_cache_dim
+ offs_k_cache[None, :] * stride_x_cache_pos
)
x_all_cache = tl.load(
x_all_cache_ptrs,
mask=(offs_m[:, None] < dim) & (offs_k_cache[None, :] < write_pos),
other=0.0,
)
x_all_cache = tl.where(
offs_k_cache[None, :] == write_pos, x_cur[:, None], x_all_cache
)
# Decayed checkpoint readout sum_n S_0(m,n) q(n), streamed over NF dstate
# tiles so the (M, N) state slice is never held whole.
offs_nt = tl.arange(0, NF_DSTATE_TILE)
ck_acc = tl.zeros([BLOCK_SIZE_M], dtype=tl.float32)
for i in tl.static_range(NF_NDS):
offs_n = i * NF_DSTATE_TILE + offs_nt
nmask = offs_n < dstate
st = tl.load(
state_ptr
+ offs_m[:, None] * stride_state_dim
+ offs_n[None, :] * stride_state_dstate,
mask=(offs_m[:, None] < dim) & nmask[None, :],
other=0.0,
)
c_chunk = tl.load(
C_ptr + offs_n * stride_C_dstate, mask=nmask, other=0.0
).to(tl.float32)
ck_acc += tl.sum(st.to(tl.float32) * c_chunk[None, :], axis=1)
checkpoint_out = ck_acc * total_decay_cache
bc_cache = tl.load(
bc_pre_ptr + offs_k_cache * stride_bc_pre_pos,
mask=offs_k_cache <= write_pos,
other=0.0,
)
cache_out = tl.sum(
x_all_cache.to(tl.float32) * (scale_cache * bc_cache)[None, :], axis=1
)
out = checkpoint_out + cache_out
# Append the current token (x, dt, B) into the buffer at write_pos.
tl.store(
x_cache_ptr + offs_m * stride_x_cache_dim + write_pos * stride_x_cache_pos,
x_cur,
mask=offs_m < dim,
)
if pid_m == 0:
B_cur = tl.load(
B_ptr + offs_n_full * stride_B_dstate,
mask=offs_n_full < dstate,
other=0.0,
)
tl.store(dt_cache_ptr + write_pos * stride_dt_cache_pos, dt_cur)
tl.store(
B_cache_ptr
+ write_pos * stride_B_cache_pos
+ offs_n_full * stride_B_cache_dstate,
B_cur,
mask=offs_n_full < dstate,
)
else:
# Flush step: state route. Reconstruct the state from cached inputs,
# S_t = total_decay * S_0 + sum_j s_j (v_j k_j^T), persist it as the new
# checkpoint, then read y = S_t q -- streamed over FL dstate tiles.
offs_k_dot = tl.arange(0, BLOCK_SIZE_K_DOT)
dt_all_dot = tl.load(
dt_cache_ptr + offs_k_dot * stride_dt_cache_pos,
mask=offs_k_dot < write_pos,
other=0.0,
).to(tl.float32)
dt_all_dot = tl.where(offs_k_dot == write_pos, dt_cur, dt_all_dot)
dA_cumsum_dot = A * tl.cumsum(dt_all_dot, axis=0)
dA_total_dot = A * tl.sum(dt_all_dot, axis=0)
total_decay_dot = tl.exp(dA_total_dot)
scale_dot = dt_all_dot * tl.exp(dA_total_dot - dA_cumsum_dot)
scale_dot = tl.where(offs_k_dot <= write_pos, scale_dot, 0.0)
# Gather buffered x over the window (history + current token).
x_all_dot_ptrs = (
x_cache_ptr
+ offs_m[:, None] * stride_x_cache_dim
+ offs_k_dot[None, :] * stride_x_cache_pos
)
x_all_dot = tl.load(
x_all_dot_ptrs,
mask=(offs_m[:, None] < dim) & (offs_k_dot[None, :] < write_pos),
other=0.0,
)
x_all_dot = tl.where(
offs_k_dot[None, :] == write_pos, x_cur[:, None], x_all_dot
)
x_all_ty = x_all_dot.to(x_ptr.dtype.element_ty)
# Distinct tile locals (_f) from the nf branch: differing tile widths
# would force a shape-mismatched merge at the if/else exit.
offs_nt_f = tl.arange(0, FL_DSTATE_TILE)
out = tl.zeros([BLOCK_SIZE_M], dtype=tl.float32)
for i in tl.static_range(FL_NDS):
offs_n_f = i * FL_DSTATE_TILE + offs_nt_f
nmask_f = offs_n_f < dstate
# Gather buffered B over the window (history + current token).
B_all_dot = tl.load(
B_cache_ptr
+ offs_k_dot[:, None] * stride_B_cache_pos
+ offs_n_f[None, :] * stride_B_cache_dstate,
mask=(offs_k_dot[:, None] < write_pos) & nmask_f[None, :],
other=0.0,
)
B_cur_tile = tl.load(
B_ptr + offs_n_f * stride_B_dstate, mask=nmask_f, other=0.0
)
B_all_dot = tl.where(
offs_k_dot[:, None] == write_pos, B_cur_tile[None, :], B_all_dot
)
# tf32x3 keeps fp32 parity with the elementwise baseline (plain tf32 on
# fp32 inputs drifts ~1e-2); bf16/fp16 inputs are unaffected by this flag.
B_scaled = (B_all_dot.to(tl.float32) * scale_dot[:, None]).to(
x_ptr.dtype.element_ty
)
delta_state = tl.dot(x_all_ty, B_scaled, input_precision="tf32x3")
state_ptrs = (
state_ptr
+ offs_m[:, None] * stride_state_dim
+ offs_n_f[None, :] * stride_state_dstate
)
state_mask = (offs_m[:, None] < dim) & nmask_f[None, :]
st_f = tl.load(state_ptrs, mask=state_mask, other=0.0)
state_new = st_f.to(tl.float32) * total_decay_dot + delta_state.to(
tl.float32
)
if USE_RS_ROUNDING:
# Stochastic-round fp32->fp16 (Blackwell cvt.rs), mirroring the
# baseline. Only the flush step stores state, so this runs at 1/L
# the baseline's per-step rate. Absolute per-element offsets seed
# the RNG so each state element draws independently.
rand_seed = tl.load(rand_seed_ptr)
rand_offsets = (
state_batch_idx * stride_state_batch
+ pid_h * stride_state_head
+ offs_m[:, None] * stride_state_dim
+ offs_n_f[None, :] * stride_state_dstate
)
if PHILOX_ROUNDS > 0:
rand = tl.randint(rand_seed, rand_offsets, PHILOX_ROUNDS)
else:
rand = tl.randint(rand_seed, rand_offsets)
tl.static_assert(
state_ptrs.dtype.element_ty == tl.float16,
"stochastic rounding requires an fp16 SSM state cache",
)
state_store = convert_rs_fp16x2(state_new, rand)
else:
state_store = state_new.to(st_f.dtype)
tl.store(state_ptrs, state_store, mask=state_mask)
c_chunk_f = tl.load(
C_ptr + offs_n_f * stride_C_dstate, mask=nmask_f, other=0.0
).to(tl.float32)
out += tl.sum(state_new * c_chunk_f[None, :], axis=1)
# Skip connection (D) and output gate (z).
if HAS_D:
D_ptr += pid_h * stride_D_head
D = tl.load(D_ptr + offs_m * stride_D_dim, mask=offs_m < dim, other=0.0).to(
tl.float32
)
out += x_cur.to(tl.float32) * D
if HAS_Z:
z_ptr += pid_b * stride_z_batch + pid_h * stride_z_head
z = tl.load(z_ptr + offs_m * stride_z_dim, mask=offs_m < dim, other=0.0).to(
tl.float32
)
out *= z * tl.sigmoid(z)
tl.store(out_ptr + offs_m * stride_out_dim, out, mask=offs_m < dim)
def selective_state_update_replayssm_output_only(
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D: torch.Tensor | None = None,
dt_bias: torch.Tensor | None = None,
z: torch.Tensor | None = None,
dt_softplus: bool = False,
x_cache: torch.Tensor | None = None,
dt_cache: torch.Tensor | None = None,
B_cache: torch.Tensor | None = None,
bc_pre: torch.Tensor | None = None,
write_pos: torch.Tensor | None = None,
is_flush: torch.Tensor | None = None,
max_cache_len: int = 16,
state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
out: torch.Tensor | None = None,
enable_stochastic_rounding: bool = False,
cache_philox_rounds: int = 0,
) -> torch.Tensor:
"""Cached-bc SSM update for vLLM's autoregressive Mamba2 decode path."""
has_heads = state.dim() > 3
if state.dim() == 3:
state = state.unsqueeze(1)
if x.dim() == 2:
x = x.unsqueeze(1)
if dt.dim() == 2:
dt = dt.unsqueeze(1)
if A.dim() == 2:
A = A.unsqueeze(0)
if B.dim() == 2:
B = B.unsqueeze(1)
if C.dim() == 2:
C = C.unsqueeze(1)
if D is not None and D.dim() == 1:
D = D.unsqueeze(0)
if z is not None and z.dim() == 2:
z = z.unsqueeze(1)
if dt_bias is not None and dt_bias.dim() == 1:
dt_bias = dt_bias.unsqueeze(0)
if out is not None and out.dim() == 2:
out = out.unsqueeze(1)
if state_batch_indices is not None and state_batch_indices.dim() == 1:
state_batch_indices = state_batch_indices.unsqueeze(1)
_, nheads, dim, dstate = state.shape
batch = x.shape[0]
assert x.shape == (batch, nheads, dim)
assert dt.shape == x.shape
assert A.shape == (nheads, dim, dstate)
ngroups = B.shape[1]
assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
assert B.shape == (batch, ngroups, dstate)
assert C.shape == B.shape
if D is not None:
assert D.shape == (nheads, dim)
if z is not None:
assert z.shape == x.shape
if dt_bias is not None:
assert dt_bias.shape == (nheads, dim)
assert out is not None and out.shape == x.shape
assert A.stride(-1) == 0 and A.stride(-2) == 0, (
"Cached kernel requires TIE_HDIM (A scalar per head)"
)
assert dt.stride(-1) == 0, "Cached kernel requires TIE_HDIM (dt scalar per head)"
if dt_bias is not None:
assert dt_bias.stride(-1) == 0, (
"Cached kernel requires TIE_HDIM (dt_bias scalar per head)"
)
assert x_cache is not None
assert dt_cache is not None
assert B_cache is not None
assert x_cache.shape[1:] == (nheads, max_cache_len, dim)
assert dt_cache.shape[1:] == (nheads, max_cache_len)
assert B_cache.shape[1:] == (ngroups, max_cache_len, dstate)
assert write_pos is not None and write_pos.shape[0] >= batch
assert write_pos.dtype == torch.int32
assert is_flush is not None and is_flush.shape[0] >= batch
assert is_flush.dtype in (torch.bool, torch.int8)
assert bc_pre is not None
assert bc_pre.shape[0] >= batch and bc_pre.shape[1] >= ngroups
assert bc_pre.shape[2] == max_cache_len
assert bc_pre.dtype == torch.float32
if state_batch_indices is not None:
assert state_batch_indices.shape[0] >= batch
assert state_batch_indices.shape[1] >= 1
block_size_k_cache = max(1, triton.next_power_of_2(max_cache_len))
block_size_k_dot = max(16, block_size_k_cache)
block_size_m, num_warps, nf_tile, fl_tile, num_stages = get_replayssm_config(
"mamba2_output_only", dstate=dstate, L=max_cache_len
)
bs_dstate = triton.next_power_of_2(dstate)
nf_dstate_tile = max(16, min(nf_tile, bs_dstate))
nf_nds = triton.cdiv(bs_dstate, nf_dstate_tile)
fl_dstate_tile = max(16, min(fl_tile, bs_dstate))
fl_nds = triton.cdiv(bs_dstate, fl_dstate_tile)
grid = lambda META: (triton.cdiv(dim, META["BLOCK_SIZE_M"]), batch, nheads)
z_strides = (z.stride(0), z.stride(1), z.stride(2)) if z is not None else (0, 0, 0)
state_indices_strides = (
(state_batch_indices.stride(0), state_batch_indices.stride(1))
if state_batch_indices is not None
else (0, 0)
)
rand_seed = (
torch.randint(0, 2**32, (1,), device=state.device)
if enable_stochastic_rounding
else None
)
with torch.accelerator.device_index(x.device.index):
# Both kernels always launch: the precompute kernel self-skips flush
# rows per row (the branch can't be hoisted out under CUDA graphs), so
# it is a no-op when every row is flushing.
_replayssm_output_only_precompute_kernel[(batch, ngroups)](
B,
C,
B_cache,
write_pos,
is_flush,
bc_pre,
state_batch_indices,
null_block_id,
batch,
ngroups,
dstate,
B.stride(0),
B.stride(1),
B.stride(2),
C.stride(0),
C.stride(1),
C.stride(2),
B_cache.stride(0),
B_cache.stride(1),
B_cache.stride(2),
B_cache.stride(3),
bc_pre.stride(0),
bc_pre.stride(1),
bc_pre.stride(2),
state_indices_strides[0],
state_indices_strides[1],
max_cache_len,
block_size_k_cache,
num_warps=2,
)
_replayssm_output_only_kernel[grid](
state,
rand_seed,
x,
dt,
dt_bias,
A,
B,
C,
D,
z,
out,
x_cache,
dt_cache,
B_cache,
bc_pre,
write_pos,
is_flush,
state_batch_indices,
null_block_id,
batch,
nheads,
dim,
dstate,
nheads // ngroups,
state.stride(0),
state.stride(1),
state.stride(2),
state.stride(3),
x.stride(0),
x.stride(1),
x.stride(2),
dt.stride(0),
dt.stride(1),
dt_bias.stride(0) if dt_bias is not None else 0,
A.stride(0),
B.stride(0),
B.stride(1),
B.stride(2),
C.stride(0),
C.stride(1),
C.stride(2),
D.stride(0) if D is not None else 0,
D.stride(1) if D is not None else 0,
z_strides[0],
z_strides[1],
z_strides[2],
out.stride(0),
out.stride(1),
out.stride(2),
x_cache.stride(0),
x_cache.stride(1),
x_cache.stride(3),
x_cache.stride(2),
dt_cache.stride(0),
dt_cache.stride(1),
dt_cache.stride(2),
B_cache.stride(0),
B_cache.stride(1),
B_cache.stride(2),
B_cache.stride(3),
bc_pre.stride(0),
bc_pre.stride(1),
bc_pre.stride(2),
state_indices_strides[0],
state_indices_strides[1],
dt_softplus,
max_cache_len,
block_size_m,
block_size_k_cache,
block_size_k_dot,
nf_dstate_tile,
nf_nds,
fl_dstate_tile,
fl_nds,
enable_stochastic_rounding,
cache_philox_rounds,
num_warps=num_warps,
num_stages=num_stages,
)
if not has_heads:
out = out.squeeze(1)
return out
+25
View File
@@ -984,6 +984,31 @@ def supports_mamba_prefix_caching(
return getattr(model, "supports_mamba_prefix_caching", False)
@runtime_checkable
class SupportsReplaySSM(Protocol):
"""The interface for models whose Mamba2 layers support ReplaySSM cached
standard decode.
This is currently experimental.
"""
supports_replayssm: ClassVar[Literal[True]] = True
@overload
def supports_replayssm(model: object) -> TypeIs[SupportsReplaySSM]: ...
@overload
def supports_replayssm(model: type[object]) -> TypeIs[type[SupportsReplaySSM]]: ...
def supports_replayssm(
model: type[object] | object,
) -> TypeIs[type[SupportsReplaySSM]] | TypeIs[SupportsReplaySSM]:
return getattr(model, "supports_replayssm", False)
@runtime_checkable
class SupportsCrossEncoding(Protocol):
"""The interface required for all models that support cross encoding."""
+25 -6
View File
@@ -69,6 +69,7 @@ from vllm.model_executor.models.interfaces import (
SupportsMambaPrefixCaching,
SupportsPP,
SupportsQuant,
SupportsReplaySSM,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -707,6 +708,7 @@ class NemotronHForCausalLM(
SupportsQuant,
MixtureOfExperts,
SupportsMambaPrefixCaching,
SupportsReplaySSM,
):
# Relevant only if self.has_moe is True
is_non_gated_moe: bool = True
@@ -742,18 +744,24 @@ class NemotronHForCausalLM(
def get_mamba_state_dtype_from_config(
cls,
vllm_config: "VllmConfig",
) -> tuple[torch.dtype, torch.dtype]:
return MambaStateDtypeCalculator.mamba2_state_dtype(
) -> tuple[torch.dtype, ...]:
cache_config = vllm_config.cache_config
base_dtype = MambaStateDtypeCalculator.mamba2_state_dtype(
vllm_config.model_config.dtype,
vllm_config.cache_config.mamba_cache_dtype,
vllm_config.cache_config.mamba_ssm_cache_dtype,
cache_config.mamba_cache_dtype,
cache_config.mamba_ssm_cache_dtype,
)
if cache_config.use_replayssm:
return MambaStateDtypeCalculator.append_replayssm_ring(
base_dtype, vllm_config.model_config.dtype
)
return base_dtype
@classmethod
def get_mamba_state_shape_from_config(
cls,
vllm_config: "VllmConfig",
) -> tuple[tuple[int, int], tuple[int, int, int]]:
) -> tuple[tuple[int, ...], ...]:
"""Calculate shapes for Mamba's convolutional and state caches.
Args:
@@ -763,12 +771,15 @@ class NemotronHForCausalLM(
Tuple containing:
- conv_state_shape: Shape for convolutional state cache
- temporal_state_shape: Shape for state space model cache
- (when use_replayssm is enabled) the x_cache/dt_cache/B_cache
ring-buffer shapes
"""
parallel_config = vllm_config.parallel_config
cache_config = vllm_config.cache_config
hf_config = vllm_config.model_config.hf_config
intermediate_size = hf_config.mamba_num_heads * hf_config.mamba_head_dim
return MambaStateShapeCalculator.mamba2_state_shape(
base_shape = MambaStateShapeCalculator.mamba2_state_shape(
intermediate_size=intermediate_size,
tp_world_size=parallel_config.tensor_parallel_size,
n_groups=hf_config.n_groups,
@@ -778,6 +789,14 @@ class NemotronHForCausalLM(
conv_kernel=hf_config.conv_kernel,
num_spec=vllm_config.num_speculative_tokens,
)
if cache_config.use_replayssm:
return MambaStateShapeCalculator.append_replayssm_ring(
base_shape,
hf_config.n_groups,
parallel_config.tensor_parallel_size,
cache_config.replayssm_buffer_len,
)
return base_shape
@classmethod
def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
+3
View File
@@ -55,6 +55,7 @@ from .interfaces import (
supports_multimodal_encoder_tp_data,
supports_multimodal_raw_input_only,
supports_pp,
supports_replayssm,
supports_transcription,
)
from .interfaces_base import (
@@ -789,6 +790,7 @@ class _ModelInfo:
is_hybrid: bool
has_noops: bool
supports_mamba_prefix_caching: bool
supports_replayssm: bool
supports_transcription: bool
supports_transcription_only: bool
@@ -815,6 +817,7 @@ class _ModelInfo:
is_attention_free=is_attention_free(model),
is_hybrid=is_hybrid(model),
supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
supports_replayssm=supports_replayssm(model),
supports_transcription=supports_transcription(model),
supports_transcription_only=(
supports_transcription(model) and model.supports_transcription_only