[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)
Signed-off-by: Johnny-Liou <a897111@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 4.8
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
tomeras91
Cyrus Leung
parent
d02df748bf
commit
866fea2b99
@@ -32,6 +32,9 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -494,12 +497,27 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
# The tuple is (conv_state, ssm_state)
|
||||
self.kv_cache = (torch.tensor([]), torch.tensor([]))
|
||||
|
||||
self.model_config = model_config
|
||||
self.cache_config = cache_config
|
||||
self.prefix = prefix
|
||||
self.use_replayssm = (
|
||||
cache_config.use_replayssm if cache_config is not None else False
|
||||
)
|
||||
self.replayssm_buffer_len = (
|
||||
cache_config.replayssm_buffer_len
|
||||
if cache_config is not None and cache_config.use_replayssm
|
||||
else None
|
||||
)
|
||||
self.mamba_config = vllm_config.mamba_config
|
||||
if self.use_replayssm and self.num_heads % self.tp_size != 0:
|
||||
raise ValueError(
|
||||
"--use-replayssm requires tensor-parallel heads to divide evenly"
|
||||
)
|
||||
# The tuple is (conv_state, ssm_state); with the cached (ReplaySSM) decode
|
||||
# kernel enabled it is (conv_state, ssm_state, x_cache, dt_cache, B_cache).
|
||||
_n_state = 5 if self.use_replayssm else 2
|
||||
self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state))
|
||||
|
||||
self.num_spec = vllm_config.num_speculative_tokens
|
||||
if self.num_spec > 0:
|
||||
@@ -591,7 +609,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
|
||||
# Triton's autotuner includes tensor dtypes in its cache key,
|
||||
# so state_dtype must match what real inference uses.
|
||||
_, ssm_state_dtype = self.get_state_dtype()
|
||||
ssm_state_dtype = self.get_state_dtype()[1]
|
||||
|
||||
# SSD kernel autotune keys depend on dtype and head dimensions,
|
||||
# not on sequence length or batch size, so a single shape suffices.
|
||||
@@ -702,6 +720,10 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
ssm_state = self.kv_cache[1]
|
||||
if self.use_replayssm:
|
||||
x_cache, dt_cache, B_cache = self.kv_cache[2:]
|
||||
else:
|
||||
x_cache = dt_cache = B_cache = None
|
||||
has_initial_states_p = attn_metadata.has_initial_states_p
|
||||
prep_initial_states = attn_metadata.prep_initial_states
|
||||
chunk_size = attn_metadata.chunk_size
|
||||
@@ -1027,37 +1049,78 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# - mamba_cache_params.ssm_state's slots will be selected
|
||||
# using state_indices_tensor_d
|
||||
# NOTE: final output is an in-place update of out tensor
|
||||
selective_state_update(
|
||||
ssm_state,
|
||||
hidden_states_d,
|
||||
dt_d,
|
||||
A_d,
|
||||
B_d,
|
||||
C_d,
|
||||
D_d,
|
||||
dt_bias,
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
dst_state_batch_indices=state_indices_tensor_d_output,
|
||||
out=preallocated_ssm_out_d.view(num_decode_tokens, -1, self.head_dim),
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
cu_seqlens=query_start_loc_d,
|
||||
is_blackwell=self.is_blackwell,
|
||||
preallocated_ssm_out_d = preallocated_ssm_out_d.view(
|
||||
num_decode_tokens, -1, self.head_dim
|
||||
)
|
||||
if self.use_replayssm:
|
||||
assert self.replayssm_buffer_len is not None
|
||||
selective_state_update_replayssm_output_only(
|
||||
ssm_state,
|
||||
hidden_states_d,
|
||||
dt_d,
|
||||
A_d,
|
||||
B_d,
|
||||
C_d,
|
||||
D_d,
|
||||
dt_bias,
|
||||
dt_softplus=True,
|
||||
x_cache=x_cache,
|
||||
dt_cache=dt_cache,
|
||||
B_cache=B_cache,
|
||||
bc_pre=attn_metadata.bc_pre_scratch,
|
||||
write_pos=attn_metadata.write_pos_d,
|
||||
is_flush=attn_metadata.is_flush_d,
|
||||
max_cache_len=self.replayssm_buffer_len,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
out=preallocated_ssm_out_d,
|
||||
# Stochastic Rounding for the vanilla decode path is read
|
||||
# from mamba_config inside ssu_dispatch; the replay kernel
|
||||
# isn't a dispatch backend, so pass it here.
|
||||
enable_stochastic_rounding=(
|
||||
self.mamba_config.enable_stochastic_rounding
|
||||
),
|
||||
cache_philox_rounds=(
|
||||
self.mamba_config.stochastic_rounding_philox_rounds
|
||||
),
|
||||
)
|
||||
else:
|
||||
selective_state_update(
|
||||
ssm_state,
|
||||
hidden_states_d,
|
||||
dt_d,
|
||||
A_d,
|
||||
B_d,
|
||||
C_d,
|
||||
D_d,
|
||||
dt_bias,
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
dst_state_batch_indices=state_indices_tensor_d_output,
|
||||
out=preallocated_ssm_out_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
cu_seqlens=query_start_loc_d,
|
||||
is_blackwell=self.is_blackwell,
|
||||
)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.mamba2_state_dtype(
|
||||
base_dtype = MambaStateDtypeCalculator.mamba2_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
self.cache_config.mamba_ssm_cache_dtype,
|
||||
)
|
||||
if self.use_replayssm:
|
||||
return MambaStateDtypeCalculator.append_replayssm_ring(
|
||||
base_dtype, self.model_config.dtype
|
||||
)
|
||||
return base_dtype
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.mamba2_state_shape(
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
|
||||
tp_world_size = get_tensor_model_parallel_world_size()
|
||||
base_shape = MambaStateShapeCalculator.mamba2_state_shape(
|
||||
intermediate_size=self.intermediate_size,
|
||||
tp_world_size=get_tensor_model_parallel_world_size(),
|
||||
tp_world_size=tp_world_size,
|
||||
n_groups=self.n_groups,
|
||||
num_heads=self.num_heads,
|
||||
head_dim=self.head_dim,
|
||||
@@ -1065,6 +1128,15 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
conv_kernel=self.conv_kernel_size,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
if self.use_replayssm:
|
||||
assert self.replayssm_buffer_len is not None
|
||||
return MambaStateShapeCalculator.append_replayssm_ring(
|
||||
base_shape,
|
||||
self.n_groups,
|
||||
tp_world_size,
|
||||
self.replayssm_buffer_len,
|
||||
)
|
||||
return base_shape
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
|
||||
@@ -80,6 +80,18 @@ class MambaStateDtypeCalculator:
|
||||
model_dtype, mamba_cache_dtype, mamba_ssm_cache_dtype
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def append_replayssm_ring(
|
||||
cls,
|
||||
base_dtypes: tuple[torch.dtype, ...],
|
||||
model_dtype: ModelDType | torch.dtype,
|
||||
) -> tuple[torch.dtype, ...]:
|
||||
"""Append the ReplaySSM ring dtypes to a base ``(conv, ssm)`` tuple:
|
||||
``(x_cache, dt_cache, B_cache)`` = ``(activation, fp32, activation)``.
|
||||
"""
|
||||
activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype)
|
||||
return (*base_dtypes, activation_dtype, torch.float32, activation_dtype)
|
||||
|
||||
@classmethod
|
||||
def _mamba_state_dtype(
|
||||
cls,
|
||||
@@ -186,6 +198,28 @@ class MambaStateShapeCalculator:
|
||||
temporal_state_shape = (divide(num_heads, tp_world_size), head_dim, state_size)
|
||||
return conv_state_shape, temporal_state_shape
|
||||
|
||||
@classmethod
|
||||
def append_replayssm_ring(
|
||||
cls,
|
||||
base_shapes: tuple[tuple[int, ...], ...],
|
||||
n_groups: int,
|
||||
tp_world_size: int,
|
||||
replayssm_buffer_len: int,
|
||||
) -> tuple[tuple[int, ...], ...]:
|
||||
"""Append the ReplaySSM ring shapes (x_cache, dt_cache, B_cache) to a
|
||||
base ``(conv, ssm)`` tuple. ``base_shapes[1]`` is the ssm shape
|
||||
``(nheads // tp, head_dim, state_size)``; B_cache uses the un-extended
|
||||
``n_groups``.
|
||||
"""
|
||||
local_nheads, head_dim, state_size = base_shapes[1]
|
||||
local_ngroups = divide(n_groups, tp_world_size)
|
||||
return (
|
||||
*base_shapes,
|
||||
(local_nheads, replayssm_buffer_len, head_dim),
|
||||
(local_nheads, replayssm_buffer_len),
|
||||
(local_ngroups, replayssm_buffer_len, state_size),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def short_conv_state_shape(
|
||||
cls,
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Launch-config selection for the ReplaySSM Mamba2 output_only decode kernel.
|
||||
|
||||
Mirrors ``mamba_ssm.py``: a hard-coded heuristic per kernel, plus an
|
||||
``override`` context manager for benchmarks/tests/config sweeps. Hardware is
|
||||
auto-detected (Blackwell vs not) so call sites need not thread it through.
|
||||
"""
|
||||
|
||||
import functools
|
||||
from contextlib import contextmanager
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _is_blackwell() -> bool:
|
||||
try:
|
||||
return current_platform.is_device_capability_family(100)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Per-kernel overrides keyed by the kernel name passed to get_replayssm_config.
|
||||
_overrides: dict[str, tuple] = {}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def override_replayssm_config(kernel: str, config: tuple):
|
||||
"""Pin ``kernel``'s launch config for the duration of the context."""
|
||||
prev = _overrides.get(kernel)
|
||||
_overrides[kernel] = config
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if prev is None:
|
||||
_overrides.pop(kernel, None)
|
||||
else:
|
||||
_overrides[kernel] = prev
|
||||
|
||||
|
||||
def _dstate_tile(dstate: int, tile: int) -> int:
|
||||
return max(16, min(tile, triton.next_power_of_2(dstate)))
|
||||
|
||||
|
||||
def _mamba2_output_only(dstate, L, is_blackwell):
|
||||
# (block_size_m, num_warps, nf_dstate_tile, fl_dstate_tile, num_stages);
|
||||
# decoupled dstate tiling, serving-batch optimum, dtype-independent per device.
|
||||
if is_blackwell:
|
||||
return 64, 1, _dstate_tile(dstate, 32), _dstate_tile(dstate, 64), 2
|
||||
return 16, 1, _dstate_tile(dstate, 64), _dstate_tile(dstate, 128), 2
|
||||
|
||||
|
||||
def get_replayssm_config(kernel: str, **shape) -> tuple:
|
||||
"""Return the launch config for ``kernel`` (override > tuned default).
|
||||
|
||||
kernel: "mamba2_output_only". ``shape`` carries the keying dims (dstate;
|
||||
``L`` for the buffer length, default 16); hardware is auto-detected.
|
||||
"""
|
||||
if kernel in _overrides:
|
||||
return _overrides[kernel]
|
||||
bw = _is_blackwell()
|
||||
if kernel == "mamba2_output_only":
|
||||
return _mamba2_output_only(shape["dstate"], shape.get("L", 16), bw)
|
||||
raise ValueError(f"unknown ReplaySSM kernel config key: {kernel}")
|
||||
@@ -0,0 +1,709 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import convert_rs_fp16x2, softplus
|
||||
from vllm.model_executor.layers.mamba.ops.replayssm_config import get_replayssm_config
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_STATE_BATCH_INDICES": lambda args: args["state_batch_indices_ptr"]
|
||||
is not None
|
||||
}
|
||||
)
|
||||
@triton.heuristics(
|
||||
{"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])}
|
||||
)
|
||||
@triton.jit
|
||||
def _replayssm_output_only_precompute_kernel(
|
||||
B_ptr,
|
||||
C_ptr,
|
||||
B_cache_ptr,
|
||||
write_pos_ptr,
|
||||
is_flush_ptr,
|
||||
bc_pre_ptr,
|
||||
state_batch_indices_ptr,
|
||||
null_block_id,
|
||||
# Matrix dimensions
|
||||
batch,
|
||||
ngroups,
|
||||
dstate,
|
||||
# Input strides
|
||||
stride_B_batch,
|
||||
stride_B_group,
|
||||
stride_B_dstate,
|
||||
stride_C_batch,
|
||||
stride_C_group,
|
||||
stride_C_dstate,
|
||||
# Cache strides
|
||||
stride_B_cache_batch,
|
||||
stride_B_cache_group,
|
||||
stride_B_cache_pos,
|
||||
stride_B_cache_dstate,
|
||||
stride_bc_pre_batch,
|
||||
stride_bc_pre_group,
|
||||
stride_bc_pre_pos,
|
||||
stride_state_indices_batch,
|
||||
stride_state_indices_T,
|
||||
# Meta-parameters
|
||||
MAX_CACHE_LEN: tl.constexpr,
|
||||
BLOCK_SIZE_K: tl.constexpr,
|
||||
# heuristic-computed
|
||||
BLOCK_SIZE_DSTATE: tl.constexpr,
|
||||
HAS_STATE_BATCH_INDICES: tl.constexpr,
|
||||
):
|
||||
pid_b = tl.program_id(axis=0)
|
||||
pid_g = tl.program_id(axis=1)
|
||||
|
||||
# On flush steps the main kernel does not read bc_pre, so skip the work.
|
||||
is_flush = tl.load(is_flush_ptr + pid_b) != 0
|
||||
if is_flush:
|
||||
return
|
||||
|
||||
if HAS_STATE_BATCH_INDICES:
|
||||
state_batch_idx = tl.load(
|
||||
state_batch_indices_ptr
|
||||
+ pid_b * stride_state_indices_batch
|
||||
+ 0 * stride_state_indices_T
|
||||
).to(tl.int64)
|
||||
if state_batch_idx == null_block_id:
|
||||
return
|
||||
else:
|
||||
state_batch_idx = pid_b
|
||||
|
||||
offs_k = tl.arange(0, BLOCK_SIZE_K)
|
||||
offs_n = tl.arange(0, BLOCK_SIZE_DSTATE)
|
||||
|
||||
write_pos = tl.load(write_pos_ptr + pid_b).to(tl.int64)
|
||||
|
||||
B_ptr += pid_b * stride_B_batch + pid_g * stride_B_group
|
||||
C_ptr += pid_b * stride_C_batch + pid_g * stride_C_group
|
||||
B_cache_ptr += state_batch_idx * stride_B_cache_batch + pid_g * stride_B_cache_group
|
||||
bc_pre_ptr += pid_b * stride_bc_pre_batch + pid_g * stride_bc_pre_group
|
||||
|
||||
B_cur = tl.load(
|
||||
B_ptr + offs_n * stride_B_dstate,
|
||||
mask=offs_n < dstate,
|
||||
other=0.0,
|
||||
)
|
||||
C = tl.load(
|
||||
C_ptr + offs_n * stride_C_dstate,
|
||||
mask=offs_n < dstate,
|
||||
other=0.0,
|
||||
)
|
||||
B_cache_ptrs = (
|
||||
B_cache_ptr
|
||||
+ offs_k[:, None] * stride_B_cache_pos
|
||||
+ offs_n[None, :] * stride_B_cache_dstate
|
||||
)
|
||||
B_cache = tl.load(
|
||||
B_cache_ptrs,
|
||||
mask=(offs_k[:, None] < write_pos) & (offs_n[None, :] < dstate),
|
||||
other=0.0,
|
||||
)
|
||||
B_all = tl.where(offs_k[:, None] == write_pos, B_cur[None, :], B_cache)
|
||||
bc = tl.sum(B_all.to(tl.float32) * C[None, :].to(tl.float32), axis=1)
|
||||
|
||||
tl.store(
|
||||
bc_pre_ptr + offs_k * stride_bc_pre_pos,
|
||||
bc,
|
||||
mask=(offs_k <= write_pos) & (offs_k < MAX_CACHE_LEN),
|
||||
)
|
||||
|
||||
|
||||
@triton.heuristics({"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None})
|
||||
@triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None})
|
||||
@triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None})
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_STATE_BATCH_INDICES": lambda args: args["state_batch_indices_ptr"]
|
||||
is not None
|
||||
}
|
||||
)
|
||||
@triton.heuristics(
|
||||
{"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])}
|
||||
)
|
||||
@triton.jit
|
||||
def _replayssm_output_only_kernel(
|
||||
# Pointers to matrices
|
||||
state_ptr,
|
||||
rand_seed_ptr,
|
||||
x_ptr,
|
||||
dt_ptr,
|
||||
dt_bias_ptr,
|
||||
A_ptr,
|
||||
B_ptr,
|
||||
C_ptr,
|
||||
D_ptr,
|
||||
z_ptr,
|
||||
out_ptr,
|
||||
x_cache_ptr,
|
||||
dt_cache_ptr,
|
||||
B_cache_ptr,
|
||||
bc_pre_ptr,
|
||||
write_pos_ptr,
|
||||
is_flush_ptr,
|
||||
state_batch_indices_ptr,
|
||||
null_block_id,
|
||||
# Matrix dimensions
|
||||
batch,
|
||||
nheads,
|
||||
dim,
|
||||
dstate,
|
||||
nheads_ngroups_ratio,
|
||||
# State strides
|
||||
stride_state_batch,
|
||||
stride_state_head,
|
||||
stride_state_dim,
|
||||
stride_state_dstate,
|
||||
# Input strides
|
||||
stride_x_batch,
|
||||
stride_x_head,
|
||||
stride_x_dim,
|
||||
stride_dt_batch,
|
||||
stride_dt_head,
|
||||
stride_dt_bias_head,
|
||||
stride_A_head,
|
||||
stride_B_batch,
|
||||
stride_B_group,
|
||||
stride_B_dstate,
|
||||
stride_C_batch,
|
||||
stride_C_group,
|
||||
stride_C_dstate,
|
||||
stride_D_head,
|
||||
stride_D_dim,
|
||||
stride_z_batch,
|
||||
stride_z_head,
|
||||
stride_z_dim,
|
||||
stride_out_batch,
|
||||
stride_out_head,
|
||||
stride_out_dim,
|
||||
# Cache strides
|
||||
stride_x_cache_batch,
|
||||
stride_x_cache_head,
|
||||
stride_x_cache_dim,
|
||||
stride_x_cache_pos,
|
||||
stride_dt_cache_batch,
|
||||
stride_dt_cache_head,
|
||||
stride_dt_cache_pos,
|
||||
stride_B_cache_batch,
|
||||
stride_B_cache_group,
|
||||
stride_B_cache_pos,
|
||||
stride_B_cache_dstate,
|
||||
stride_bc_pre_batch,
|
||||
stride_bc_pre_group,
|
||||
stride_bc_pre_pos,
|
||||
stride_state_indices_batch,
|
||||
stride_state_indices_T,
|
||||
# Meta-parameters
|
||||
DT_SOFTPLUS: tl.constexpr,
|
||||
MAX_CACHE_LEN: tl.constexpr,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
BLOCK_SIZE_K_CACHE: tl.constexpr,
|
||||
BLOCK_SIZE_K_DOT: tl.constexpr,
|
||||
NF_DSTATE_TILE: tl.constexpr,
|
||||
NF_NDS: tl.constexpr,
|
||||
FL_DSTATE_TILE: tl.constexpr,
|
||||
FL_NDS: tl.constexpr,
|
||||
USE_RS_ROUNDING: tl.constexpr,
|
||||
PHILOX_ROUNDS: tl.constexpr,
|
||||
# heuristic-computed
|
||||
BLOCK_SIZE_DSTATE: tl.constexpr,
|
||||
HAS_DT_BIAS: tl.constexpr,
|
||||
HAS_D: tl.constexpr,
|
||||
HAS_Z: tl.constexpr,
|
||||
HAS_STATE_BATCH_INDICES: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(axis=0)
|
||||
pid_b = tl.program_id(axis=1)
|
||||
pid_h = tl.program_id(axis=2)
|
||||
|
||||
# Resolve the physical state slot for this decode row; skip padded rows.
|
||||
if HAS_STATE_BATCH_INDICES:
|
||||
state_batch_idx = tl.load(
|
||||
state_batch_indices_ptr
|
||||
+ pid_b * stride_state_indices_batch
|
||||
+ 0 * stride_state_indices_T
|
||||
).to(tl.int64)
|
||||
if state_batch_idx == null_block_id:
|
||||
return
|
||||
else:
|
||||
state_batch_idx = pid_b
|
||||
|
||||
offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
|
||||
offs_n_full = tl.arange(0, BLOCK_SIZE_DSTATE)
|
||||
|
||||
# Buffer cursor (number of cached tokens so far) and the flush flag.
|
||||
write_pos = tl.load(write_pos_ptr + pid_b).to(tl.int64)
|
||||
is_flush = tl.load(is_flush_ptr + pid_b) != 0
|
||||
|
||||
# Advance every pointer to this (row, head, group).
|
||||
state_ptr += state_batch_idx * stride_state_batch + pid_h * stride_state_head
|
||||
x_ptr += pid_b * stride_x_batch + pid_h * stride_x_head
|
||||
dt_ptr += pid_b * stride_dt_batch + pid_h * stride_dt_head
|
||||
B_ptr += pid_b * stride_B_batch + (pid_h // nheads_ngroups_ratio) * stride_B_group
|
||||
C_ptr += pid_b * stride_C_batch + (pid_h // nheads_ngroups_ratio) * stride_C_group
|
||||
out_ptr += pid_b * stride_out_batch + pid_h * stride_out_head
|
||||
x_cache_ptr += state_batch_idx * stride_x_cache_batch + pid_h * stride_x_cache_head
|
||||
dt_cache_ptr += (
|
||||
state_batch_idx * stride_dt_cache_batch + pid_h * stride_dt_cache_head
|
||||
)
|
||||
B_cache_ptr += (
|
||||
state_batch_idx * stride_B_cache_batch
|
||||
+ (pid_h // nheads_ngroups_ratio) * stride_B_cache_group
|
||||
)
|
||||
bc_pre_ptr += (
|
||||
pid_b * stride_bc_pre_batch
|
||||
+ (pid_h // nheads_ngroups_ratio) * stride_bc_pre_group
|
||||
)
|
||||
|
||||
# Current-token dt (+ bias, softplus), scalar A, and current x. C, the
|
||||
# checkpoint state S_0, and current-token B are read per dstate tile below.
|
||||
dt_cur = tl.load(dt_ptr).to(tl.float32)
|
||||
if HAS_DT_BIAS:
|
||||
dt_cur += tl.load(dt_bias_ptr + pid_h * stride_dt_bias_head).to(tl.float32)
|
||||
if DT_SOFTPLUS:
|
||||
dt_cur = tl.where(dt_cur <= 20.0, softplus(dt_cur), dt_cur)
|
||||
A = tl.load(A_ptr + pid_h * stride_A_head).to(tl.float32)
|
||||
x_cur = tl.load(x_ptr + offs_m * stride_x_dim, mask=offs_m < dim, other=0.0)
|
||||
|
||||
if not is_flush:
|
||||
# Output-only route: read y without materializing the state, using the
|
||||
# precomputed k^T q products (`bc`):
|
||||
# y = total_decay * (S_0 q) + sum_j s_j (k_j^T q) v_j.
|
||||
# Then append the current token to the buffer.
|
||||
offs_k_cache = tl.arange(0, BLOCK_SIZE_K_CACHE)
|
||||
# dt over the window (history + current token), then the decay weights.
|
||||
dt_all_cache = tl.load(
|
||||
dt_cache_ptr + offs_k_cache * stride_dt_cache_pos,
|
||||
mask=offs_k_cache < write_pos,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
dt_all_cache = tl.where(offs_k_cache == write_pos, dt_cur, dt_all_cache)
|
||||
dA_cumsum_cache = A * tl.cumsum(dt_all_cache, axis=0)
|
||||
dA_total_cache = A * tl.sum(dt_all_cache, axis=0)
|
||||
total_decay_cache = tl.exp(dA_total_cache)
|
||||
scale_cache = dt_all_cache * tl.exp(dA_total_cache - dA_cumsum_cache)
|
||||
scale_cache = tl.where(offs_k_cache <= write_pos, scale_cache, 0.0)
|
||||
|
||||
# Gather buffered x over the window (history + current token).
|
||||
x_all_cache_ptrs = (
|
||||
x_cache_ptr
|
||||
+ offs_m[:, None] * stride_x_cache_dim
|
||||
+ offs_k_cache[None, :] * stride_x_cache_pos
|
||||
)
|
||||
x_all_cache = tl.load(
|
||||
x_all_cache_ptrs,
|
||||
mask=(offs_m[:, None] < dim) & (offs_k_cache[None, :] < write_pos),
|
||||
other=0.0,
|
||||
)
|
||||
x_all_cache = tl.where(
|
||||
offs_k_cache[None, :] == write_pos, x_cur[:, None], x_all_cache
|
||||
)
|
||||
|
||||
# Decayed checkpoint readout sum_n S_0(m,n) q(n), streamed over NF dstate
|
||||
# tiles so the (M, N) state slice is never held whole.
|
||||
offs_nt = tl.arange(0, NF_DSTATE_TILE)
|
||||
ck_acc = tl.zeros([BLOCK_SIZE_M], dtype=tl.float32)
|
||||
for i in tl.static_range(NF_NDS):
|
||||
offs_n = i * NF_DSTATE_TILE + offs_nt
|
||||
nmask = offs_n < dstate
|
||||
st = tl.load(
|
||||
state_ptr
|
||||
+ offs_m[:, None] * stride_state_dim
|
||||
+ offs_n[None, :] * stride_state_dstate,
|
||||
mask=(offs_m[:, None] < dim) & nmask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
c_chunk = tl.load(
|
||||
C_ptr + offs_n * stride_C_dstate, mask=nmask, other=0.0
|
||||
).to(tl.float32)
|
||||
ck_acc += tl.sum(st.to(tl.float32) * c_chunk[None, :], axis=1)
|
||||
checkpoint_out = ck_acc * total_decay_cache
|
||||
bc_cache = tl.load(
|
||||
bc_pre_ptr + offs_k_cache * stride_bc_pre_pos,
|
||||
mask=offs_k_cache <= write_pos,
|
||||
other=0.0,
|
||||
)
|
||||
cache_out = tl.sum(
|
||||
x_all_cache.to(tl.float32) * (scale_cache * bc_cache)[None, :], axis=1
|
||||
)
|
||||
out = checkpoint_out + cache_out
|
||||
|
||||
# Append the current token (x, dt, B) into the buffer at write_pos.
|
||||
tl.store(
|
||||
x_cache_ptr + offs_m * stride_x_cache_dim + write_pos * stride_x_cache_pos,
|
||||
x_cur,
|
||||
mask=offs_m < dim,
|
||||
)
|
||||
if pid_m == 0:
|
||||
B_cur = tl.load(
|
||||
B_ptr + offs_n_full * stride_B_dstate,
|
||||
mask=offs_n_full < dstate,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(dt_cache_ptr + write_pos * stride_dt_cache_pos, dt_cur)
|
||||
tl.store(
|
||||
B_cache_ptr
|
||||
+ write_pos * stride_B_cache_pos
|
||||
+ offs_n_full * stride_B_cache_dstate,
|
||||
B_cur,
|
||||
mask=offs_n_full < dstate,
|
||||
)
|
||||
else:
|
||||
# Flush step: state route. Reconstruct the state from cached inputs,
|
||||
# S_t = total_decay * S_0 + sum_j s_j (v_j k_j^T), persist it as the new
|
||||
# checkpoint, then read y = S_t q -- streamed over FL dstate tiles.
|
||||
offs_k_dot = tl.arange(0, BLOCK_SIZE_K_DOT)
|
||||
dt_all_dot = tl.load(
|
||||
dt_cache_ptr + offs_k_dot * stride_dt_cache_pos,
|
||||
mask=offs_k_dot < write_pos,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
dt_all_dot = tl.where(offs_k_dot == write_pos, dt_cur, dt_all_dot)
|
||||
dA_cumsum_dot = A * tl.cumsum(dt_all_dot, axis=0)
|
||||
dA_total_dot = A * tl.sum(dt_all_dot, axis=0)
|
||||
total_decay_dot = tl.exp(dA_total_dot)
|
||||
scale_dot = dt_all_dot * tl.exp(dA_total_dot - dA_cumsum_dot)
|
||||
scale_dot = tl.where(offs_k_dot <= write_pos, scale_dot, 0.0)
|
||||
|
||||
# Gather buffered x over the window (history + current token).
|
||||
x_all_dot_ptrs = (
|
||||
x_cache_ptr
|
||||
+ offs_m[:, None] * stride_x_cache_dim
|
||||
+ offs_k_dot[None, :] * stride_x_cache_pos
|
||||
)
|
||||
x_all_dot = tl.load(
|
||||
x_all_dot_ptrs,
|
||||
mask=(offs_m[:, None] < dim) & (offs_k_dot[None, :] < write_pos),
|
||||
other=0.0,
|
||||
)
|
||||
x_all_dot = tl.where(
|
||||
offs_k_dot[None, :] == write_pos, x_cur[:, None], x_all_dot
|
||||
)
|
||||
x_all_ty = x_all_dot.to(x_ptr.dtype.element_ty)
|
||||
|
||||
# Distinct tile locals (_f) from the nf branch: differing tile widths
|
||||
# would force a shape-mismatched merge at the if/else exit.
|
||||
offs_nt_f = tl.arange(0, FL_DSTATE_TILE)
|
||||
out = tl.zeros([BLOCK_SIZE_M], dtype=tl.float32)
|
||||
for i in tl.static_range(FL_NDS):
|
||||
offs_n_f = i * FL_DSTATE_TILE + offs_nt_f
|
||||
nmask_f = offs_n_f < dstate
|
||||
# Gather buffered B over the window (history + current token).
|
||||
B_all_dot = tl.load(
|
||||
B_cache_ptr
|
||||
+ offs_k_dot[:, None] * stride_B_cache_pos
|
||||
+ offs_n_f[None, :] * stride_B_cache_dstate,
|
||||
mask=(offs_k_dot[:, None] < write_pos) & nmask_f[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
B_cur_tile = tl.load(
|
||||
B_ptr + offs_n_f * stride_B_dstate, mask=nmask_f, other=0.0
|
||||
)
|
||||
B_all_dot = tl.where(
|
||||
offs_k_dot[:, None] == write_pos, B_cur_tile[None, :], B_all_dot
|
||||
)
|
||||
# tf32x3 keeps fp32 parity with the elementwise baseline (plain tf32 on
|
||||
# fp32 inputs drifts ~1e-2); bf16/fp16 inputs are unaffected by this flag.
|
||||
B_scaled = (B_all_dot.to(tl.float32) * scale_dot[:, None]).to(
|
||||
x_ptr.dtype.element_ty
|
||||
)
|
||||
delta_state = tl.dot(x_all_ty, B_scaled, input_precision="tf32x3")
|
||||
state_ptrs = (
|
||||
state_ptr
|
||||
+ offs_m[:, None] * stride_state_dim
|
||||
+ offs_n_f[None, :] * stride_state_dstate
|
||||
)
|
||||
state_mask = (offs_m[:, None] < dim) & nmask_f[None, :]
|
||||
st_f = tl.load(state_ptrs, mask=state_mask, other=0.0)
|
||||
state_new = st_f.to(tl.float32) * total_decay_dot + delta_state.to(
|
||||
tl.float32
|
||||
)
|
||||
if USE_RS_ROUNDING:
|
||||
# Stochastic-round fp32->fp16 (Blackwell cvt.rs), mirroring the
|
||||
# baseline. Only the flush step stores state, so this runs at 1/L
|
||||
# the baseline's per-step rate. Absolute per-element offsets seed
|
||||
# the RNG so each state element draws independently.
|
||||
rand_seed = tl.load(rand_seed_ptr)
|
||||
rand_offsets = (
|
||||
state_batch_idx * stride_state_batch
|
||||
+ pid_h * stride_state_head
|
||||
+ offs_m[:, None] * stride_state_dim
|
||||
+ offs_n_f[None, :] * stride_state_dstate
|
||||
)
|
||||
if PHILOX_ROUNDS > 0:
|
||||
rand = tl.randint(rand_seed, rand_offsets, PHILOX_ROUNDS)
|
||||
else:
|
||||
rand = tl.randint(rand_seed, rand_offsets)
|
||||
tl.static_assert(
|
||||
state_ptrs.dtype.element_ty == tl.float16,
|
||||
"stochastic rounding requires an fp16 SSM state cache",
|
||||
)
|
||||
state_store = convert_rs_fp16x2(state_new, rand)
|
||||
else:
|
||||
state_store = state_new.to(st_f.dtype)
|
||||
tl.store(state_ptrs, state_store, mask=state_mask)
|
||||
c_chunk_f = tl.load(
|
||||
C_ptr + offs_n_f * stride_C_dstate, mask=nmask_f, other=0.0
|
||||
).to(tl.float32)
|
||||
out += tl.sum(state_new * c_chunk_f[None, :], axis=1)
|
||||
|
||||
# Skip connection (D) and output gate (z).
|
||||
if HAS_D:
|
||||
D_ptr += pid_h * stride_D_head
|
||||
D = tl.load(D_ptr + offs_m * stride_D_dim, mask=offs_m < dim, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
out += x_cur.to(tl.float32) * D
|
||||
if HAS_Z:
|
||||
z_ptr += pid_b * stride_z_batch + pid_h * stride_z_head
|
||||
z = tl.load(z_ptr + offs_m * stride_z_dim, mask=offs_m < dim, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
out *= z * tl.sigmoid(z)
|
||||
|
||||
tl.store(out_ptr + offs_m * stride_out_dim, out, mask=offs_m < dim)
|
||||
|
||||
|
||||
def selective_state_update_replayssm_output_only(
|
||||
state: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
dt: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D: torch.Tensor | None = None,
|
||||
dt_bias: torch.Tensor | None = None,
|
||||
z: torch.Tensor | None = None,
|
||||
dt_softplus: bool = False,
|
||||
x_cache: torch.Tensor | None = None,
|
||||
dt_cache: torch.Tensor | None = None,
|
||||
B_cache: torch.Tensor | None = None,
|
||||
bc_pre: torch.Tensor | None = None,
|
||||
write_pos: torch.Tensor | None = None,
|
||||
is_flush: torch.Tensor | None = None,
|
||||
max_cache_len: int = 16,
|
||||
state_batch_indices: torch.Tensor | None = None,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
out: torch.Tensor | None = None,
|
||||
enable_stochastic_rounding: bool = False,
|
||||
cache_philox_rounds: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Cached-bc SSM update for vLLM's autoregressive Mamba2 decode path."""
|
||||
has_heads = state.dim() > 3
|
||||
if state.dim() == 3:
|
||||
state = state.unsqueeze(1)
|
||||
if x.dim() == 2:
|
||||
x = x.unsqueeze(1)
|
||||
if dt.dim() == 2:
|
||||
dt = dt.unsqueeze(1)
|
||||
if A.dim() == 2:
|
||||
A = A.unsqueeze(0)
|
||||
if B.dim() == 2:
|
||||
B = B.unsqueeze(1)
|
||||
if C.dim() == 2:
|
||||
C = C.unsqueeze(1)
|
||||
if D is not None and D.dim() == 1:
|
||||
D = D.unsqueeze(0)
|
||||
if z is not None and z.dim() == 2:
|
||||
z = z.unsqueeze(1)
|
||||
if dt_bias is not None and dt_bias.dim() == 1:
|
||||
dt_bias = dt_bias.unsqueeze(0)
|
||||
if out is not None and out.dim() == 2:
|
||||
out = out.unsqueeze(1)
|
||||
if state_batch_indices is not None and state_batch_indices.dim() == 1:
|
||||
state_batch_indices = state_batch_indices.unsqueeze(1)
|
||||
|
||||
_, nheads, dim, dstate = state.shape
|
||||
batch = x.shape[0]
|
||||
assert x.shape == (batch, nheads, dim)
|
||||
assert dt.shape == x.shape
|
||||
assert A.shape == (nheads, dim, dstate)
|
||||
ngroups = B.shape[1]
|
||||
assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
|
||||
assert B.shape == (batch, ngroups, dstate)
|
||||
assert C.shape == B.shape
|
||||
if D is not None:
|
||||
assert D.shape == (nheads, dim)
|
||||
if z is not None:
|
||||
assert z.shape == x.shape
|
||||
if dt_bias is not None:
|
||||
assert dt_bias.shape == (nheads, dim)
|
||||
assert out is not None and out.shape == x.shape
|
||||
|
||||
assert A.stride(-1) == 0 and A.stride(-2) == 0, (
|
||||
"Cached kernel requires TIE_HDIM (A scalar per head)"
|
||||
)
|
||||
assert dt.stride(-1) == 0, "Cached kernel requires TIE_HDIM (dt scalar per head)"
|
||||
if dt_bias is not None:
|
||||
assert dt_bias.stride(-1) == 0, (
|
||||
"Cached kernel requires TIE_HDIM (dt_bias scalar per head)"
|
||||
)
|
||||
|
||||
assert x_cache is not None
|
||||
assert dt_cache is not None
|
||||
assert B_cache is not None
|
||||
assert x_cache.shape[1:] == (nheads, max_cache_len, dim)
|
||||
assert dt_cache.shape[1:] == (nheads, max_cache_len)
|
||||
assert B_cache.shape[1:] == (ngroups, max_cache_len, dstate)
|
||||
assert write_pos is not None and write_pos.shape[0] >= batch
|
||||
assert write_pos.dtype == torch.int32
|
||||
assert is_flush is not None and is_flush.shape[0] >= batch
|
||||
assert is_flush.dtype in (torch.bool, torch.int8)
|
||||
assert bc_pre is not None
|
||||
assert bc_pre.shape[0] >= batch and bc_pre.shape[1] >= ngroups
|
||||
assert bc_pre.shape[2] == max_cache_len
|
||||
assert bc_pre.dtype == torch.float32
|
||||
if state_batch_indices is not None:
|
||||
assert state_batch_indices.shape[0] >= batch
|
||||
assert state_batch_indices.shape[1] >= 1
|
||||
|
||||
block_size_k_cache = max(1, triton.next_power_of_2(max_cache_len))
|
||||
block_size_k_dot = max(16, block_size_k_cache)
|
||||
block_size_m, num_warps, nf_tile, fl_tile, num_stages = get_replayssm_config(
|
||||
"mamba2_output_only", dstate=dstate, L=max_cache_len
|
||||
)
|
||||
bs_dstate = triton.next_power_of_2(dstate)
|
||||
nf_dstate_tile = max(16, min(nf_tile, bs_dstate))
|
||||
nf_nds = triton.cdiv(bs_dstate, nf_dstate_tile)
|
||||
fl_dstate_tile = max(16, min(fl_tile, bs_dstate))
|
||||
fl_nds = triton.cdiv(bs_dstate, fl_dstate_tile)
|
||||
|
||||
grid = lambda META: (triton.cdiv(dim, META["BLOCK_SIZE_M"]), batch, nheads)
|
||||
z_strides = (z.stride(0), z.stride(1), z.stride(2)) if z is not None else (0, 0, 0)
|
||||
state_indices_strides = (
|
||||
(state_batch_indices.stride(0), state_batch_indices.stride(1))
|
||||
if state_batch_indices is not None
|
||||
else (0, 0)
|
||||
)
|
||||
rand_seed = (
|
||||
torch.randint(0, 2**32, (1,), device=state.device)
|
||||
if enable_stochastic_rounding
|
||||
else None
|
||||
)
|
||||
|
||||
with torch.accelerator.device_index(x.device.index):
|
||||
# Both kernels always launch: the precompute kernel self-skips flush
|
||||
# rows per row (the branch can't be hoisted out under CUDA graphs), so
|
||||
# it is a no-op when every row is flushing.
|
||||
_replayssm_output_only_precompute_kernel[(batch, ngroups)](
|
||||
B,
|
||||
C,
|
||||
B_cache,
|
||||
write_pos,
|
||||
is_flush,
|
||||
bc_pre,
|
||||
state_batch_indices,
|
||||
null_block_id,
|
||||
batch,
|
||||
ngroups,
|
||||
dstate,
|
||||
B.stride(0),
|
||||
B.stride(1),
|
||||
B.stride(2),
|
||||
C.stride(0),
|
||||
C.stride(1),
|
||||
C.stride(2),
|
||||
B_cache.stride(0),
|
||||
B_cache.stride(1),
|
||||
B_cache.stride(2),
|
||||
B_cache.stride(3),
|
||||
bc_pre.stride(0),
|
||||
bc_pre.stride(1),
|
||||
bc_pre.stride(2),
|
||||
state_indices_strides[0],
|
||||
state_indices_strides[1],
|
||||
max_cache_len,
|
||||
block_size_k_cache,
|
||||
num_warps=2,
|
||||
)
|
||||
_replayssm_output_only_kernel[grid](
|
||||
state,
|
||||
rand_seed,
|
||||
x,
|
||||
dt,
|
||||
dt_bias,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D,
|
||||
z,
|
||||
out,
|
||||
x_cache,
|
||||
dt_cache,
|
||||
B_cache,
|
||||
bc_pre,
|
||||
write_pos,
|
||||
is_flush,
|
||||
state_batch_indices,
|
||||
null_block_id,
|
||||
batch,
|
||||
nheads,
|
||||
dim,
|
||||
dstate,
|
||||
nheads // ngroups,
|
||||
state.stride(0),
|
||||
state.stride(1),
|
||||
state.stride(2),
|
||||
state.stride(3),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x.stride(2),
|
||||
dt.stride(0),
|
||||
dt.stride(1),
|
||||
dt_bias.stride(0) if dt_bias is not None else 0,
|
||||
A.stride(0),
|
||||
B.stride(0),
|
||||
B.stride(1),
|
||||
B.stride(2),
|
||||
C.stride(0),
|
||||
C.stride(1),
|
||||
C.stride(2),
|
||||
D.stride(0) if D is not None else 0,
|
||||
D.stride(1) if D is not None else 0,
|
||||
z_strides[0],
|
||||
z_strides[1],
|
||||
z_strides[2],
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
x_cache.stride(0),
|
||||
x_cache.stride(1),
|
||||
x_cache.stride(3),
|
||||
x_cache.stride(2),
|
||||
dt_cache.stride(0),
|
||||
dt_cache.stride(1),
|
||||
dt_cache.stride(2),
|
||||
B_cache.stride(0),
|
||||
B_cache.stride(1),
|
||||
B_cache.stride(2),
|
||||
B_cache.stride(3),
|
||||
bc_pre.stride(0),
|
||||
bc_pre.stride(1),
|
||||
bc_pre.stride(2),
|
||||
state_indices_strides[0],
|
||||
state_indices_strides[1],
|
||||
dt_softplus,
|
||||
max_cache_len,
|
||||
block_size_m,
|
||||
block_size_k_cache,
|
||||
block_size_k_dot,
|
||||
nf_dstate_tile,
|
||||
nf_nds,
|
||||
fl_dstate_tile,
|
||||
fl_nds,
|
||||
enable_stochastic_rounding,
|
||||
cache_philox_rounds,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
|
||||
if not has_heads:
|
||||
out = out.squeeze(1)
|
||||
return out
|
||||
@@ -984,6 +984,31 @@ def supports_mamba_prefix_caching(
|
||||
return getattr(model, "supports_mamba_prefix_caching", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsReplaySSM(Protocol):
|
||||
"""The interface for models whose Mamba2 layers support ReplaySSM cached
|
||||
standard decode.
|
||||
|
||||
This is currently experimental.
|
||||
"""
|
||||
|
||||
supports_replayssm: ClassVar[Literal[True]] = True
|
||||
|
||||
|
||||
@overload
|
||||
def supports_replayssm(model: object) -> TypeIs[SupportsReplaySSM]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_replayssm(model: type[object]) -> TypeIs[type[SupportsReplaySSM]]: ...
|
||||
|
||||
|
||||
def supports_replayssm(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsReplaySSM]] | TypeIs[SupportsReplaySSM]:
|
||||
return getattr(model, "supports_replayssm", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsCrossEncoding(Protocol):
|
||||
"""The interface required for all models that support cross encoding."""
|
||||
|
||||
@@ -69,6 +69,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
SupportsReplaySSM,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -707,6 +708,7 @@ class NemotronHForCausalLM(
|
||||
SupportsQuant,
|
||||
MixtureOfExperts,
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsReplaySSM,
|
||||
):
|
||||
# Relevant only if self.has_moe is True
|
||||
is_non_gated_moe: bool = True
|
||||
@@ -742,18 +744,24 @@ class NemotronHForCausalLM(
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
return MambaStateDtypeCalculator.mamba2_state_dtype(
|
||||
) -> tuple[torch.dtype, ...]:
|
||||
cache_config = vllm_config.cache_config
|
||||
base_dtype = MambaStateDtypeCalculator.mamba2_state_dtype(
|
||||
vllm_config.model_config.dtype,
|
||||
vllm_config.cache_config.mamba_cache_dtype,
|
||||
vllm_config.cache_config.mamba_ssm_cache_dtype,
|
||||
cache_config.mamba_cache_dtype,
|
||||
cache_config.mamba_ssm_cache_dtype,
|
||||
)
|
||||
if cache_config.use_replayssm:
|
||||
return MambaStateDtypeCalculator.append_replayssm_ring(
|
||||
base_dtype, vllm_config.model_config.dtype
|
||||
)
|
||||
return base_dtype
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> tuple[tuple[int, int], tuple[int, int, int]]:
|
||||
) -> tuple[tuple[int, ...], ...]:
|
||||
"""Calculate shapes for Mamba's convolutional and state caches.
|
||||
|
||||
Args:
|
||||
@@ -763,12 +771,15 @@ class NemotronHForCausalLM(
|
||||
Tuple containing:
|
||||
- conv_state_shape: Shape for convolutional state cache
|
||||
- temporal_state_shape: Shape for state space model cache
|
||||
- (when use_replayssm is enabled) the x_cache/dt_cache/B_cache
|
||||
ring-buffer shapes
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
cache_config = vllm_config.cache_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
intermediate_size = hf_config.mamba_num_heads * hf_config.mamba_head_dim
|
||||
|
||||
return MambaStateShapeCalculator.mamba2_state_shape(
|
||||
base_shape = MambaStateShapeCalculator.mamba2_state_shape(
|
||||
intermediate_size=intermediate_size,
|
||||
tp_world_size=parallel_config.tensor_parallel_size,
|
||||
n_groups=hf_config.n_groups,
|
||||
@@ -778,6 +789,14 @@ class NemotronHForCausalLM(
|
||||
conv_kernel=hf_config.conv_kernel,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
if cache_config.use_replayssm:
|
||||
return MambaStateShapeCalculator.append_replayssm_ring(
|
||||
base_shape,
|
||||
hf_config.n_groups,
|
||||
parallel_config.tensor_parallel_size,
|
||||
cache_config.replayssm_buffer_len,
|
||||
)
|
||||
return base_shape
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
|
||||
|
||||
@@ -55,6 +55,7 @@ from .interfaces import (
|
||||
supports_multimodal_encoder_tp_data,
|
||||
supports_multimodal_raw_input_only,
|
||||
supports_pp,
|
||||
supports_replayssm,
|
||||
supports_transcription,
|
||||
)
|
||||
from .interfaces_base import (
|
||||
@@ -789,6 +790,7 @@ class _ModelInfo:
|
||||
is_hybrid: bool
|
||||
has_noops: bool
|
||||
supports_mamba_prefix_caching: bool
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
|
||||
@@ -815,6 +817,7 @@ class _ModelInfo:
|
||||
is_attention_free=is_attention_free(model),
|
||||
is_hybrid=is_hybrid(model),
|
||||
supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
|
||||
supports_replayssm=supports_replayssm(model),
|
||||
supports_transcription=supports_transcription(model),
|
||||
supports_transcription_only=(
|
||||
supports_transcription(model) and model.supports_transcription_only
|
||||
|
||||
Reference in New Issue
Block a user