[Mamba] Share RecoverSSM commit kernels and helpers across models

RecoverSSM's commit-plan and conv-window compaction kernels are model independent.
Move them from models/kimi_k3 into model_executor/layers/mamba/ops/recoverssm.py
(Kimi-K3 keeps its private names as import aliases; kernel source unchanged), add
mamba/recoverssm_utils.py (activation predicate, argument validation) and the
generic spec-row/request mapping helpers to v1/attention/backends/recoverssm_metadata.py.

CacheConfig.use_kda_recoverssm becomes use_recoverssm: the switch now selects RecoverSSM
for any model whose recurrent layers implement it, not only KDA.

No functional change for Kimi-K3.

Signed-off-by: Jürgen Schmied <juergenschmied70@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Jürgen Schmied
2026-09-27 00:07:04 +02:00
co-authored by Claude
parent 77871126f9
commit a80e41d690
15 changed files with 285 additions and 192 deletions
+1 -1
View File
@@ -78,7 +78,7 @@ class MambaBase(AttentionLayerBase):
# never writes the baseline's per-draft-token state slots.
num_speculative_blocks=(
0
if vllm_config.cache_config.use_kda_recoverssm
if vllm_config.cache_config.use_recoverssm
else vllm_config.num_speculative_tokens
),
)
@@ -0,0 +1,196 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-independent RecoverSSM Triton kernels shared by every recurrent-state
implementation.
RecoverSSM verifies a speculative window from a read-only checkpoint and, after
sampling, commits the accepted prefix once. The per-model parts (the KDA and GDN verify
kernels and their state reconstruction) live with their models; the kernels they share
are here:
- ``prepare_commit_plan_kernel``: per request, the commit length and the destination
state blocks (final and, in align mode, the block-boundary block) from the accepted
counts and the block table;
- ``compact_conv_state_kernel``: moves the accepted suffix of an extended short-conv
window to the front of the destination block (and writes the boundary window in align
mode)."""
from vllm.triton_utils import tl, triton
@triton.heuristics(
{
"HAS_REQUEST_INDICES": lambda args: args["request_indices_ptr"] is not None,
"ALIGN_MODE": lambda args: args["block_table_ptr"] is not None,
}
)
@triton.jit
def prepare_commit_plan_kernel(
num_accepted_ptr,
request_indices_ptr,
state_indices_ptr,
query_start_loc_ptr,
block_table_ptr,
num_computed_ptr,
commit_lens_ptr,
final_state_indices_ptr,
boundary_state_indices_ptr,
boundary_recovery_lens_ptr,
null_block_id,
mamba_block_size,
block_table_width,
stride_num_accepted,
stride_request_indices,
stride_state_indices,
stride_query_start_loc,
stride_block_table_row,
stride_block_table_col,
stride_num_computed,
SPEC_QUERY_LEN: tl.constexpr,
HAS_REQUEST_INDICES: tl.constexpr,
ALIGN_MODE: tl.constexpr,
):
spec_idx = tl.program_id(0)
source_state_idx = tl.load(state_indices_ptr + spec_idx * stride_state_indices).to(
tl.int64
)
request_idx = spec_idx
if HAS_REQUEST_INDICES:
request_idx = tl.load(
request_indices_ptr + spec_idx * stride_request_indices
).to(tl.int64)
num_accepted = tl.load(num_accepted_ptr + request_idx * stride_num_accepted).to(
tl.int32
)
bos = tl.load(query_start_loc_ptr + spec_idx * stride_query_start_loc).to(tl.int64)
eos = tl.load(query_start_loc_ptr + (spec_idx + 1) * stride_query_start_loc).to(
tl.int64
)
query_len = (eos - bos).to(tl.int32)
commit_len = tl.minimum(tl.maximum(num_accepted, 0), query_len)
commit_len = tl.minimum(commit_len, SPEC_QUERY_LEN)
final_state_idx = source_state_idx
boundary_state_idx = null_block_id
boundary_recovery_len = 0
if ALIGN_MODE:
num_computed = tl.load(num_computed_ptr + request_idx * stride_num_computed).to(
tl.int32
)
final_num_computed = num_computed + commit_len
final_state_col = tl.minimum(
final_num_computed // mamba_block_size, block_table_width - 1
)
final_state_idx = tl.load(
block_table_ptr
+ request_idx * stride_block_table_row
+ final_state_col * stride_block_table_col
).to(tl.int64)
next_boundary = (num_computed // mamba_block_size + 1) * mamba_block_size
crosses_boundary = final_num_computed >= next_boundary
boundary_recovery_len = next_boundary - num_computed
boundary_state_idx = tl.load(
block_table_ptr
+ request_idx * stride_block_table_row
+ (next_boundary // mamba_block_size - 1) * stride_block_table_col,
mask=crosses_boundary,
other=null_block_id,
).to(tl.int64)
valid = (source_state_idx > null_block_id) & (commit_len > 0)
tl.store(commit_lens_ptr + spec_idx, tl.where(valid, commit_len, 0))
tl.store(
final_state_indices_ptr + spec_idx,
tl.where(valid, final_state_idx, null_block_id),
)
tl.store(
boundary_state_indices_ptr + spec_idx,
tl.where(valid, boundary_state_idx, null_block_id),
)
tl.store(
boundary_recovery_lens_ptr + spec_idx,
tl.where(valid, boundary_recovery_len, 0),
)
@triton.jit
def compact_conv_state_kernel(
conv_state_ref_ptr,
conv_state_base_addrs_ptr,
conv_state_block_strides_ptr,
conv_state_dim_strides_ptr,
conv_state_token_strides_ptr,
state_indices_ptr,
commit_lens_ptr,
final_state_indices_ptr,
boundary_state_indices_ptr,
boundary_recovery_lens_ptr,
null_block_id,
conv_dim,
conv_history_len,
stride_state_indices,
BLOCK_D: tl.constexpr,
BLOCK_HISTORY: tl.constexpr,
ALIGN_MODE: tl.constexpr,
):
pid_d = tl.program_id(0)
pid_b = tl.program_id(1)
pid_l = tl.program_id(2)
source_state_idx = tl.load(state_indices_ptr + pid_b * stride_state_indices).to(
tl.int64
)
if source_state_idx <= null_block_id:
return
commit_len = tl.load(commit_lens_ptr + pid_b)
if commit_len == 0:
return
final_state_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
boundary_state_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
boundary_recovery_len = tl.load(boundary_recovery_lens_ptr + pid_b)
if final_state_idx <= null_block_id:
return
base_addr = tl.load(conv_state_base_addrs_ptr + pid_l)
block_stride = tl.load(conv_state_block_strides_ptr + pid_l)
dim_stride = tl.load(conv_state_dim_strides_ptr + pid_l)
token_stride = tl.load(conv_state_token_strides_ptr + pid_l)
conv_state_ptr = base_addr.to(tl.pointer_type(conv_state_ref_ptr.dtype.element_ty))
source_ptr = conv_state_ptr + source_state_idx * block_stride
final_ptr = conv_state_ptr + final_state_idx * block_stride
offs_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
offs_h = tl.arange(0, BLOCK_HISTORY)
mask = (offs_d[:, None] < conv_dim) & (offs_h[None, :] < conv_history_len)
final_values = tl.load(
source_ptr
+ offs_d[:, None] * dim_stride
+ (commit_len - 1 + offs_h[None, :]) * token_stride,
mask=mask,
)
if ALIGN_MODE:
boundary_values = tl.load(
source_ptr
+ offs_d[:, None] * dim_stride
+ (boundary_recovery_len - 1 + offs_h[None, :]) * token_stride,
mask=mask & (boundary_state_idx > null_block_id),
)
boundary_ptr = conv_state_ptr + boundary_state_idx * block_stride
tl.store(
boundary_ptr
+ offs_d[:, None] * dim_stride
+ offs_h[None, :] * token_stride,
boundary_values,
mask=mask & (boundary_state_idx > null_block_id),
)
tl.store(
final_ptr + offs_d[:, None] * dim_stride + offs_h[None, :] * token_stride,
final_values,
mask=mask,
)
__all__ = [
"compact_conv_state_kernel",
"prepare_commit_plan_kernel",
]
@@ -0,0 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-independent RecoverSSM Python helpers: the activation predicate shared by
layers, models and backends, and argument validation that raises ``ValueError``
before any raw-pointer kernel launch."""
def recoverssm_require(cond: bool, msg: str, component: str = "RecoverSSM") -> None:
if not cond:
raise ValueError(f"{component}: {msg}")
def uses_recoverssm(cache_config, num_speculative_tokens: int) -> bool:
"""True when this model instance runs RecoverSSM for its recurrent states.
VllmConfig sets ``cache_config.use_recoverssm`` for every model whose recurrent
states support it (Kimi-K3 KDA, Qwen GDN/PLE). Without speculative tokens there is
nothing to recover, so the stock path runs."""
return (
bool(getattr(cache_config, "use_recoverssm", False))
and num_speculative_tokens > 0
)
__all__ = ["recoverssm_require", "uses_recoverssm"]