[Mamba] Share RecoverSSM commit kernels and helpers across models
RecoverSSM's commit-plan and conv-window compaction kernels are model independent. Move them from models/kimi_k3 into model_executor/layers/mamba/ops/recoverssm.py (Kimi-K3 keeps its private names as import aliases; kernel source unchanged), add mamba/recoverssm_utils.py (activation predicate, argument validation) and the generic spec-row/request mapping helpers to v1/attention/backends/recoverssm_metadata.py. CacheConfig.use_kda_recoverssm becomes use_recoverssm: the switch now selects RecoverSSM for any model whose recurrent layers implement it, not only KDA. No functional change for Kimi-K3. Signed-off-by: Jürgen Schmied <juergenschmied70@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -78,7 +78,7 @@ class MambaBase(AttentionLayerBase):
|
||||
# never writes the baseline's per-draft-token state slots.
|
||||
num_speculative_blocks=(
|
||||
0
|
||||
if vllm_config.cache_config.use_kda_recoverssm
|
||||
if vllm_config.cache_config.use_recoverssm
|
||||
else vllm_config.num_speculative_tokens
|
||||
),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-independent RecoverSSM Triton kernels shared by every recurrent-state
|
||||
implementation.
|
||||
|
||||
RecoverSSM verifies a speculative window from a read-only checkpoint and, after
|
||||
sampling, commits the accepted prefix once. The per-model parts (the KDA and GDN verify
|
||||
kernels and their state reconstruction) live with their models; the kernels they share
|
||||
are here:
|
||||
|
||||
- ``prepare_commit_plan_kernel``: per request, the commit length and the destination
|
||||
state blocks (final and, in align mode, the block-boundary block) from the accepted
|
||||
counts and the block table;
|
||||
- ``compact_conv_state_kernel``: moves the accepted suffix of an extended short-conv
|
||||
window to the front of the destination block (and writes the boundary window in align
|
||||
mode)."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_REQUEST_INDICES": lambda args: args["request_indices_ptr"] is not None,
|
||||
"ALIGN_MODE": lambda args: args["block_table_ptr"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit
|
||||
def prepare_commit_plan_kernel(
|
||||
num_accepted_ptr,
|
||||
request_indices_ptr,
|
||||
state_indices_ptr,
|
||||
query_start_loc_ptr,
|
||||
block_table_ptr,
|
||||
num_computed_ptr,
|
||||
commit_lens_ptr,
|
||||
final_state_indices_ptr,
|
||||
boundary_state_indices_ptr,
|
||||
boundary_recovery_lens_ptr,
|
||||
null_block_id,
|
||||
mamba_block_size,
|
||||
block_table_width,
|
||||
stride_num_accepted,
|
||||
stride_request_indices,
|
||||
stride_state_indices,
|
||||
stride_query_start_loc,
|
||||
stride_block_table_row,
|
||||
stride_block_table_col,
|
||||
stride_num_computed,
|
||||
SPEC_QUERY_LEN: tl.constexpr,
|
||||
HAS_REQUEST_INDICES: tl.constexpr,
|
||||
ALIGN_MODE: tl.constexpr,
|
||||
):
|
||||
spec_idx = tl.program_id(0)
|
||||
source_state_idx = tl.load(state_indices_ptr + spec_idx * stride_state_indices).to(
|
||||
tl.int64
|
||||
)
|
||||
request_idx = spec_idx
|
||||
if HAS_REQUEST_INDICES:
|
||||
request_idx = tl.load(
|
||||
request_indices_ptr + spec_idx * stride_request_indices
|
||||
).to(tl.int64)
|
||||
num_accepted = tl.load(num_accepted_ptr + request_idx * stride_num_accepted).to(
|
||||
tl.int32
|
||||
)
|
||||
bos = tl.load(query_start_loc_ptr + spec_idx * stride_query_start_loc).to(tl.int64)
|
||||
eos = tl.load(query_start_loc_ptr + (spec_idx + 1) * stride_query_start_loc).to(
|
||||
tl.int64
|
||||
)
|
||||
query_len = (eos - bos).to(tl.int32)
|
||||
commit_len = tl.minimum(tl.maximum(num_accepted, 0), query_len)
|
||||
commit_len = tl.minimum(commit_len, SPEC_QUERY_LEN)
|
||||
|
||||
final_state_idx = source_state_idx
|
||||
boundary_state_idx = null_block_id
|
||||
boundary_recovery_len = 0
|
||||
if ALIGN_MODE:
|
||||
num_computed = tl.load(num_computed_ptr + request_idx * stride_num_computed).to(
|
||||
tl.int32
|
||||
)
|
||||
final_num_computed = num_computed + commit_len
|
||||
final_state_col = tl.minimum(
|
||||
final_num_computed // mamba_block_size, block_table_width - 1
|
||||
)
|
||||
final_state_idx = tl.load(
|
||||
block_table_ptr
|
||||
+ request_idx * stride_block_table_row
|
||||
+ final_state_col * stride_block_table_col
|
||||
).to(tl.int64)
|
||||
next_boundary = (num_computed // mamba_block_size + 1) * mamba_block_size
|
||||
crosses_boundary = final_num_computed >= next_boundary
|
||||
boundary_recovery_len = next_boundary - num_computed
|
||||
boundary_state_idx = tl.load(
|
||||
block_table_ptr
|
||||
+ request_idx * stride_block_table_row
|
||||
+ (next_boundary // mamba_block_size - 1) * stride_block_table_col,
|
||||
mask=crosses_boundary,
|
||||
other=null_block_id,
|
||||
).to(tl.int64)
|
||||
valid = (source_state_idx > null_block_id) & (commit_len > 0)
|
||||
tl.store(commit_lens_ptr + spec_idx, tl.where(valid, commit_len, 0))
|
||||
tl.store(
|
||||
final_state_indices_ptr + spec_idx,
|
||||
tl.where(valid, final_state_idx, null_block_id),
|
||||
)
|
||||
tl.store(
|
||||
boundary_state_indices_ptr + spec_idx,
|
||||
tl.where(valid, boundary_state_idx, null_block_id),
|
||||
)
|
||||
tl.store(
|
||||
boundary_recovery_lens_ptr + spec_idx,
|
||||
tl.where(valid, boundary_recovery_len, 0),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def compact_conv_state_kernel(
|
||||
conv_state_ref_ptr,
|
||||
conv_state_base_addrs_ptr,
|
||||
conv_state_block_strides_ptr,
|
||||
conv_state_dim_strides_ptr,
|
||||
conv_state_token_strides_ptr,
|
||||
state_indices_ptr,
|
||||
commit_lens_ptr,
|
||||
final_state_indices_ptr,
|
||||
boundary_state_indices_ptr,
|
||||
boundary_recovery_lens_ptr,
|
||||
null_block_id,
|
||||
conv_dim,
|
||||
conv_history_len,
|
||||
stride_state_indices,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_HISTORY: tl.constexpr,
|
||||
ALIGN_MODE: tl.constexpr,
|
||||
):
|
||||
pid_d = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_l = tl.program_id(2)
|
||||
source_state_idx = tl.load(state_indices_ptr + pid_b * stride_state_indices).to(
|
||||
tl.int64
|
||||
)
|
||||
if source_state_idx <= null_block_id:
|
||||
return
|
||||
|
||||
commit_len = tl.load(commit_lens_ptr + pid_b)
|
||||
if commit_len == 0:
|
||||
return
|
||||
final_state_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_state_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_recovery_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||
|
||||
if final_state_idx <= null_block_id:
|
||||
return
|
||||
|
||||
base_addr = tl.load(conv_state_base_addrs_ptr + pid_l)
|
||||
block_stride = tl.load(conv_state_block_strides_ptr + pid_l)
|
||||
dim_stride = tl.load(conv_state_dim_strides_ptr + pid_l)
|
||||
token_stride = tl.load(conv_state_token_strides_ptr + pid_l)
|
||||
conv_state_ptr = base_addr.to(tl.pointer_type(conv_state_ref_ptr.dtype.element_ty))
|
||||
source_ptr = conv_state_ptr + source_state_idx * block_stride
|
||||
final_ptr = conv_state_ptr + final_state_idx * block_stride
|
||||
|
||||
offs_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
|
||||
offs_h = tl.arange(0, BLOCK_HISTORY)
|
||||
mask = (offs_d[:, None] < conv_dim) & (offs_h[None, :] < conv_history_len)
|
||||
final_values = tl.load(
|
||||
source_ptr
|
||||
+ offs_d[:, None] * dim_stride
|
||||
+ (commit_len - 1 + offs_h[None, :]) * token_stride,
|
||||
mask=mask,
|
||||
)
|
||||
if ALIGN_MODE:
|
||||
boundary_values = tl.load(
|
||||
source_ptr
|
||||
+ offs_d[:, None] * dim_stride
|
||||
+ (boundary_recovery_len - 1 + offs_h[None, :]) * token_stride,
|
||||
mask=mask & (boundary_state_idx > null_block_id),
|
||||
)
|
||||
boundary_ptr = conv_state_ptr + boundary_state_idx * block_stride
|
||||
tl.store(
|
||||
boundary_ptr
|
||||
+ offs_d[:, None] * dim_stride
|
||||
+ offs_h[None, :] * token_stride,
|
||||
boundary_values,
|
||||
mask=mask & (boundary_state_idx > null_block_id),
|
||||
)
|
||||
tl.store(
|
||||
final_ptr + offs_d[:, None] * dim_stride + offs_h[None, :] * token_stride,
|
||||
final_values,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"compact_conv_state_kernel",
|
||||
"prepare_commit_plan_kernel",
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-independent RecoverSSM Python helpers: the activation predicate shared by
|
||||
layers, models and backends, and argument validation that raises ``ValueError``
|
||||
before any raw-pointer kernel launch."""
|
||||
|
||||
|
||||
def recoverssm_require(cond: bool, msg: str, component: str = "RecoverSSM") -> None:
|
||||
if not cond:
|
||||
raise ValueError(f"{component}: {msg}")
|
||||
|
||||
|
||||
def uses_recoverssm(cache_config, num_speculative_tokens: int) -> bool:
|
||||
"""True when this model instance runs RecoverSSM for its recurrent states.
|
||||
|
||||
VllmConfig sets ``cache_config.use_recoverssm`` for every model whose recurrent
|
||||
states support it (Kimi-K3 KDA, Qwen GDN/PLE). Without speculative tokens there is
|
||||
nothing to recover, so the stock path runs."""
|
||||
return (
|
||||
bool(getattr(cache_config, "use_recoverssm", False))
|
||||
and num_speculative_tokens > 0
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["recoverssm_require", "uses_recoverssm"]
|
||||
Reference in New Issue
Block a user