Files
5567cc1b25 [Mamba] RecoverSSM align: keep a boundary-ending state in its block
A speculative window whose accepted tokens end exactly on a recurrent block
boundary selected the following block-table column for the committed state
(commit plan) and recorded it as the running column (postprocess). That
column can still be unallocated, so the next step reads an uninitialized
page. Select the block that holds the last committed token, matching the
(n - 1) // block convention used elsewhere in align mode. Applies to
Kimi-K3 KDA and to the Qwen GDN/PLE RecoverSSM paths that share the plan.

Fix and test from local-inference-lab/vllm a02f585d (Martin Vit, root cause
reported by astra); the GDN/PLE align tests now expect the same convention.

Signed-off-by: Jürgen Schmied <juergenschmied70@gmail.com>
Co-authored-by: Martin Vit <martin@voipmonitor.org>
Co-authored-by: Claude <noreply@anthropic.com>
2026-09-27 11:12:40 +02:00

201 lines
7.0 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-independent RecoverSSM Triton kernels shared by every recurrent-state
implementation.
RecoverSSM verifies a speculative window from a read-only checkpoint and, after
sampling, commits the accepted prefix once. The per-model parts (the KDA and GDN verify
kernels and their state reconstruction) live with their models; the kernels they share
are here:
- ``prepare_commit_plan_kernel``: per request, the commit length and the destination
state blocks (final and, in align mode, the block-boundary block) from the accepted
counts and the block table;
- ``compact_conv_state_kernel``: moves the accepted suffix of an extended short-conv
window to the front of the destination block (and writes the boundary window in align
mode)."""
from vllm.triton_utils import tl, triton
@triton.heuristics(
{
"HAS_REQUEST_INDICES": lambda args: args["request_indices_ptr"] is not None,
"ALIGN_MODE": lambda args: args["block_table_ptr"] is not None,
}
)
@triton.jit
def prepare_commit_plan_kernel(
num_accepted_ptr,
request_indices_ptr,
state_indices_ptr,
query_start_loc_ptr,
block_table_ptr,
num_computed_ptr,
commit_lens_ptr,
final_state_indices_ptr,
boundary_state_indices_ptr,
boundary_recovery_lens_ptr,
null_block_id,
mamba_block_size,
block_table_width,
stride_num_accepted,
stride_request_indices,
stride_state_indices,
stride_query_start_loc,
stride_block_table_row,
stride_block_table_col,
stride_num_computed,
SPEC_QUERY_LEN: tl.constexpr,
HAS_REQUEST_INDICES: tl.constexpr,
ALIGN_MODE: tl.constexpr,
):
spec_idx = tl.program_id(0)
source_state_idx = tl.load(state_indices_ptr + spec_idx * stride_state_indices).to(
tl.int64
)
request_idx = spec_idx
if HAS_REQUEST_INDICES:
request_idx = tl.load(
request_indices_ptr + spec_idx * stride_request_indices
).to(tl.int64)
num_accepted = tl.load(num_accepted_ptr + request_idx * stride_num_accepted).to(
tl.int32
)
bos = tl.load(query_start_loc_ptr + spec_idx * stride_query_start_loc).to(tl.int64)
eos = tl.load(query_start_loc_ptr + (spec_idx + 1) * stride_query_start_loc).to(
tl.int64
)
query_len = (eos - bos).to(tl.int32)
commit_len = tl.minimum(tl.maximum(num_accepted, 0), query_len)
commit_len = tl.minimum(commit_len, SPEC_QUERY_LEN)
final_state_idx = source_state_idx
boundary_state_idx = null_block_id
boundary_recovery_len = 0
if ALIGN_MODE:
num_computed = tl.load(num_computed_ptr + request_idx * stride_num_computed).to(
tl.int32
)
final_num_computed = num_computed + commit_len
# The committed state belongs to the block holding the last committed
# token. A window ending exactly on a boundary stays in that block; the
# next block may still be unallocated.
final_state_col = tl.minimum(
tl.maximum(final_num_computed - 1, 0) // mamba_block_size,
block_table_width - 1,
)
final_state_idx = tl.load(
block_table_ptr
+ request_idx * stride_block_table_row
+ final_state_col * stride_block_table_col
).to(tl.int64)
next_boundary = (num_computed // mamba_block_size + 1) * mamba_block_size
crosses_boundary = final_num_computed >= next_boundary
boundary_recovery_len = next_boundary - num_computed
boundary_state_idx = tl.load(
block_table_ptr
+ request_idx * stride_block_table_row
+ (next_boundary // mamba_block_size - 1) * stride_block_table_col,
mask=crosses_boundary,
other=null_block_id,
).to(tl.int64)
valid = (source_state_idx > null_block_id) & (commit_len > 0)
tl.store(commit_lens_ptr + spec_idx, tl.where(valid, commit_len, 0))
tl.store(
final_state_indices_ptr + spec_idx,
tl.where(valid, final_state_idx, null_block_id),
)
tl.store(
boundary_state_indices_ptr + spec_idx,
tl.where(valid, boundary_state_idx, null_block_id),
)
tl.store(
boundary_recovery_lens_ptr + spec_idx,
tl.where(valid, boundary_recovery_len, 0),
)
@triton.jit
def compact_conv_state_kernel(
conv_state_ref_ptr,
conv_state_base_addrs_ptr,
conv_state_block_strides_ptr,
conv_state_dim_strides_ptr,
conv_state_token_strides_ptr,
state_indices_ptr,
commit_lens_ptr,
final_state_indices_ptr,
boundary_state_indices_ptr,
boundary_recovery_lens_ptr,
null_block_id,
conv_dim,
conv_history_len,
stride_state_indices,
BLOCK_D: tl.constexpr,
BLOCK_HISTORY: tl.constexpr,
ALIGN_MODE: tl.constexpr,
):
pid_d = tl.program_id(0)
pid_b = tl.program_id(1)
pid_l = tl.program_id(2)
source_state_idx = tl.load(state_indices_ptr + pid_b * stride_state_indices).to(
tl.int64
)
if source_state_idx <= null_block_id:
return
commit_len = tl.load(commit_lens_ptr + pid_b)
if commit_len == 0:
return
final_state_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
boundary_state_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
boundary_recovery_len = tl.load(boundary_recovery_lens_ptr + pid_b)
if final_state_idx <= null_block_id:
return
base_addr = tl.load(conv_state_base_addrs_ptr + pid_l)
block_stride = tl.load(conv_state_block_strides_ptr + pid_l)
dim_stride = tl.load(conv_state_dim_strides_ptr + pid_l)
token_stride = tl.load(conv_state_token_strides_ptr + pid_l)
conv_state_ptr = base_addr.to(tl.pointer_type(conv_state_ref_ptr.dtype.element_ty))
source_ptr = conv_state_ptr + source_state_idx * block_stride
final_ptr = conv_state_ptr + final_state_idx * block_stride
offs_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
offs_h = tl.arange(0, BLOCK_HISTORY)
mask = (offs_d[:, None] < conv_dim) & (offs_h[None, :] < conv_history_len)
final_values = tl.load(
source_ptr
+ offs_d[:, None] * dim_stride
+ (commit_len - 1 + offs_h[None, :]) * token_stride,
mask=mask,
)
if ALIGN_MODE:
boundary_values = tl.load(
source_ptr
+ offs_d[:, None] * dim_stride
+ (boundary_recovery_len - 1 + offs_h[None, :]) * token_stride,
mask=mask & (boundary_state_idx > null_block_id),
)
boundary_ptr = conv_state_ptr + boundary_state_idx * block_stride
tl.store(
boundary_ptr
+ offs_d[:, None] * dim_stride
+ offs_h[None, :] * token_stride,
boundary_values,
mask=mask & (boundary_state_idx > null_block_id),
)
tl.store(
final_ptr + offs_d[:, None] * dim_stride + offs_h[None, :] * token_stride,
final_values,
mask=mask,
)
__all__ = [
"compact_conv_state_kernel",
"prepare_commit_plan_kernel",
]