A speculative window whose accepted tokens end exactly on a recurrent block boundary selected the following block-table column for the committed state (commit plan) and recorded it as the running column (postprocess). That column can still be unallocated, so the next step reads an uninitialized page. Select the block that holds the last committed token, matching the (n - 1) // block convention used elsewhere in align mode. Applies to Kimi-K3 KDA and to the Qwen GDN/PLE RecoverSSM paths that share the plan. Fix and test from local-inference-lab/vllm a02f585d (Martin Vit, root cause reported by astra); the GDN/PLE align tests now expect the same convention. Signed-off-by: Jürgen Schmied <juergenschmied70@gmail.com> Co-authored-by: Martin Vit <martin@voipmonitor.org> Co-authored-by: Claude <noreply@anthropic.com>
201 lines
7.0 KiB
Python
201 lines
7.0 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Model-independent RecoverSSM Triton kernels shared by every recurrent-state
|
|
implementation.
|
|
|
|
RecoverSSM verifies a speculative window from a read-only checkpoint and, after
|
|
sampling, commits the accepted prefix once. The per-model parts (the KDA and GDN verify
|
|
kernels and their state reconstruction) live with their models; the kernels they share
|
|
are here:
|
|
|
|
- ``prepare_commit_plan_kernel``: per request, the commit length and the destination
|
|
state blocks (final and, in align mode, the block-boundary block) from the accepted
|
|
counts and the block table;
|
|
- ``compact_conv_state_kernel``: moves the accepted suffix of an extended short-conv
|
|
window to the front of the destination block (and writes the boundary window in align
|
|
mode)."""
|
|
|
|
from vllm.triton_utils import tl, triton
|
|
|
|
|
|
@triton.heuristics(
|
|
{
|
|
"HAS_REQUEST_INDICES": lambda args: args["request_indices_ptr"] is not None,
|
|
"ALIGN_MODE": lambda args: args["block_table_ptr"] is not None,
|
|
}
|
|
)
|
|
@triton.jit
|
|
def prepare_commit_plan_kernel(
|
|
num_accepted_ptr,
|
|
request_indices_ptr,
|
|
state_indices_ptr,
|
|
query_start_loc_ptr,
|
|
block_table_ptr,
|
|
num_computed_ptr,
|
|
commit_lens_ptr,
|
|
final_state_indices_ptr,
|
|
boundary_state_indices_ptr,
|
|
boundary_recovery_lens_ptr,
|
|
null_block_id,
|
|
mamba_block_size,
|
|
block_table_width,
|
|
stride_num_accepted,
|
|
stride_request_indices,
|
|
stride_state_indices,
|
|
stride_query_start_loc,
|
|
stride_block_table_row,
|
|
stride_block_table_col,
|
|
stride_num_computed,
|
|
SPEC_QUERY_LEN: tl.constexpr,
|
|
HAS_REQUEST_INDICES: tl.constexpr,
|
|
ALIGN_MODE: tl.constexpr,
|
|
):
|
|
spec_idx = tl.program_id(0)
|
|
source_state_idx = tl.load(state_indices_ptr + spec_idx * stride_state_indices).to(
|
|
tl.int64
|
|
)
|
|
request_idx = spec_idx
|
|
if HAS_REQUEST_INDICES:
|
|
request_idx = tl.load(
|
|
request_indices_ptr + spec_idx * stride_request_indices
|
|
).to(tl.int64)
|
|
num_accepted = tl.load(num_accepted_ptr + request_idx * stride_num_accepted).to(
|
|
tl.int32
|
|
)
|
|
bos = tl.load(query_start_loc_ptr + spec_idx * stride_query_start_loc).to(tl.int64)
|
|
eos = tl.load(query_start_loc_ptr + (spec_idx + 1) * stride_query_start_loc).to(
|
|
tl.int64
|
|
)
|
|
query_len = (eos - bos).to(tl.int32)
|
|
commit_len = tl.minimum(tl.maximum(num_accepted, 0), query_len)
|
|
commit_len = tl.minimum(commit_len, SPEC_QUERY_LEN)
|
|
|
|
final_state_idx = source_state_idx
|
|
boundary_state_idx = null_block_id
|
|
boundary_recovery_len = 0
|
|
if ALIGN_MODE:
|
|
num_computed = tl.load(num_computed_ptr + request_idx * stride_num_computed).to(
|
|
tl.int32
|
|
)
|
|
final_num_computed = num_computed + commit_len
|
|
# The committed state belongs to the block holding the last committed
|
|
# token. A window ending exactly on a boundary stays in that block; the
|
|
# next block may still be unallocated.
|
|
final_state_col = tl.minimum(
|
|
tl.maximum(final_num_computed - 1, 0) // mamba_block_size,
|
|
block_table_width - 1,
|
|
)
|
|
final_state_idx = tl.load(
|
|
block_table_ptr
|
|
+ request_idx * stride_block_table_row
|
|
+ final_state_col * stride_block_table_col
|
|
).to(tl.int64)
|
|
next_boundary = (num_computed // mamba_block_size + 1) * mamba_block_size
|
|
crosses_boundary = final_num_computed >= next_boundary
|
|
boundary_recovery_len = next_boundary - num_computed
|
|
boundary_state_idx = tl.load(
|
|
block_table_ptr
|
|
+ request_idx * stride_block_table_row
|
|
+ (next_boundary // mamba_block_size - 1) * stride_block_table_col,
|
|
mask=crosses_boundary,
|
|
other=null_block_id,
|
|
).to(tl.int64)
|
|
valid = (source_state_idx > null_block_id) & (commit_len > 0)
|
|
tl.store(commit_lens_ptr + spec_idx, tl.where(valid, commit_len, 0))
|
|
tl.store(
|
|
final_state_indices_ptr + spec_idx,
|
|
tl.where(valid, final_state_idx, null_block_id),
|
|
)
|
|
tl.store(
|
|
boundary_state_indices_ptr + spec_idx,
|
|
tl.where(valid, boundary_state_idx, null_block_id),
|
|
)
|
|
tl.store(
|
|
boundary_recovery_lens_ptr + spec_idx,
|
|
tl.where(valid, boundary_recovery_len, 0),
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def compact_conv_state_kernel(
|
|
conv_state_ref_ptr,
|
|
conv_state_base_addrs_ptr,
|
|
conv_state_block_strides_ptr,
|
|
conv_state_dim_strides_ptr,
|
|
conv_state_token_strides_ptr,
|
|
state_indices_ptr,
|
|
commit_lens_ptr,
|
|
final_state_indices_ptr,
|
|
boundary_state_indices_ptr,
|
|
boundary_recovery_lens_ptr,
|
|
null_block_id,
|
|
conv_dim,
|
|
conv_history_len,
|
|
stride_state_indices,
|
|
BLOCK_D: tl.constexpr,
|
|
BLOCK_HISTORY: tl.constexpr,
|
|
ALIGN_MODE: tl.constexpr,
|
|
):
|
|
pid_d = tl.program_id(0)
|
|
pid_b = tl.program_id(1)
|
|
pid_l = tl.program_id(2)
|
|
source_state_idx = tl.load(state_indices_ptr + pid_b * stride_state_indices).to(
|
|
tl.int64
|
|
)
|
|
if source_state_idx <= null_block_id:
|
|
return
|
|
|
|
commit_len = tl.load(commit_lens_ptr + pid_b)
|
|
if commit_len == 0:
|
|
return
|
|
final_state_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
|
boundary_state_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
|
boundary_recovery_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
|
|
|
if final_state_idx <= null_block_id:
|
|
return
|
|
|
|
base_addr = tl.load(conv_state_base_addrs_ptr + pid_l)
|
|
block_stride = tl.load(conv_state_block_strides_ptr + pid_l)
|
|
dim_stride = tl.load(conv_state_dim_strides_ptr + pid_l)
|
|
token_stride = tl.load(conv_state_token_strides_ptr + pid_l)
|
|
conv_state_ptr = base_addr.to(tl.pointer_type(conv_state_ref_ptr.dtype.element_ty))
|
|
source_ptr = conv_state_ptr + source_state_idx * block_stride
|
|
final_ptr = conv_state_ptr + final_state_idx * block_stride
|
|
|
|
offs_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
|
|
offs_h = tl.arange(0, BLOCK_HISTORY)
|
|
mask = (offs_d[:, None] < conv_dim) & (offs_h[None, :] < conv_history_len)
|
|
final_values = tl.load(
|
|
source_ptr
|
|
+ offs_d[:, None] * dim_stride
|
|
+ (commit_len - 1 + offs_h[None, :]) * token_stride,
|
|
mask=mask,
|
|
)
|
|
if ALIGN_MODE:
|
|
boundary_values = tl.load(
|
|
source_ptr
|
|
+ offs_d[:, None] * dim_stride
|
|
+ (boundary_recovery_len - 1 + offs_h[None, :]) * token_stride,
|
|
mask=mask & (boundary_state_idx > null_block_id),
|
|
)
|
|
boundary_ptr = conv_state_ptr + boundary_state_idx * block_stride
|
|
tl.store(
|
|
boundary_ptr
|
|
+ offs_d[:, None] * dim_stride
|
|
+ offs_h[None, :] * token_stride,
|
|
boundary_values,
|
|
mask=mask & (boundary_state_idx > null_block_id),
|
|
)
|
|
tl.store(
|
|
final_ptr + offs_d[:, None] * dim_stride + offs_h[None, :] * token_stride,
|
|
final_values,
|
|
mask=mask,
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"compact_conv_state_kernel",
|
|
"prepare_commit_plan_kernel",
|
|
]
|