[Spec Decode][MRV2] RecoverSSM for Qwen GDN and PLE short conv (PR #58863)
Intake of open upstream PR vllm-project/vllm#58863. Active only with VLLM_USE_V2_MODEL_RUNNER=1; inert otherwise. Speculative decoding on Gated DeltaNet snapshotted the full recurrent state after every draft position and reserved one Mamba block per draft position. On a 24 GB card that is reserved VRAM we never use, plus per-step copy traffic. RecoverSSM, the protocol Kimi-K3 KDA already uses, replaces this: verify runs from a read-only checkpoint and stores a small per-token replay record [correction (V), normalized key (K), gate (1)] in fp32; commit replays only the accepted tokens and writes the recurrent state once, plus the block-boundary state in align mode. The PLE dilated short conv compacts its accepted window with the same plan. Conflict in vllm/models/qwen4_exp/nvidia/mtp.py: our get_top_tokens() work put LocalArgmaxMixin on Qwen4ExpMTP, the PR adds SupportsReplaySSM to the same base list. Kept both. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -78,7 +78,7 @@ class MambaBase(AttentionLayerBase):
|
||||
# never writes the baseline's per-draft-token state slots.
|
||||
num_speculative_blocks=(
|
||||
0
|
||||
if vllm_config.cache_config.use_kda_recoverssm
|
||||
if vllm_config.cache_config.use_recoverssm
|
||||
else vllm_config.num_speculative_tokens
|
||||
),
|
||||
)
|
||||
|
||||
@@ -370,10 +370,15 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
|
||||
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
||||
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
def _uses_gdn_recoverssm(self) -> bool:
|
||||
"""One predicate for the replay state's shape, dtype and the attention
|
||||
backend."""
|
||||
from vllm.model_executor.layers.mamba.recoverssm_utils import uses_recoverssm
|
||||
|
||||
return uses_recoverssm(self.cache_config, self.num_spec)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
|
||||
shapes = MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
self.tp_size,
|
||||
self.num_k_heads,
|
||||
self.num_v_heads,
|
||||
@@ -382,6 +387,32 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.conv_kernel_size,
|
||||
self.num_spec,
|
||||
)
|
||||
if self._uses_gdn_recoverssm():
|
||||
# per-token replay record [HV, spec_query_len, V + K + 1] fp32
|
||||
shapes = (
|
||||
*shapes,
|
||||
(
|
||||
self.num_v_heads // self.tp_size,
|
||||
self.num_spec + 1,
|
||||
self.head_v_dim + self.head_k_dim + 1,
|
||||
),
|
||||
)
|
||||
return shapes
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
dtypes = super().get_state_dtype()
|
||||
if self._uses_gdn_recoverssm():
|
||||
dtypes = (*dtypes, torch.float32)
|
||||
return dtypes
|
||||
|
||||
def get_attn_backend(self):
|
||||
if self._uses_gdn_recoverssm():
|
||||
from vllm.v1.attention.backends.gdn_recoverssm import (
|
||||
GDNRecoverSSMAttentionBackend,
|
||||
)
|
||||
|
||||
return GDNRecoverSSMAttentionBackend
|
||||
return super().get_attn_backend()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -530,6 +561,17 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.gdn_decode_kernel = "XPU"
|
||||
|
||||
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
|
||||
# RecoverSSM verify replaces the per-draft snapshot path (Triton decode
|
||||
# only)
|
||||
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
|
||||
if self.use_gdn_recoverssm:
|
||||
self.gdn_decode_kernel = "triton"
|
||||
self.enable_fused_gdn_decode = False
|
||||
self.enable_packed_recurrent_decode = False
|
||||
logger.info_once(
|
||||
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
|
||||
self.num_spec + 1,
|
||||
)
|
||||
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
@@ -540,7 +582,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
def _fused_gdn_decode_unsupported_reason(
|
||||
self, vllm_config: VllmConfig
|
||||
) -> str | None:
|
||||
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
|
||||
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2]
|
||||
if (
|
||||
self.gqa_interleaved_layout
|
||||
or self.head_k_dim != 128
|
||||
@@ -800,6 +842,25 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
|
||||
return mixed_qkv_out, z_out, b_out, a_out
|
||||
|
||||
def _recoverssm_verify_qkv(self, mixed_qkv):
|
||||
"""Q/K/V for the RecoverSSM verify.
|
||||
|
||||
The verify kernel reads token-strided heads, so for a 2-D [tokens, q|k|v] conv
|
||||
output with a contiguous last dimension it gets views instead of copies. Any
|
||||
other layout falls back to the contiguous copies of rearrange_mixed_qkv().
|
||||
Returns (query, key, value, is_view)."""
|
||||
if mixed_qkv.ndim != 2 or mixed_qkv.stride(-1) != 1:
|
||||
return (*self.rearrange_mixed_qkv(mixed_qkv), False)
|
||||
q_dim = self.key_dim // self.tp_size
|
||||
v_dim = self.value_dim // self.tp_size
|
||||
num_tokens = mixed_qkv.shape[0]
|
||||
query = mixed_qkv[:, :q_dim].view(1, num_tokens, -1, self.head_k_dim)
|
||||
key = mixed_qkv[:, q_dim : 2 * q_dim].view(1, num_tokens, -1, self.head_k_dim)
|
||||
value = mixed_qkv[:, 2 * q_dim : 2 * q_dim + v_dim].view(
|
||||
1, num_tokens, -1, self.head_v_dim
|
||||
)
|
||||
return query, key, value, True
|
||||
|
||||
def rearrange_mixed_qkv(self, mixed_qkv):
|
||||
"""Split packed qkv into contiguous (1, seq, heads, dim) tensors.
|
||||
|
||||
@@ -1085,7 +1146,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
dtype = qkv_or_qkvz.dtype
|
||||
num_k_heads = self.num_k_heads // self.tp_size
|
||||
num_v_heads = self.num_v_heads // self.tp_size
|
||||
_, state_dtype = self.get_state_dtype()
|
||||
state_dtype = self.get_state_dtype()[1] # 3 dtypes with the replay record
|
||||
|
||||
# All kernels use BT = chunk_size, so a single pass with T = chunk_size
|
||||
# is sufficient to populate every autotuner cache. Mirror the real
|
||||
@@ -1343,7 +1404,11 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
],
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_state_indices_tensor.size(-1),
|
||||
max_query_len=(
|
||||
self.num_spec + 1
|
||||
if getattr(self, "use_gdn_recoverssm", False)
|
||||
else spec_state_indices_tensor.size(-1)
|
||||
),
|
||||
validate_data=False,
|
||||
)
|
||||
|
||||
@@ -1380,7 +1445,17 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
else:
|
||||
mixed_qkv_non_spec = None
|
||||
|
||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||
if (
|
||||
spec_sequence_masks is not None
|
||||
and mixed_qkv_spec is not None
|
||||
and self.use_gdn_recoverssm
|
||||
):
|
||||
query_spec, key_spec, value_spec, verify_views = (
|
||||
self._recoverssm_verify_qkv(mixed_qkv_spec)
|
||||
)
|
||||
else:
|
||||
verify_views = False
|
||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||
|
||||
# Split mixed non-spec-decode+prefill to process independently
|
||||
split_non_spec = (
|
||||
@@ -1443,7 +1518,43 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
# 2. Recurrent attention
|
||||
|
||||
# 2.1: Process the multi-query part
|
||||
if spec_sequence_masks is not None:
|
||||
if spec_sequence_masks is not None and getattr(
|
||||
self, "use_gdn_recoverssm", False
|
||||
):
|
||||
# verify the window off the checkpoint; the commit after sampling
|
||||
# writes the state once
|
||||
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
|
||||
gdn_recoverssm_verify,
|
||||
)
|
||||
|
||||
_n = attn_metadata.num_spec_decodes
|
||||
assert spec_query_start_loc is not None
|
||||
assert spec_state_indices_tensor is not None
|
||||
core_attn_out_spec = gdn_recoverssm_verify(
|
||||
self.A_log,
|
||||
a_spec,
|
||||
b_spec,
|
||||
self.dt_bias,
|
||||
query_spec,
|
||||
key_spec,
|
||||
value_spec,
|
||||
checkpoint_state=ssm_state,
|
||||
replay_cache=self_kv_cache[2],
|
||||
query_start_loc=spec_query_start_loc[: _n + 1],
|
||||
state_indices=spec_state_indices_tensor[:_n, 0],
|
||||
spec_query_len=self.num_spec + 1,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
# spec-only batch: write straight into the layer output
|
||||
out=(
|
||||
core_attn_out[:num_actual_tokens].unsqueeze(0)
|
||||
if verify_views
|
||||
and mixed_qkv_non_spec is None
|
||||
and query_spec.shape[1] == num_actual_tokens
|
||||
else None
|
||||
),
|
||||
)
|
||||
last_recurrent_state = None
|
||||
elif spec_sequence_masks is not None:
|
||||
core_attn_out_spec, last_recurrent_state = (
|
||||
fused_sigmoid_gating_delta_rule_update(
|
||||
A_log=self.A_log,
|
||||
@@ -1558,7 +1669,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
0, non_spec_token_indx, core_attn_out_non_spec.squeeze(0)
|
||||
)
|
||||
elif spec_sequence_masks is not None:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||
if core_attn_out_spec.data_ptr() != core_attn_out.data_ptr():
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||
else:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
|
||||
|
||||
|
||||
@@ -0,0 +1,695 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RecoverSSM for Gated DeltaNet (Qwen-style GDN): speculative verify from one
|
||||
checkpoint without per-draft state snapshots, and accepted-state recovery after
|
||||
sampling.
|
||||
|
||||
GDN port of the Kimi-K3 KDA RecoverSSM path
|
||||
(vllm/models/kimi_k3/nvidia/ops/recoverssm.py). Differences from KDA: the gate is a
|
||||
scalar per value head (g = -exp(A_log) * softplus(a + dt_bias)), q/k heads are grouped
|
||||
over value heads (HV // H), and q/k are L2-normalised inside the kernel when requested.
|
||||
The verify step caches, per token, the delta-rule correction c_t = beta_t * (v_t - S'_t
|
||||
k_t) (V floats), the normalised key (K floats) and the log-decay g_t (1 float) in one
|
||||
fp32 "replay" record. Recovery folds the accepted prefix: S_n = exp(sum_t g_t) S_0 +
|
||||
sum_t exp(sum_{s>t} g_s) c_t k_t^T and writes the checkpoint once (plus the
|
||||
block-boundary state in align mode)."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.recoverssm import (
|
||||
compact_conv_state_kernel,
|
||||
prepare_commit_plan_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.recoverssm_utils import recoverssm_require
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def _require(cond: bool, msg: str) -> None:
|
||||
recoverssm_require(cond, msg, "GDN RecoverSSM")
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gdn_recoverssm_verify_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
A_log_ptr,
|
||||
dt_bias_ptr,
|
||||
state_ptr,
|
||||
replay_ptr,
|
||||
out_ptr,
|
||||
query_start_loc_ptr,
|
||||
state_indices_ptr,
|
||||
scale,
|
||||
softplus_beta,
|
||||
softplus_threshold,
|
||||
null_block_id,
|
||||
stride_q_token,
|
||||
stride_k_token,
|
||||
stride_v_token,
|
||||
stride_a_token,
|
||||
stride_b_token,
|
||||
stride_state_block,
|
||||
stride_state_head,
|
||||
stride_state_v,
|
||||
stride_state_k,
|
||||
stride_replay_block,
|
||||
stride_replay_head,
|
||||
stride_replay_pos,
|
||||
stride_replay_dim,
|
||||
stride_out_token,
|
||||
stride_qsl,
|
||||
stride_si,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
SPEC_QUERY_LEN: tl.constexpr,
|
||||
USE_QK_L2NORM: tl.constexpr,
|
||||
):
|
||||
pid_v = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_hv = tl.program_id(2)
|
||||
pid_h = pid_hv // (HV // H)
|
||||
bos = tl.load(query_start_loc_ptr + pid_b * stride_qsl).to(tl.int64)
|
||||
eos = tl.load(query_start_loc_ptr + (pid_b + 1) * stride_qsl).to(tl.int64)
|
||||
query_len = eos - bos
|
||||
state_idx = tl.load(state_indices_ptr + pid_b * stride_si).to(tl.int64)
|
||||
offs_k = tl.arange(0, BK)
|
||||
offs_v = pid_v * BV + tl.arange(0, BV)
|
||||
mask_k = offs_k < K
|
||||
mask_v = offs_v < V
|
||||
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||
if state_idx <= null_block_id:
|
||||
for t in tl.static_range(SPEC_QUERY_LEN):
|
||||
tl.store(
|
||||
out_ptr + (bos + t) * stride_out_token + pid_hv * V + offs_v,
|
||||
tl.zeros([BV], dtype=tl.float32),
|
||||
mask=(t < query_len) & mask_v,
|
||||
)
|
||||
return
|
||||
state = tl.load(
|
||||
state_ptr
|
||||
+ state_idx * stride_state_block
|
||||
+ pid_hv * stride_state_head
|
||||
+ offs_v[:, None] * stride_state_v
|
||||
+ offs_k[None, :] * stride_state_k,
|
||||
mask=mask_state,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
neg_A = -tl.exp(tl.load(A_log_ptr + pid_hv).to(tl.float32))
|
||||
dt_bias = tl.load(dt_bias_ptr + pid_hv).to(tl.float32)
|
||||
rec_base = (
|
||||
replay_ptr + state_idx * stride_replay_block + pid_hv * stride_replay_head
|
||||
)
|
||||
for t in tl.static_range(SPEC_QUERY_LEN):
|
||||
valid = t < query_len
|
||||
tok = bos + t
|
||||
q = tl.load(
|
||||
q_ptr + tok * stride_q_token + pid_h * K + offs_k,
|
||||
mask=valid & mask_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
k = tl.load(
|
||||
k_ptr + tok * stride_k_token + pid_h * K + offs_k,
|
||||
mask=valid & mask_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
v = tl.load(
|
||||
v_ptr + tok * stride_v_token + pid_hv * V + offs_v,
|
||||
mask=valid & mask_v,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
x = (
|
||||
tl.load(a_ptr + tok * stride_a_token + pid_hv, mask=valid, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
+ dt_bias
|
||||
)
|
||||
sp = tl.where(
|
||||
softplus_beta * x <= softplus_threshold,
|
||||
(1 / softplus_beta) * tl.log(1 + tl.exp(softplus_beta * x)),
|
||||
x,
|
||||
)
|
||||
g = neg_A * sp
|
||||
beta = tl.sigmoid(
|
||||
tl.load(b_ptr + tok * stride_b_token + pid_hv, mask=valid, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
)
|
||||
if USE_QK_L2NORM:
|
||||
q = q * tl.rsqrt(tl.sum(q * q) + 1e-6)
|
||||
k = k * tl.rsqrt(tl.sum(k * k) + 1e-6)
|
||||
q = q * scale
|
||||
new_state = state * tl.exp(g)
|
||||
corr = (v - tl.sum(new_state * k[None, :], 1)) * beta
|
||||
new_state = new_state + corr[:, None] * k[None, :]
|
||||
state = tl.where(valid, new_state, state)
|
||||
o = tl.sum(state * q[None, :], 1)
|
||||
tl.store(
|
||||
out_ptr + tok * stride_out_token + pid_hv * V + offs_v,
|
||||
o.to(out_ptr.dtype.element_ty),
|
||||
mask=valid & mask_v,
|
||||
)
|
||||
rec = rec_base + t * stride_replay_pos
|
||||
tl.store(rec + offs_v * stride_replay_dim, corr, mask=valid & mask_v)
|
||||
if pid_v == 0:
|
||||
tl.store(rec + (V + offs_k) * stride_replay_dim, k, mask=valid & mask_k)
|
||||
tl.store(rec + (V + K) * stride_replay_dim, g, mask=valid)
|
||||
|
||||
|
||||
def gdn_recoverssm_verify(
|
||||
A_log: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
*,
|
||||
checkpoint_state: torch.Tensor,
|
||||
replay_cache: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
spec_query_len: int,
|
||||
scale: float | None = None,
|
||||
softplus_beta: float = 1.0,
|
||||
softplus_threshold: float = 20.0,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Verify a GDN speculative window against its checkpoint without modifying the
|
||||
checkpoint. q, k: [1, T, H, K]; v: [1, T, HV, V]; a, b: [T, HV] (or [1, T, HV]);
|
||||
checkpoint_state: [blocks, HV, V, K]; replay_cache: [blocks, HV, spec_query_len, V +
|
||||
K + 1] fp32. Returns out [1, T, HV, V]."""
|
||||
_require(
|
||||
q.ndim == 4 and q.shape[0] == 1, "q must have shape [1, tokens, heads, dim]"
|
||||
)
|
||||
_, total, H, K = q.shape
|
||||
_require(k.shape == q.shape, "q and k shapes differ")
|
||||
_require(
|
||||
v.ndim == 4 and tuple(v.shape[:2]) == (1, total),
|
||||
"v must have shape [1, tokens, value heads, dim]",
|
||||
)
|
||||
HV, V = v.shape[-2], v.shape[-1]
|
||||
_require(
|
||||
H > 0 and HV >= H and HV % H == 0,
|
||||
"value heads must be a positive multiple of the q/k heads",
|
||||
)
|
||||
_require(
|
||||
a.numel() == total * HV and b.numel() == total * HV,
|
||||
"gate or beta shape is incompatible",
|
||||
)
|
||||
a2 = a.reshape(total, HV)
|
||||
b2 = b.reshape(total, HV)
|
||||
_require(
|
||||
a2.stride(1) == 1 and b2.stride(1) == 1,
|
||||
"gate and beta heads must be contiguous",
|
||||
)
|
||||
_require(
|
||||
tuple(A_log.shape) == (HV,) and tuple(dt_bias.shape) == (HV,),
|
||||
"A_log or dt_bias shape is incompatible",
|
||||
)
|
||||
_require(
|
||||
all(t_.stride(-1) == 1 and t_.stride(-2) == K for t_ in (q, k)),
|
||||
"q and k heads must be contiguous",
|
||||
)
|
||||
_require(v.stride(-1) == 1 and v.stride(-2) == V, "v heads must be contiguous")
|
||||
_require(checkpoint_state.ndim == 4, "checkpoint must be four-dimensional")
|
||||
nb = checkpoint_state.shape[0]
|
||||
_require(
|
||||
tuple(checkpoint_state.shape[1:]) == (HV, V, K),
|
||||
"checkpoint shape is incompatible",
|
||||
)
|
||||
_rec = (nb, HV, spec_query_len, V + K + 1)
|
||||
_require(tuple(replay_cache.shape) == _rec, f"replay buffer needs shape {_rec}")
|
||||
_require(replay_cache.dtype == torch.float32, "replay buffer must use float32")
|
||||
_require(state_indices.ndim == 1, "state indices must be one-dimensional")
|
||||
_require(
|
||||
query_start_loc.ndim == 1
|
||||
and query_start_loc.shape[0] == state_indices.shape[0] + 1,
|
||||
"query metadata is incompatible",
|
||||
)
|
||||
_require(
|
||||
total <= state_indices.shape[0] * spec_query_len,
|
||||
"speculative decode input exceeds its activation capacity",
|
||||
)
|
||||
_dev = q.device
|
||||
_require(
|
||||
all(
|
||||
t_.device == _dev
|
||||
for t_ in (
|
||||
k,
|
||||
v,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
checkpoint_state,
|
||||
replay_cache,
|
||||
query_start_loc,
|
||||
state_indices,
|
||||
)
|
||||
if t_ is not None
|
||||
)
|
||||
and (out is None or out.device == _dev),
|
||||
"inputs must be on the same device",
|
||||
)
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
if out is None:
|
||||
out = torch.empty(1, total, HV, V, dtype=v.dtype, device=v.device)
|
||||
_require(
|
||||
tuple(out.shape) == (1, total, HV, V) and out.stride()[2:] == (V, 1),
|
||||
"output shape or layout is incompatible",
|
||||
)
|
||||
batch = state_indices.shape[0]
|
||||
if total == 0 or batch == 0:
|
||||
return out
|
||||
BK = triton.next_power_of_2(K)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
grid = (triton.cdiv(V, BV), batch, HV)
|
||||
_gdn_recoverssm_verify_kernel[grid](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
a2,
|
||||
b2,
|
||||
A_log,
|
||||
dt_bias,
|
||||
checkpoint_state,
|
||||
replay_cache,
|
||||
out,
|
||||
query_start_loc,
|
||||
state_indices,
|
||||
scale,
|
||||
softplus_beta,
|
||||
softplus_threshold,
|
||||
NULL_BLOCK_ID,
|
||||
q.stride(1),
|
||||
k.stride(1),
|
||||
v.stride(1),
|
||||
a2.stride(0),
|
||||
b2.stride(0),
|
||||
checkpoint_state.stride(0),
|
||||
checkpoint_state.stride(1),
|
||||
checkpoint_state.stride(2),
|
||||
checkpoint_state.stride(3),
|
||||
replay_cache.stride(0),
|
||||
replay_cache.stride(1),
|
||||
replay_cache.stride(2),
|
||||
replay_cache.stride(3),
|
||||
out.stride(1),
|
||||
query_start_loc.stride(0),
|
||||
state_indices.stride(0),
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
SPEC_QUERY_LEN=spec_query_len,
|
||||
USE_QK_L2NORM=use_qk_l2norm_in_kernel,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _commit_gdn_state_kernel(
|
||||
state_ref_ptr,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
replay_ref_ptr,
|
||||
replay_base_addrs_ptr,
|
||||
replay_block_strides_ptr,
|
||||
state_indices_ptr,
|
||||
commit_lens_ptr,
|
||||
final_state_indices_ptr,
|
||||
boundary_state_indices_ptr,
|
||||
boundary_recovery_lens_ptr,
|
||||
null_block_id,
|
||||
stride_state_head,
|
||||
stride_state_v,
|
||||
stride_state_k,
|
||||
stride_replay_head,
|
||||
stride_replay_pos,
|
||||
stride_replay_dim,
|
||||
stride_si,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
ALIGN_MODE: tl.constexpr,
|
||||
):
|
||||
pid_v = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_lh = tl.program_id(2)
|
||||
pid_l = pid_lh // NUM_HEADS
|
||||
pid_h = pid_lh % NUM_HEADS
|
||||
src = tl.load(state_indices_ptr + pid_b * stride_si).to(tl.int64)
|
||||
if src <= null_block_id:
|
||||
return
|
||||
n = tl.load(commit_lens_ptr + pid_b)
|
||||
if n == 0:
|
||||
return
|
||||
final_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||
if final_idx <= null_block_id:
|
||||
return
|
||||
state_ptr = tl.load(state_base_addrs_ptr + pid_l).to(
|
||||
tl.pointer_type(state_ref_ptr.dtype.element_ty)
|
||||
)
|
||||
sbs = tl.load(state_block_strides_ptr + pid_l)
|
||||
replay_ptr = tl.load(replay_base_addrs_ptr + pid_l).to(
|
||||
tl.pointer_type(replay_ref_ptr.dtype.element_ty)
|
||||
)
|
||||
rbs = tl.load(replay_block_strides_ptr + pid_l)
|
||||
rec_base = replay_ptr + src * rbs + pid_h * stride_replay_head
|
||||
offs_k = tl.arange(0, BK)
|
||||
offs_v = pid_v * BV + tl.arange(0, BV)
|
||||
mask_k = offs_k < K
|
||||
mask_v = offs_v < V
|
||||
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||
sp = offs_v[:, None] * stride_state_v + offs_k[None, :] * stride_state_k
|
||||
s0 = tl.load(
|
||||
state_ptr + src * sbs + pid_h * stride_state_head + sp,
|
||||
mask=mask_state,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
fin_decay = 1.0
|
||||
fin_corr = tl.zeros([BV, BK], tl.float32)
|
||||
bnd_decay = 1.0
|
||||
bnd_corr = tl.zeros([BV, BK], tl.float32)
|
||||
for r in range(n):
|
||||
t = n - r - 1
|
||||
rec = rec_base + t * stride_replay_pos
|
||||
c = tl.load(rec + offs_v * stride_replay_dim, mask=mask_v, other=0.0)
|
||||
kk = tl.load(rec + (V + offs_k) * stride_replay_dim, mask=mask_k, other=0.0)
|
||||
g = tl.load(rec + (V + K) * stride_replay_dim)
|
||||
upd = c[:, None] * kk[None, :]
|
||||
dec = tl.exp(g)
|
||||
fin_corr += upd * fin_decay
|
||||
fin_decay *= dec
|
||||
if ALIGN_MODE:
|
||||
before = t < boundary_len
|
||||
bnd_corr += tl.where(before, upd * bnd_decay, 0.0)
|
||||
bnd_decay *= tl.where(before, dec, 1.0)
|
||||
if ALIGN_MODE:
|
||||
bs = s0 * bnd_decay + bnd_corr
|
||||
tl.store(
|
||||
state_ptr + boundary_idx * sbs + pid_h * stride_state_head + sp,
|
||||
bs.to(state_ref_ptr.dtype.element_ty),
|
||||
mask=mask_state & (boundary_idx > null_block_id),
|
||||
)
|
||||
fs = s0 * fin_decay + fin_corr
|
||||
tl.store(
|
||||
state_ptr + final_idx * sbs + pid_h * stride_state_head + sp,
|
||||
fs.to(state_ref_ptr.dtype.element_ty),
|
||||
mask=mask_state,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GDNRecoverSSMCommitContext:
|
||||
conv_states: tuple[torch.Tensor, ...]
|
||||
conv_state_base_addrs: torch.Tensor
|
||||
conv_state_block_strides: torch.Tensor
|
||||
conv_state_dim_strides: torch.Tensor
|
||||
conv_state_token_strides: torch.Tensor
|
||||
conv_history_len: int
|
||||
checkpoints: tuple[torch.Tensor, ...]
|
||||
state_base_addrs: torch.Tensor
|
||||
state_block_strides: torch.Tensor
|
||||
replays: tuple[torch.Tensor, ...]
|
||||
replay_base_addrs: torch.Tensor
|
||||
replay_block_strides: torch.Tensor
|
||||
commit_lens: torch.Tensor
|
||||
final_state_indices: torch.Tensor
|
||||
boundary_state_indices: torch.Tensor
|
||||
boundary_recovery_lens: torch.Tensor
|
||||
spec_query_len: int
|
||||
|
||||
@classmethod
|
||||
def from_tensors(
|
||||
cls,
|
||||
conv_states: Sequence[torch.Tensor],
|
||||
checkpoints: Sequence[torch.Tensor],
|
||||
replays: Sequence[torch.Tensor],
|
||||
*,
|
||||
spec_query_len: int,
|
||||
max_num_reqs: int,
|
||||
conv_dim_first: bool = True,
|
||||
) -> "GDNRecoverSSMCommitContext":
|
||||
_require(len(checkpoints) > 0, "commit requires at least one layer")
|
||||
_require(
|
||||
len(conv_states) == len(checkpoints) == len(replays),
|
||||
"conv, state and replay lists differ",
|
||||
)
|
||||
if not conv_dim_first:
|
||||
conv_states = [s.transpose(-1, -2) for s in conv_states]
|
||||
ref = checkpoints[0]
|
||||
_require(ref.ndim == 4, "checkpoint must be four-dimensional")
|
||||
nb, HV, V, K = ref.shape
|
||||
_dev = ref.device
|
||||
for s in checkpoints:
|
||||
_require(
|
||||
s.shape == ref.shape
|
||||
and s.dtype == ref.dtype
|
||||
and s.stride()[1:] == ref.stride()[1:]
|
||||
and s.device == _dev,
|
||||
"layers need matching checkpoints",
|
||||
)
|
||||
_rec = (nb, HV, spec_query_len, V + K + 1)
|
||||
for r in replays:
|
||||
_require(
|
||||
tuple(r.shape) == _rec
|
||||
and r.dtype == torch.float32
|
||||
and r.stride()[1:] == replays[0].stride()[1:]
|
||||
and r.device == _dev,
|
||||
f"layers need matching float32 replay buffers of shape {_rec}",
|
||||
)
|
||||
conv_ref = conv_states[0]
|
||||
_require(
|
||||
conv_ref.ndim == 3 and conv_ref.shape[0] == nb,
|
||||
"conv state must be [blocks, dim, window] with the checkpoint's "
|
||||
"block count",
|
||||
)
|
||||
for c in conv_states:
|
||||
_require(
|
||||
c.shape == conv_ref.shape
|
||||
and c.dtype == conv_ref.dtype
|
||||
and c.stride()[1:] == conv_ref.stride()[1:]
|
||||
and c.device == _dev,
|
||||
"layers need matching conv states",
|
||||
)
|
||||
conv_dim, conv_len = conv_ref.shape[1:]
|
||||
hist = conv_len - spec_query_len + 1
|
||||
_require(hist > 0, "conv state is shorter than its window")
|
||||
dev = ref.device
|
||||
addr = lambda ts: torch.tensor(
|
||||
[t.data_ptr() for t in ts], dtype=torch.int64, device=dev
|
||||
)
|
||||
bstr = lambda ts: torch.tensor(
|
||||
[t.stride(0) for t in ts], dtype=torch.int64, device=dev
|
||||
)
|
||||
z = lambda: torch.empty(max_num_reqs, dtype=torch.int32, device=dev)
|
||||
return cls(
|
||||
tuple(conv_states),
|
||||
addr(conv_states),
|
||||
bstr(conv_states),
|
||||
torch.tensor(
|
||||
[s.stride(1) for s in conv_states], dtype=torch.int64, device=dev
|
||||
),
|
||||
torch.tensor(
|
||||
[s.stride(2) for s in conv_states], dtype=torch.int64, device=dev
|
||||
),
|
||||
hist,
|
||||
tuple(checkpoints),
|
||||
addr(checkpoints),
|
||||
bstr(checkpoints),
|
||||
tuple(replays),
|
||||
addr(replays),
|
||||
bstr(replays),
|
||||
z(),
|
||||
z(),
|
||||
z(),
|
||||
z(),
|
||||
spec_query_len,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls, layers: Sequence[Any], *, spec_query_len: int, max_num_reqs: int
|
||||
) -> "GDNRecoverSSMCommitContext":
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
|
||||
if any(len(layer.kv_cache) != 3 for layer in layers):
|
||||
raise ValueError("GDN RecoverSSM pages must contain conv, state and replay")
|
||||
return cls.from_tensors(
|
||||
[layer.kv_cache[0] for layer in layers],
|
||||
[layer.kv_cache[1] for layer in layers],
|
||||
[layer.kv_cache[2] for layer in layers],
|
||||
spec_query_len=spec_query_len,
|
||||
max_num_reqs=max_num_reqs,
|
||||
conv_dim_first=is_conv_state_dim_first(),
|
||||
)
|
||||
|
||||
def commit(
|
||||
self,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
request_indices: torch.Tensor | None = None,
|
||||
block_table: torch.Tensor | None = None,
|
||||
num_computed_tokens: torch.Tensor | None = None,
|
||||
mamba_block_size: int | None = None,
|
||||
commit_conv: bool = True,
|
||||
) -> None:
|
||||
"""Fold accepted GDN and convolution inputs into every layer's checkpoint."""
|
||||
batch = state_indices.shape[0]
|
||||
if batch == 0:
|
||||
return
|
||||
_require(state_indices.ndim == 1, "state indices must be one-dimensional")
|
||||
_require(
|
||||
batch <= self.commit_lens.shape[0], "commit batch exceeds its plan capacity"
|
||||
)
|
||||
_require(
|
||||
query_start_loc.ndim == 1 and query_start_loc.shape[0] == batch + 1,
|
||||
"commit metadata is incompatible",
|
||||
)
|
||||
_require(
|
||||
request_indices is None or request_indices.shape[0] >= batch,
|
||||
"request mapping is too short",
|
||||
)
|
||||
_require(
|
||||
num_accepted_tokens.ndim == 1
|
||||
and (request_indices is not None or num_accepted_tokens.shape[0] >= batch),
|
||||
"accepted-token counts are too short",
|
||||
)
|
||||
_align = (block_table, num_computed_tokens, mamba_block_size)
|
||||
_require(
|
||||
all(x is None for x in _align) or all(x is not None for x in _align),
|
||||
"align metadata is incomplete",
|
||||
)
|
||||
_require(
|
||||
mamba_block_size is None or mamba_block_size >= self.spec_query_len,
|
||||
"align block size must cover one speculative window",
|
||||
)
|
||||
_require(
|
||||
block_table is None or block_table.ndim == 2,
|
||||
"block table must be two-dimensional",
|
||||
)
|
||||
_dev = self.checkpoints[0].device
|
||||
_require(
|
||||
all(
|
||||
t_ is None or t_.device == _dev
|
||||
for t_ in (
|
||||
num_accepted_tokens,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
request_indices,
|
||||
block_table,
|
||||
num_computed_tokens,
|
||||
)
|
||||
),
|
||||
"commit inputs must be on the same device",
|
||||
)
|
||||
bt_stride = (0, 0) if block_table is None else block_table.stride()
|
||||
prepare_commit_plan_kernel[(batch,)](
|
||||
num_accepted_tokens,
|
||||
request_indices,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
block_table,
|
||||
num_computed_tokens,
|
||||
self.commit_lens,
|
||||
self.final_state_indices,
|
||||
self.boundary_state_indices,
|
||||
self.boundary_recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
mamba_block_size or 1,
|
||||
block_table.shape[1] if block_table is not None else 1,
|
||||
num_accepted_tokens.stride(0),
|
||||
request_indices.stride(0) if request_indices is not None else 0,
|
||||
state_indices.stride(0),
|
||||
query_start_loc.stride(0),
|
||||
bt_stride[0],
|
||||
bt_stride[1],
|
||||
0 if num_computed_tokens is None else num_computed_tokens.stride(0),
|
||||
SPEC_QUERY_LEN=self.spec_query_len,
|
||||
num_warps=1,
|
||||
)
|
||||
num_layers = len(self.checkpoints)
|
||||
if commit_conv:
|
||||
conv_ref = self.conv_states[0]
|
||||
conv_dim = conv_ref.shape[1]
|
||||
compact_conv_state_kernel[(triton.cdiv(conv_dim, 256), batch, num_layers)](
|
||||
conv_ref,
|
||||
self.conv_state_base_addrs,
|
||||
self.conv_state_block_strides,
|
||||
self.conv_state_dim_strides,
|
||||
self.conv_state_token_strides,
|
||||
state_indices,
|
||||
self.commit_lens,
|
||||
self.final_state_indices,
|
||||
self.boundary_state_indices,
|
||||
self.boundary_recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
conv_dim,
|
||||
self.conv_history_len,
|
||||
state_indices.stride(0),
|
||||
BLOCK_D=256,
|
||||
BLOCK_HISTORY=triton.next_power_of_2(self.conv_history_len),
|
||||
ALIGN_MODE=block_table is not None,
|
||||
num_warps=4,
|
||||
)
|
||||
ref = self.checkpoints[0]
|
||||
_, HV, V, K = ref.shape
|
||||
BK = triton.next_power_of_2(K)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
rr = self.replays[0]
|
||||
_commit_gdn_state_kernel[(triton.cdiv(V, BV), batch, num_layers * HV)](
|
||||
ref,
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
rr,
|
||||
self.replay_base_addrs,
|
||||
self.replay_block_strides,
|
||||
state_indices,
|
||||
self.commit_lens,
|
||||
self.final_state_indices,
|
||||
self.boundary_state_indices,
|
||||
self.boundary_recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
ref.stride(1),
|
||||
ref.stride(2),
|
||||
ref.stride(3),
|
||||
rr.stride(1),
|
||||
rr.stride(2),
|
||||
rr.stride(3),
|
||||
state_indices.stride(0),
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
NUM_HEADS=HV,
|
||||
ALIGN_MODE=block_table is not None,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["GDNRecoverSSMCommitContext", "gdn_recoverssm_verify"]
|
||||
@@ -0,0 +1,200 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-independent RecoverSSM Triton kernels shared by every recurrent-state
|
||||
implementation.
|
||||
|
||||
RecoverSSM verifies a speculative window from a read-only checkpoint and, after
|
||||
sampling, commits the accepted prefix once. The per-model parts (the KDA and GDN verify
|
||||
kernels and their state reconstruction) live with their models; the kernels they share
|
||||
are here:
|
||||
|
||||
- ``prepare_commit_plan_kernel``: per request, the commit length and the destination
|
||||
state blocks (final and, in align mode, the block-boundary block) from the accepted
|
||||
counts and the block table;
|
||||
- ``compact_conv_state_kernel``: moves the accepted suffix of an extended short-conv
|
||||
window to the front of the destination block (and writes the boundary window in align
|
||||
mode)."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"HAS_REQUEST_INDICES": lambda args: args["request_indices_ptr"] is not None,
|
||||
"ALIGN_MODE": lambda args: args["block_table_ptr"] is not None,
|
||||
}
|
||||
)
|
||||
@triton.jit
|
||||
def prepare_commit_plan_kernel(
|
||||
num_accepted_ptr,
|
||||
request_indices_ptr,
|
||||
state_indices_ptr,
|
||||
query_start_loc_ptr,
|
||||
block_table_ptr,
|
||||
num_computed_ptr,
|
||||
commit_lens_ptr,
|
||||
final_state_indices_ptr,
|
||||
boundary_state_indices_ptr,
|
||||
boundary_recovery_lens_ptr,
|
||||
null_block_id,
|
||||
mamba_block_size,
|
||||
block_table_width,
|
||||
stride_num_accepted,
|
||||
stride_request_indices,
|
||||
stride_state_indices,
|
||||
stride_query_start_loc,
|
||||
stride_block_table_row,
|
||||
stride_block_table_col,
|
||||
stride_num_computed,
|
||||
SPEC_QUERY_LEN: tl.constexpr,
|
||||
HAS_REQUEST_INDICES: tl.constexpr,
|
||||
ALIGN_MODE: tl.constexpr,
|
||||
):
|
||||
spec_idx = tl.program_id(0)
|
||||
source_state_idx = tl.load(state_indices_ptr + spec_idx * stride_state_indices).to(
|
||||
tl.int64
|
||||
)
|
||||
request_idx = spec_idx
|
||||
if HAS_REQUEST_INDICES:
|
||||
request_idx = tl.load(
|
||||
request_indices_ptr + spec_idx * stride_request_indices
|
||||
).to(tl.int64)
|
||||
num_accepted = tl.load(num_accepted_ptr + request_idx * stride_num_accepted).to(
|
||||
tl.int32
|
||||
)
|
||||
bos = tl.load(query_start_loc_ptr + spec_idx * stride_query_start_loc).to(tl.int64)
|
||||
eos = tl.load(query_start_loc_ptr + (spec_idx + 1) * stride_query_start_loc).to(
|
||||
tl.int64
|
||||
)
|
||||
query_len = (eos - bos).to(tl.int32)
|
||||
commit_len = tl.minimum(tl.maximum(num_accepted, 0), query_len)
|
||||
commit_len = tl.minimum(commit_len, SPEC_QUERY_LEN)
|
||||
|
||||
final_state_idx = source_state_idx
|
||||
boundary_state_idx = null_block_id
|
||||
boundary_recovery_len = 0
|
||||
if ALIGN_MODE:
|
||||
num_computed = tl.load(num_computed_ptr + request_idx * stride_num_computed).to(
|
||||
tl.int32
|
||||
)
|
||||
final_num_computed = num_computed + commit_len
|
||||
# The committed state belongs to the block holding the last committed
|
||||
# token. A window ending exactly on a boundary stays in that block; the
|
||||
# next block may still be unallocated.
|
||||
final_state_col = tl.minimum(
|
||||
tl.maximum(final_num_computed - 1, 0) // mamba_block_size,
|
||||
block_table_width - 1,
|
||||
)
|
||||
final_state_idx = tl.load(
|
||||
block_table_ptr
|
||||
+ request_idx * stride_block_table_row
|
||||
+ final_state_col * stride_block_table_col
|
||||
).to(tl.int64)
|
||||
next_boundary = (num_computed // mamba_block_size + 1) * mamba_block_size
|
||||
crosses_boundary = final_num_computed >= next_boundary
|
||||
boundary_recovery_len = next_boundary - num_computed
|
||||
boundary_state_idx = tl.load(
|
||||
block_table_ptr
|
||||
+ request_idx * stride_block_table_row
|
||||
+ (next_boundary // mamba_block_size - 1) * stride_block_table_col,
|
||||
mask=crosses_boundary,
|
||||
other=null_block_id,
|
||||
).to(tl.int64)
|
||||
valid = (source_state_idx > null_block_id) & (commit_len > 0)
|
||||
tl.store(commit_lens_ptr + spec_idx, tl.where(valid, commit_len, 0))
|
||||
tl.store(
|
||||
final_state_indices_ptr + spec_idx,
|
||||
tl.where(valid, final_state_idx, null_block_id),
|
||||
)
|
||||
tl.store(
|
||||
boundary_state_indices_ptr + spec_idx,
|
||||
tl.where(valid, boundary_state_idx, null_block_id),
|
||||
)
|
||||
tl.store(
|
||||
boundary_recovery_lens_ptr + spec_idx,
|
||||
tl.where(valid, boundary_recovery_len, 0),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def compact_conv_state_kernel(
|
||||
conv_state_ref_ptr,
|
||||
conv_state_base_addrs_ptr,
|
||||
conv_state_block_strides_ptr,
|
||||
conv_state_dim_strides_ptr,
|
||||
conv_state_token_strides_ptr,
|
||||
state_indices_ptr,
|
||||
commit_lens_ptr,
|
||||
final_state_indices_ptr,
|
||||
boundary_state_indices_ptr,
|
||||
boundary_recovery_lens_ptr,
|
||||
null_block_id,
|
||||
conv_dim,
|
||||
conv_history_len,
|
||||
stride_state_indices,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_HISTORY: tl.constexpr,
|
||||
ALIGN_MODE: tl.constexpr,
|
||||
):
|
||||
pid_d = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_l = tl.program_id(2)
|
||||
source_state_idx = tl.load(state_indices_ptr + pid_b * stride_state_indices).to(
|
||||
tl.int64
|
||||
)
|
||||
if source_state_idx <= null_block_id:
|
||||
return
|
||||
|
||||
commit_len = tl.load(commit_lens_ptr + pid_b)
|
||||
if commit_len == 0:
|
||||
return
|
||||
final_state_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_state_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_recovery_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||
|
||||
if final_state_idx <= null_block_id:
|
||||
return
|
||||
|
||||
base_addr = tl.load(conv_state_base_addrs_ptr + pid_l)
|
||||
block_stride = tl.load(conv_state_block_strides_ptr + pid_l)
|
||||
dim_stride = tl.load(conv_state_dim_strides_ptr + pid_l)
|
||||
token_stride = tl.load(conv_state_token_strides_ptr + pid_l)
|
||||
conv_state_ptr = base_addr.to(tl.pointer_type(conv_state_ref_ptr.dtype.element_ty))
|
||||
source_ptr = conv_state_ptr + source_state_idx * block_stride
|
||||
final_ptr = conv_state_ptr + final_state_idx * block_stride
|
||||
|
||||
offs_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
|
||||
offs_h = tl.arange(0, BLOCK_HISTORY)
|
||||
mask = (offs_d[:, None] < conv_dim) & (offs_h[None, :] < conv_history_len)
|
||||
final_values = tl.load(
|
||||
source_ptr
|
||||
+ offs_d[:, None] * dim_stride
|
||||
+ (commit_len - 1 + offs_h[None, :]) * token_stride,
|
||||
mask=mask,
|
||||
)
|
||||
if ALIGN_MODE:
|
||||
boundary_values = tl.load(
|
||||
source_ptr
|
||||
+ offs_d[:, None] * dim_stride
|
||||
+ (boundary_recovery_len - 1 + offs_h[None, :]) * token_stride,
|
||||
mask=mask & (boundary_state_idx > null_block_id),
|
||||
)
|
||||
boundary_ptr = conv_state_ptr + boundary_state_idx * block_stride
|
||||
tl.store(
|
||||
boundary_ptr
|
||||
+ offs_d[:, None] * dim_stride
|
||||
+ offs_h[None, :] * token_stride,
|
||||
boundary_values,
|
||||
mask=mask & (boundary_state_idx > null_block_id),
|
||||
)
|
||||
tl.store(
|
||||
final_ptr + offs_d[:, None] * dim_stride + offs_h[None, :] * token_stride,
|
||||
final_values,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"compact_conv_state_kernel",
|
||||
"prepare_commit_plan_kernel",
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-independent RecoverSSM Python helpers: the activation predicate shared by
|
||||
layers, models and backends, and argument validation that raises ``ValueError``
|
||||
before any raw-pointer kernel launch."""
|
||||
|
||||
|
||||
def recoverssm_require(cond: bool, msg: str, component: str = "RecoverSSM") -> None:
|
||||
if not cond:
|
||||
raise ValueError(f"{component}: {msg}")
|
||||
|
||||
|
||||
def uses_recoverssm(cache_config, num_speculative_tokens: int) -> bool:
|
||||
"""True when this model instance runs RecoverSSM for its recurrent states.
|
||||
|
||||
VllmConfig sets ``cache_config.use_recoverssm`` for every model whose recurrent
|
||||
states support it (Kimi-K3 KDA, Qwen GDN/PLE). Without speculative tokens there is
|
||||
nothing to recover, so the stock path runs."""
|
||||
return (
|
||||
bool(getattr(cache_config, "use_recoverssm", False))
|
||||
and num_speculative_tokens > 0
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["recoverssm_require", "uses_recoverssm"]
|
||||
Reference in New Issue
Block a user