[Spec Decode][Qwen4Exp] RecoverSSM for Qwen GDN and the PLE short conv
With --use-replayssm and speculative decoding, Qwen4Exp's Gated DeltaNet layers verify the draft window from a read-only checkpoint and keep a per-token replay record [correction(V), normalized key(K), gate(1)] in fp32. After sampling, one commit replays the accepted tokens and writes the recurrent state once (plus the block-boundary state in align mode), instead of snapshotting the state after every draft position. The PLE dilated short conv follows the same protocol (accepted-suffix compaction). num_speculative_blocks = 0, so no per-draft Mamba blocks are reserved. - GDN verify/commit kernels: model_executor/layers/mamba/gdn/recoverssm_gdn.py - metadata/backends: v1/attention/backends/gdn_recoverssm.py, ple_recoverssm.py - the verify reads strided q/k/v views of the conv output and writes in place - builders declare AttentionCGSupport.NEVER: FULL_AND_PIECEWISE falls back to PIECEWISE; the fallback now also counts breakable CUDA graphs (empty splitting_ops, with or without torch.compile) as piecewise attention instead of dropping to NONE - the generic Mamba metadata builder skips the Mamba2 ReplaySSM ring-buffer workspaces when --use-replayssm selects RecoverSSM (the PLE short conv uses that builder) - tests: GDN verify/commit vs the native kernels (fp32 and bf16 state, align boundaries, mixed and ragged batches, extreme gates), PLE vs token-by-token decode, config selection Signed-off-by: Jürgen Schmied <juergenschmied70@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -370,10 +370,15 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
|
||||
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
||||
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
def get_state_shape(
|
||||
self,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
def _uses_gdn_recoverssm(self) -> bool:
|
||||
"""One predicate for the replay state's shape, dtype and the attention
|
||||
backend."""
|
||||
from vllm.model_executor.layers.mamba.recoverssm_utils import uses_recoverssm
|
||||
|
||||
return uses_recoverssm(self.cache_config, self.num_spec)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
|
||||
shapes = MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||
self.tp_size,
|
||||
self.num_k_heads,
|
||||
self.num_v_heads,
|
||||
@@ -382,6 +387,32 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.conv_kernel_size,
|
||||
self.num_spec,
|
||||
)
|
||||
if self._uses_gdn_recoverssm():
|
||||
# per-token replay record [HV, spec_query_len, V + K + 1] fp32
|
||||
shapes = (
|
||||
*shapes,
|
||||
(
|
||||
self.num_v_heads // self.tp_size,
|
||||
self.num_spec + 1,
|
||||
self.head_v_dim + self.head_k_dim + 1,
|
||||
),
|
||||
)
|
||||
return shapes
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
dtypes = super().get_state_dtype()
|
||||
if self._uses_gdn_recoverssm():
|
||||
dtypes = (*dtypes, torch.float32)
|
||||
return dtypes
|
||||
|
||||
def get_attn_backend(self):
|
||||
if self._uses_gdn_recoverssm():
|
||||
from vllm.v1.attention.backends.gdn_recoverssm import (
|
||||
GDNRecoverSSMAttentionBackend,
|
||||
)
|
||||
|
||||
return GDNRecoverSSMAttentionBackend
|
||||
return super().get_attn_backend()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -530,6 +561,17 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.gdn_decode_kernel = "XPU"
|
||||
|
||||
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
|
||||
# RecoverSSM verify replaces the per-draft snapshot path (Triton decode
|
||||
# only)
|
||||
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
|
||||
if self.use_gdn_recoverssm:
|
||||
self.gdn_decode_kernel = "triton"
|
||||
self.enable_fused_gdn_decode = False
|
||||
self.enable_packed_recurrent_decode = False
|
||||
logger.info_once(
|
||||
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
|
||||
self.num_spec + 1,
|
||||
)
|
||||
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
@@ -540,7 +582,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
def _fused_gdn_decode_unsupported_reason(
|
||||
self, vllm_config: VllmConfig
|
||||
) -> str | None:
|
||||
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
|
||||
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2]
|
||||
if (
|
||||
self.gqa_interleaved_layout
|
||||
or self.head_k_dim != 128
|
||||
@@ -800,6 +842,25 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
|
||||
return mixed_qkv_out, z_out, b_out, a_out
|
||||
|
||||
def _recoverssm_verify_qkv(self, mixed_qkv):
|
||||
"""Q/K/V for the RecoverSSM verify.
|
||||
|
||||
The verify kernel reads token-strided heads, so for a 2-D [tokens, q|k|v] conv
|
||||
output with a contiguous last dimension it gets views instead of copies. Any
|
||||
other layout falls back to the contiguous copies of rearrange_mixed_qkv().
|
||||
Returns (query, key, value, is_view)."""
|
||||
if mixed_qkv.ndim != 2 or mixed_qkv.stride(-1) != 1:
|
||||
return (*self.rearrange_mixed_qkv(mixed_qkv), False)
|
||||
q_dim = self.key_dim // self.tp_size
|
||||
v_dim = self.value_dim // self.tp_size
|
||||
num_tokens = mixed_qkv.shape[0]
|
||||
query = mixed_qkv[:, :q_dim].view(1, num_tokens, -1, self.head_k_dim)
|
||||
key = mixed_qkv[:, q_dim : 2 * q_dim].view(1, num_tokens, -1, self.head_k_dim)
|
||||
value = mixed_qkv[:, 2 * q_dim : 2 * q_dim + v_dim].view(
|
||||
1, num_tokens, -1, self.head_v_dim
|
||||
)
|
||||
return query, key, value, True
|
||||
|
||||
def rearrange_mixed_qkv(self, mixed_qkv):
|
||||
"""Split packed qkv into contiguous (1, seq, heads, dim) tensors.
|
||||
|
||||
@@ -1085,7 +1146,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
dtype = qkv_or_qkvz.dtype
|
||||
num_k_heads = self.num_k_heads // self.tp_size
|
||||
num_v_heads = self.num_v_heads // self.tp_size
|
||||
_, state_dtype = self.get_state_dtype()
|
||||
state_dtype = self.get_state_dtype()[1] # 3 dtypes with the replay record
|
||||
|
||||
# All kernels use BT = chunk_size, so a single pass with T = chunk_size
|
||||
# is sufficient to populate every autotuner cache. Mirror the real
|
||||
@@ -1343,7 +1404,11 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
],
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
max_query_len=spec_state_indices_tensor.size(-1),
|
||||
max_query_len=(
|
||||
self.num_spec + 1
|
||||
if getattr(self, "use_gdn_recoverssm", False)
|
||||
else spec_state_indices_tensor.size(-1)
|
||||
),
|
||||
validate_data=False,
|
||||
)
|
||||
|
||||
@@ -1380,7 +1445,17 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
else:
|
||||
mixed_qkv_non_spec = None
|
||||
|
||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||
if (
|
||||
spec_sequence_masks is not None
|
||||
and mixed_qkv_spec is not None
|
||||
and self.use_gdn_recoverssm
|
||||
):
|
||||
query_spec, key_spec, value_spec, verify_views = (
|
||||
self._recoverssm_verify_qkv(mixed_qkv_spec)
|
||||
)
|
||||
else:
|
||||
verify_views = False
|
||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||
|
||||
# Split mixed non-spec-decode+prefill to process independently
|
||||
split_non_spec = (
|
||||
@@ -1443,7 +1518,43 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
# 2. Recurrent attention
|
||||
|
||||
# 2.1: Process the multi-query part
|
||||
if spec_sequence_masks is not None:
|
||||
if spec_sequence_masks is not None and getattr(
|
||||
self, "use_gdn_recoverssm", False
|
||||
):
|
||||
# verify the window off the checkpoint; the commit after sampling
|
||||
# writes the state once
|
||||
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
|
||||
gdn_recoverssm_verify,
|
||||
)
|
||||
|
||||
_n = attn_metadata.num_spec_decodes
|
||||
assert spec_query_start_loc is not None
|
||||
assert spec_state_indices_tensor is not None
|
||||
core_attn_out_spec = gdn_recoverssm_verify(
|
||||
self.A_log,
|
||||
a_spec,
|
||||
b_spec,
|
||||
self.dt_bias,
|
||||
query_spec,
|
||||
key_spec,
|
||||
value_spec,
|
||||
checkpoint_state=ssm_state,
|
||||
replay_cache=self_kv_cache[2],
|
||||
query_start_loc=spec_query_start_loc[: _n + 1],
|
||||
state_indices=spec_state_indices_tensor[:_n, 0],
|
||||
spec_query_len=self.num_spec + 1,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
# spec-only batch: write straight into the layer output
|
||||
out=(
|
||||
core_attn_out[:num_actual_tokens].unsqueeze(0)
|
||||
if verify_views
|
||||
and mixed_qkv_non_spec is None
|
||||
and query_spec.shape[1] == num_actual_tokens
|
||||
else None
|
||||
),
|
||||
)
|
||||
last_recurrent_state = None
|
||||
elif spec_sequence_masks is not None:
|
||||
core_attn_out_spec, last_recurrent_state = (
|
||||
fused_sigmoid_gating_delta_rule_update(
|
||||
A_log=self.A_log,
|
||||
@@ -1558,7 +1669,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
0, non_spec_token_indx, core_attn_out_non_spec.squeeze(0)
|
||||
)
|
||||
elif spec_sequence_masks is not None:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||
if core_attn_out_spec.data_ptr() != core_attn_out.data_ptr():
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||
else:
|
||||
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
|
||||
|
||||
|
||||
@@ -0,0 +1,695 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RecoverSSM for Gated DeltaNet (Qwen-style GDN): speculative verify from one
|
||||
checkpoint without per-draft state snapshots, and accepted-state recovery after
|
||||
sampling.
|
||||
|
||||
GDN port of the Kimi-K3 KDA RecoverSSM path
|
||||
(vllm/models/kimi_k3/nvidia/ops/recoverssm.py). Differences from KDA: the gate is a
|
||||
scalar per value head (g = -exp(A_log) * softplus(a + dt_bias)), q/k heads are grouped
|
||||
over value heads (HV // H), and q/k are L2-normalised inside the kernel when requested.
|
||||
The verify step caches, per token, the delta-rule correction c_t = beta_t * (v_t - S'_t
|
||||
k_t) (V floats), the normalised key (K floats) and the log-decay g_t (1 float) in one
|
||||
fp32 "replay" record. Recovery folds the accepted prefix: S_n = exp(sum_t g_t) S_0 +
|
||||
sum_t exp(sum_{s>t} g_s) c_t k_t^T and writes the checkpoint once (plus the
|
||||
block-boundary state in align mode)."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.recoverssm import (
|
||||
compact_conv_state_kernel,
|
||||
prepare_commit_plan_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.recoverssm_utils import recoverssm_require
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def _require(cond: bool, msg: str) -> None:
|
||||
recoverssm_require(cond, msg, "GDN RecoverSSM")
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gdn_recoverssm_verify_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
A_log_ptr,
|
||||
dt_bias_ptr,
|
||||
state_ptr,
|
||||
replay_ptr,
|
||||
out_ptr,
|
||||
query_start_loc_ptr,
|
||||
state_indices_ptr,
|
||||
scale,
|
||||
softplus_beta,
|
||||
softplus_threshold,
|
||||
null_block_id,
|
||||
stride_q_token,
|
||||
stride_k_token,
|
||||
stride_v_token,
|
||||
stride_a_token,
|
||||
stride_b_token,
|
||||
stride_state_block,
|
||||
stride_state_head,
|
||||
stride_state_v,
|
||||
stride_state_k,
|
||||
stride_replay_block,
|
||||
stride_replay_head,
|
||||
stride_replay_pos,
|
||||
stride_replay_dim,
|
||||
stride_out_token,
|
||||
stride_qsl,
|
||||
stride_si,
|
||||
H: tl.constexpr,
|
||||
HV: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
SPEC_QUERY_LEN: tl.constexpr,
|
||||
USE_QK_L2NORM: tl.constexpr,
|
||||
):
|
||||
pid_v = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_hv = tl.program_id(2)
|
||||
pid_h = pid_hv // (HV // H)
|
||||
bos = tl.load(query_start_loc_ptr + pid_b * stride_qsl).to(tl.int64)
|
||||
eos = tl.load(query_start_loc_ptr + (pid_b + 1) * stride_qsl).to(tl.int64)
|
||||
query_len = eos - bos
|
||||
state_idx = tl.load(state_indices_ptr + pid_b * stride_si).to(tl.int64)
|
||||
offs_k = tl.arange(0, BK)
|
||||
offs_v = pid_v * BV + tl.arange(0, BV)
|
||||
mask_k = offs_k < K
|
||||
mask_v = offs_v < V
|
||||
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||
if state_idx <= null_block_id:
|
||||
for t in tl.static_range(SPEC_QUERY_LEN):
|
||||
tl.store(
|
||||
out_ptr + (bos + t) * stride_out_token + pid_hv * V + offs_v,
|
||||
tl.zeros([BV], dtype=tl.float32),
|
||||
mask=(t < query_len) & mask_v,
|
||||
)
|
||||
return
|
||||
state = tl.load(
|
||||
state_ptr
|
||||
+ state_idx * stride_state_block
|
||||
+ pid_hv * stride_state_head
|
||||
+ offs_v[:, None] * stride_state_v
|
||||
+ offs_k[None, :] * stride_state_k,
|
||||
mask=mask_state,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
neg_A = -tl.exp(tl.load(A_log_ptr + pid_hv).to(tl.float32))
|
||||
dt_bias = tl.load(dt_bias_ptr + pid_hv).to(tl.float32)
|
||||
rec_base = (
|
||||
replay_ptr + state_idx * stride_replay_block + pid_hv * stride_replay_head
|
||||
)
|
||||
for t in tl.static_range(SPEC_QUERY_LEN):
|
||||
valid = t < query_len
|
||||
tok = bos + t
|
||||
q = tl.load(
|
||||
q_ptr + tok * stride_q_token + pid_h * K + offs_k,
|
||||
mask=valid & mask_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
k = tl.load(
|
||||
k_ptr + tok * stride_k_token + pid_h * K + offs_k,
|
||||
mask=valid & mask_k,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
v = tl.load(
|
||||
v_ptr + tok * stride_v_token + pid_hv * V + offs_v,
|
||||
mask=valid & mask_v,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
x = (
|
||||
tl.load(a_ptr + tok * stride_a_token + pid_hv, mask=valid, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
+ dt_bias
|
||||
)
|
||||
sp = tl.where(
|
||||
softplus_beta * x <= softplus_threshold,
|
||||
(1 / softplus_beta) * tl.log(1 + tl.exp(softplus_beta * x)),
|
||||
x,
|
||||
)
|
||||
g = neg_A * sp
|
||||
beta = tl.sigmoid(
|
||||
tl.load(b_ptr + tok * stride_b_token + pid_hv, mask=valid, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
)
|
||||
if USE_QK_L2NORM:
|
||||
q = q * tl.rsqrt(tl.sum(q * q) + 1e-6)
|
||||
k = k * tl.rsqrt(tl.sum(k * k) + 1e-6)
|
||||
q = q * scale
|
||||
new_state = state * tl.exp(g)
|
||||
corr = (v - tl.sum(new_state * k[None, :], 1)) * beta
|
||||
new_state = new_state + corr[:, None] * k[None, :]
|
||||
state = tl.where(valid, new_state, state)
|
||||
o = tl.sum(state * q[None, :], 1)
|
||||
tl.store(
|
||||
out_ptr + tok * stride_out_token + pid_hv * V + offs_v,
|
||||
o.to(out_ptr.dtype.element_ty),
|
||||
mask=valid & mask_v,
|
||||
)
|
||||
rec = rec_base + t * stride_replay_pos
|
||||
tl.store(rec + offs_v * stride_replay_dim, corr, mask=valid & mask_v)
|
||||
if pid_v == 0:
|
||||
tl.store(rec + (V + offs_k) * stride_replay_dim, k, mask=valid & mask_k)
|
||||
tl.store(rec + (V + K) * stride_replay_dim, g, mask=valid)
|
||||
|
||||
|
||||
def gdn_recoverssm_verify(
|
||||
A_log: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
*,
|
||||
checkpoint_state: torch.Tensor,
|
||||
replay_cache: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
spec_query_len: int,
|
||||
scale: float | None = None,
|
||||
softplus_beta: float = 1.0,
|
||||
softplus_threshold: float = 20.0,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Verify a GDN speculative window against its checkpoint without modifying the
|
||||
checkpoint. q, k: [1, T, H, K]; v: [1, T, HV, V]; a, b: [T, HV] (or [1, T, HV]);
|
||||
checkpoint_state: [blocks, HV, V, K]; replay_cache: [blocks, HV, spec_query_len, V +
|
||||
K + 1] fp32. Returns out [1, T, HV, V]."""
|
||||
_require(
|
||||
q.ndim == 4 and q.shape[0] == 1, "q must have shape [1, tokens, heads, dim]"
|
||||
)
|
||||
_, total, H, K = q.shape
|
||||
_require(k.shape == q.shape, "q and k shapes differ")
|
||||
_require(
|
||||
v.ndim == 4 and tuple(v.shape[:2]) == (1, total),
|
||||
"v must have shape [1, tokens, value heads, dim]",
|
||||
)
|
||||
HV, V = v.shape[-2], v.shape[-1]
|
||||
_require(
|
||||
H > 0 and HV >= H and HV % H == 0,
|
||||
"value heads must be a positive multiple of the q/k heads",
|
||||
)
|
||||
_require(
|
||||
a.numel() == total * HV and b.numel() == total * HV,
|
||||
"gate or beta shape is incompatible",
|
||||
)
|
||||
a2 = a.reshape(total, HV)
|
||||
b2 = b.reshape(total, HV)
|
||||
_require(
|
||||
a2.stride(1) == 1 and b2.stride(1) == 1,
|
||||
"gate and beta heads must be contiguous",
|
||||
)
|
||||
_require(
|
||||
tuple(A_log.shape) == (HV,) and tuple(dt_bias.shape) == (HV,),
|
||||
"A_log or dt_bias shape is incompatible",
|
||||
)
|
||||
_require(
|
||||
all(t_.stride(-1) == 1 and t_.stride(-2) == K for t_ in (q, k)),
|
||||
"q and k heads must be contiguous",
|
||||
)
|
||||
_require(v.stride(-1) == 1 and v.stride(-2) == V, "v heads must be contiguous")
|
||||
_require(checkpoint_state.ndim == 4, "checkpoint must be four-dimensional")
|
||||
nb = checkpoint_state.shape[0]
|
||||
_require(
|
||||
tuple(checkpoint_state.shape[1:]) == (HV, V, K),
|
||||
"checkpoint shape is incompatible",
|
||||
)
|
||||
_rec = (nb, HV, spec_query_len, V + K + 1)
|
||||
_require(tuple(replay_cache.shape) == _rec, f"replay buffer needs shape {_rec}")
|
||||
_require(replay_cache.dtype == torch.float32, "replay buffer must use float32")
|
||||
_require(state_indices.ndim == 1, "state indices must be one-dimensional")
|
||||
_require(
|
||||
query_start_loc.ndim == 1
|
||||
and query_start_loc.shape[0] == state_indices.shape[0] + 1,
|
||||
"query metadata is incompatible",
|
||||
)
|
||||
_require(
|
||||
total <= state_indices.shape[0] * spec_query_len,
|
||||
"speculative decode input exceeds its activation capacity",
|
||||
)
|
||||
_dev = q.device
|
||||
_require(
|
||||
all(
|
||||
t_.device == _dev
|
||||
for t_ in (
|
||||
k,
|
||||
v,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
checkpoint_state,
|
||||
replay_cache,
|
||||
query_start_loc,
|
||||
state_indices,
|
||||
)
|
||||
if t_ is not None
|
||||
)
|
||||
and (out is None or out.device == _dev),
|
||||
"inputs must be on the same device",
|
||||
)
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
if out is None:
|
||||
out = torch.empty(1, total, HV, V, dtype=v.dtype, device=v.device)
|
||||
_require(
|
||||
tuple(out.shape) == (1, total, HV, V) and out.stride()[2:] == (V, 1),
|
||||
"output shape or layout is incompatible",
|
||||
)
|
||||
batch = state_indices.shape[0]
|
||||
if total == 0 or batch == 0:
|
||||
return out
|
||||
BK = triton.next_power_of_2(K)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
grid = (triton.cdiv(V, BV), batch, HV)
|
||||
_gdn_recoverssm_verify_kernel[grid](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
a2,
|
||||
b2,
|
||||
A_log,
|
||||
dt_bias,
|
||||
checkpoint_state,
|
||||
replay_cache,
|
||||
out,
|
||||
query_start_loc,
|
||||
state_indices,
|
||||
scale,
|
||||
softplus_beta,
|
||||
softplus_threshold,
|
||||
NULL_BLOCK_ID,
|
||||
q.stride(1),
|
||||
k.stride(1),
|
||||
v.stride(1),
|
||||
a2.stride(0),
|
||||
b2.stride(0),
|
||||
checkpoint_state.stride(0),
|
||||
checkpoint_state.stride(1),
|
||||
checkpoint_state.stride(2),
|
||||
checkpoint_state.stride(3),
|
||||
replay_cache.stride(0),
|
||||
replay_cache.stride(1),
|
||||
replay_cache.stride(2),
|
||||
replay_cache.stride(3),
|
||||
out.stride(1),
|
||||
query_start_loc.stride(0),
|
||||
state_indices.stride(0),
|
||||
H=H,
|
||||
HV=HV,
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
SPEC_QUERY_LEN=spec_query_len,
|
||||
USE_QK_L2NORM=use_qk_l2norm_in_kernel,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _commit_gdn_state_kernel(
|
||||
state_ref_ptr,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
replay_ref_ptr,
|
||||
replay_base_addrs_ptr,
|
||||
replay_block_strides_ptr,
|
||||
state_indices_ptr,
|
||||
commit_lens_ptr,
|
||||
final_state_indices_ptr,
|
||||
boundary_state_indices_ptr,
|
||||
boundary_recovery_lens_ptr,
|
||||
null_block_id,
|
||||
stride_state_head,
|
||||
stride_state_v,
|
||||
stride_state_k,
|
||||
stride_replay_head,
|
||||
stride_replay_pos,
|
||||
stride_replay_dim,
|
||||
stride_si,
|
||||
K: tl.constexpr,
|
||||
V: tl.constexpr,
|
||||
BK: tl.constexpr,
|
||||
BV: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
ALIGN_MODE: tl.constexpr,
|
||||
):
|
||||
pid_v = tl.program_id(0)
|
||||
pid_b = tl.program_id(1)
|
||||
pid_lh = tl.program_id(2)
|
||||
pid_l = pid_lh // NUM_HEADS
|
||||
pid_h = pid_lh % NUM_HEADS
|
||||
src = tl.load(state_indices_ptr + pid_b * stride_si).to(tl.int64)
|
||||
if src <= null_block_id:
|
||||
return
|
||||
n = tl.load(commit_lens_ptr + pid_b)
|
||||
if n == 0:
|
||||
return
|
||||
final_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||
boundary_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||
if final_idx <= null_block_id:
|
||||
return
|
||||
state_ptr = tl.load(state_base_addrs_ptr + pid_l).to(
|
||||
tl.pointer_type(state_ref_ptr.dtype.element_ty)
|
||||
)
|
||||
sbs = tl.load(state_block_strides_ptr + pid_l)
|
||||
replay_ptr = tl.load(replay_base_addrs_ptr + pid_l).to(
|
||||
tl.pointer_type(replay_ref_ptr.dtype.element_ty)
|
||||
)
|
||||
rbs = tl.load(replay_block_strides_ptr + pid_l)
|
||||
rec_base = replay_ptr + src * rbs + pid_h * stride_replay_head
|
||||
offs_k = tl.arange(0, BK)
|
||||
offs_v = pid_v * BV + tl.arange(0, BV)
|
||||
mask_k = offs_k < K
|
||||
mask_v = offs_v < V
|
||||
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||
sp = offs_v[:, None] * stride_state_v + offs_k[None, :] * stride_state_k
|
||||
s0 = tl.load(
|
||||
state_ptr + src * sbs + pid_h * stride_state_head + sp,
|
||||
mask=mask_state,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
fin_decay = 1.0
|
||||
fin_corr = tl.zeros([BV, BK], tl.float32)
|
||||
bnd_decay = 1.0
|
||||
bnd_corr = tl.zeros([BV, BK], tl.float32)
|
||||
for r in range(n):
|
||||
t = n - r - 1
|
||||
rec = rec_base + t * stride_replay_pos
|
||||
c = tl.load(rec + offs_v * stride_replay_dim, mask=mask_v, other=0.0)
|
||||
kk = tl.load(rec + (V + offs_k) * stride_replay_dim, mask=mask_k, other=0.0)
|
||||
g = tl.load(rec + (V + K) * stride_replay_dim)
|
||||
upd = c[:, None] * kk[None, :]
|
||||
dec = tl.exp(g)
|
||||
fin_corr += upd * fin_decay
|
||||
fin_decay *= dec
|
||||
if ALIGN_MODE:
|
||||
before = t < boundary_len
|
||||
bnd_corr += tl.where(before, upd * bnd_decay, 0.0)
|
||||
bnd_decay *= tl.where(before, dec, 1.0)
|
||||
if ALIGN_MODE:
|
||||
bs = s0 * bnd_decay + bnd_corr
|
||||
tl.store(
|
||||
state_ptr + boundary_idx * sbs + pid_h * stride_state_head + sp,
|
||||
bs.to(state_ref_ptr.dtype.element_ty),
|
||||
mask=mask_state & (boundary_idx > null_block_id),
|
||||
)
|
||||
fs = s0 * fin_decay + fin_corr
|
||||
tl.store(
|
||||
state_ptr + final_idx * sbs + pid_h * stride_state_head + sp,
|
||||
fs.to(state_ref_ptr.dtype.element_ty),
|
||||
mask=mask_state,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GDNRecoverSSMCommitContext:
|
||||
conv_states: tuple[torch.Tensor, ...]
|
||||
conv_state_base_addrs: torch.Tensor
|
||||
conv_state_block_strides: torch.Tensor
|
||||
conv_state_dim_strides: torch.Tensor
|
||||
conv_state_token_strides: torch.Tensor
|
||||
conv_history_len: int
|
||||
checkpoints: tuple[torch.Tensor, ...]
|
||||
state_base_addrs: torch.Tensor
|
||||
state_block_strides: torch.Tensor
|
||||
replays: tuple[torch.Tensor, ...]
|
||||
replay_base_addrs: torch.Tensor
|
||||
replay_block_strides: torch.Tensor
|
||||
commit_lens: torch.Tensor
|
||||
final_state_indices: torch.Tensor
|
||||
boundary_state_indices: torch.Tensor
|
||||
boundary_recovery_lens: torch.Tensor
|
||||
spec_query_len: int
|
||||
|
||||
@classmethod
|
||||
def from_tensors(
|
||||
cls,
|
||||
conv_states: Sequence[torch.Tensor],
|
||||
checkpoints: Sequence[torch.Tensor],
|
||||
replays: Sequence[torch.Tensor],
|
||||
*,
|
||||
spec_query_len: int,
|
||||
max_num_reqs: int,
|
||||
conv_dim_first: bool = True,
|
||||
) -> "GDNRecoverSSMCommitContext":
|
||||
_require(len(checkpoints) > 0, "commit requires at least one layer")
|
||||
_require(
|
||||
len(conv_states) == len(checkpoints) == len(replays),
|
||||
"conv, state and replay lists differ",
|
||||
)
|
||||
if not conv_dim_first:
|
||||
conv_states = [s.transpose(-1, -2) for s in conv_states]
|
||||
ref = checkpoints[0]
|
||||
_require(ref.ndim == 4, "checkpoint must be four-dimensional")
|
||||
nb, HV, V, K = ref.shape
|
||||
_dev = ref.device
|
||||
for s in checkpoints:
|
||||
_require(
|
||||
s.shape == ref.shape
|
||||
and s.dtype == ref.dtype
|
||||
and s.stride()[1:] == ref.stride()[1:]
|
||||
and s.device == _dev,
|
||||
"layers need matching checkpoints",
|
||||
)
|
||||
_rec = (nb, HV, spec_query_len, V + K + 1)
|
||||
for r in replays:
|
||||
_require(
|
||||
tuple(r.shape) == _rec
|
||||
and r.dtype == torch.float32
|
||||
and r.stride()[1:] == replays[0].stride()[1:]
|
||||
and r.device == _dev,
|
||||
f"layers need matching float32 replay buffers of shape {_rec}",
|
||||
)
|
||||
conv_ref = conv_states[0]
|
||||
_require(
|
||||
conv_ref.ndim == 3 and conv_ref.shape[0] == nb,
|
||||
"conv state must be [blocks, dim, window] with the checkpoint's "
|
||||
"block count",
|
||||
)
|
||||
for c in conv_states:
|
||||
_require(
|
||||
c.shape == conv_ref.shape
|
||||
and c.dtype == conv_ref.dtype
|
||||
and c.stride()[1:] == conv_ref.stride()[1:]
|
||||
and c.device == _dev,
|
||||
"layers need matching conv states",
|
||||
)
|
||||
conv_dim, conv_len = conv_ref.shape[1:]
|
||||
hist = conv_len - spec_query_len + 1
|
||||
_require(hist > 0, "conv state is shorter than its window")
|
||||
dev = ref.device
|
||||
addr = lambda ts: torch.tensor(
|
||||
[t.data_ptr() for t in ts], dtype=torch.int64, device=dev
|
||||
)
|
||||
bstr = lambda ts: torch.tensor(
|
||||
[t.stride(0) for t in ts], dtype=torch.int64, device=dev
|
||||
)
|
||||
z = lambda: torch.empty(max_num_reqs, dtype=torch.int32, device=dev)
|
||||
return cls(
|
||||
tuple(conv_states),
|
||||
addr(conv_states),
|
||||
bstr(conv_states),
|
||||
torch.tensor(
|
||||
[s.stride(1) for s in conv_states], dtype=torch.int64, device=dev
|
||||
),
|
||||
torch.tensor(
|
||||
[s.stride(2) for s in conv_states], dtype=torch.int64, device=dev
|
||||
),
|
||||
hist,
|
||||
tuple(checkpoints),
|
||||
addr(checkpoints),
|
||||
bstr(checkpoints),
|
||||
tuple(replays),
|
||||
addr(replays),
|
||||
bstr(replays),
|
||||
z(),
|
||||
z(),
|
||||
z(),
|
||||
z(),
|
||||
spec_query_len,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls, layers: Sequence[Any], *, spec_query_len: int, max_num_reqs: int
|
||||
) -> "GDNRecoverSSMCommitContext":
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
|
||||
if any(len(layer.kv_cache) != 3 for layer in layers):
|
||||
raise ValueError("GDN RecoverSSM pages must contain conv, state and replay")
|
||||
return cls.from_tensors(
|
||||
[layer.kv_cache[0] for layer in layers],
|
||||
[layer.kv_cache[1] for layer in layers],
|
||||
[layer.kv_cache[2] for layer in layers],
|
||||
spec_query_len=spec_query_len,
|
||||
max_num_reqs=max_num_reqs,
|
||||
conv_dim_first=is_conv_state_dim_first(),
|
||||
)
|
||||
|
||||
def commit(
|
||||
self,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
state_indices: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
request_indices: torch.Tensor | None = None,
|
||||
block_table: torch.Tensor | None = None,
|
||||
num_computed_tokens: torch.Tensor | None = None,
|
||||
mamba_block_size: int | None = None,
|
||||
commit_conv: bool = True,
|
||||
) -> None:
|
||||
"""Fold accepted GDN and convolution inputs into every layer's checkpoint."""
|
||||
batch = state_indices.shape[0]
|
||||
if batch == 0:
|
||||
return
|
||||
_require(state_indices.ndim == 1, "state indices must be one-dimensional")
|
||||
_require(
|
||||
batch <= self.commit_lens.shape[0], "commit batch exceeds its plan capacity"
|
||||
)
|
||||
_require(
|
||||
query_start_loc.ndim == 1 and query_start_loc.shape[0] == batch + 1,
|
||||
"commit metadata is incompatible",
|
||||
)
|
||||
_require(
|
||||
request_indices is None or request_indices.shape[0] >= batch,
|
||||
"request mapping is too short",
|
||||
)
|
||||
_require(
|
||||
num_accepted_tokens.ndim == 1
|
||||
and (request_indices is not None or num_accepted_tokens.shape[0] >= batch),
|
||||
"accepted-token counts are too short",
|
||||
)
|
||||
_align = (block_table, num_computed_tokens, mamba_block_size)
|
||||
_require(
|
||||
all(x is None for x in _align) or all(x is not None for x in _align),
|
||||
"align metadata is incomplete",
|
||||
)
|
||||
_require(
|
||||
mamba_block_size is None or mamba_block_size >= self.spec_query_len,
|
||||
"align block size must cover one speculative window",
|
||||
)
|
||||
_require(
|
||||
block_table is None or block_table.ndim == 2,
|
||||
"block table must be two-dimensional",
|
||||
)
|
||||
_dev = self.checkpoints[0].device
|
||||
_require(
|
||||
all(
|
||||
t_ is None or t_.device == _dev
|
||||
for t_ in (
|
||||
num_accepted_tokens,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
request_indices,
|
||||
block_table,
|
||||
num_computed_tokens,
|
||||
)
|
||||
),
|
||||
"commit inputs must be on the same device",
|
||||
)
|
||||
bt_stride = (0, 0) if block_table is None else block_table.stride()
|
||||
prepare_commit_plan_kernel[(batch,)](
|
||||
num_accepted_tokens,
|
||||
request_indices,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
block_table,
|
||||
num_computed_tokens,
|
||||
self.commit_lens,
|
||||
self.final_state_indices,
|
||||
self.boundary_state_indices,
|
||||
self.boundary_recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
mamba_block_size or 1,
|
||||
block_table.shape[1] if block_table is not None else 1,
|
||||
num_accepted_tokens.stride(0),
|
||||
request_indices.stride(0) if request_indices is not None else 0,
|
||||
state_indices.stride(0),
|
||||
query_start_loc.stride(0),
|
||||
bt_stride[0],
|
||||
bt_stride[1],
|
||||
0 if num_computed_tokens is None else num_computed_tokens.stride(0),
|
||||
SPEC_QUERY_LEN=self.spec_query_len,
|
||||
num_warps=1,
|
||||
)
|
||||
num_layers = len(self.checkpoints)
|
||||
if commit_conv:
|
||||
conv_ref = self.conv_states[0]
|
||||
conv_dim = conv_ref.shape[1]
|
||||
compact_conv_state_kernel[(triton.cdiv(conv_dim, 256), batch, num_layers)](
|
||||
conv_ref,
|
||||
self.conv_state_base_addrs,
|
||||
self.conv_state_block_strides,
|
||||
self.conv_state_dim_strides,
|
||||
self.conv_state_token_strides,
|
||||
state_indices,
|
||||
self.commit_lens,
|
||||
self.final_state_indices,
|
||||
self.boundary_state_indices,
|
||||
self.boundary_recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
conv_dim,
|
||||
self.conv_history_len,
|
||||
state_indices.stride(0),
|
||||
BLOCK_D=256,
|
||||
BLOCK_HISTORY=triton.next_power_of_2(self.conv_history_len),
|
||||
ALIGN_MODE=block_table is not None,
|
||||
num_warps=4,
|
||||
)
|
||||
ref = self.checkpoints[0]
|
||||
_, HV, V, K = ref.shape
|
||||
BK = triton.next_power_of_2(K)
|
||||
BV = min(triton.next_power_of_2(V), 32)
|
||||
rr = self.replays[0]
|
||||
_commit_gdn_state_kernel[(triton.cdiv(V, BV), batch, num_layers * HV)](
|
||||
ref,
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
rr,
|
||||
self.replay_base_addrs,
|
||||
self.replay_block_strides,
|
||||
state_indices,
|
||||
self.commit_lens,
|
||||
self.final_state_indices,
|
||||
self.boundary_state_indices,
|
||||
self.boundary_recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
ref.stride(1),
|
||||
ref.stride(2),
|
||||
ref.stride(3),
|
||||
rr.stride(1),
|
||||
rr.stride(2),
|
||||
rr.stride(3),
|
||||
state_indices.stride(0),
|
||||
K=K,
|
||||
V=V,
|
||||
BK=BK,
|
||||
BV=BV,
|
||||
NUM_HEADS=HV,
|
||||
ALIGN_MODE=block_table is not None,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["GDNRecoverSSMCommitContext", "gdn_recoverssm_verify"]
|
||||
Reference in New Issue
Block a user