[Spec Decode][MRV2] RecoverSSM for Qwen GDN and PLE short conv (PR #58863)

Intake of open upstream PR vllm-project/vllm#58863. Active only with
VLLM_USE_V2_MODEL_RUNNER=1; inert otherwise.

Speculative decoding on Gated DeltaNet snapshotted the full recurrent
state after every draft position and reserved one Mamba block per draft
position. On a 24 GB card that is reserved VRAM we never use, plus
per-step copy traffic.

RecoverSSM, the protocol Kimi-K3 KDA already uses, replaces this:
verify runs from a read-only checkpoint and stores a small per-token
replay record [correction (V), normalized key (K), gate (1)] in fp32;
commit replays only the accepted tokens and writes the recurrent state
once, plus the block-boundary state in align mode. The PLE dilated
short conv compacts its accepted window with the same plan.

Conflict in vllm/models/qwen4_exp/nvidia/mtp.py: our get_top_tokens()
work put LocalArgmaxMixin on Qwen4ExpMTP, the PR adds SupportsReplaySSM
to the same base list. Kept both.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-29 13:39:09 +07:00
co-authored by Hinata
31 changed files with 2777 additions and 226 deletions
+602
View File
@@ -0,0 +1,602 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GDN RecoverSSM: verify against the native fused kernel, commit against the native
per-token states, and the boundary checks (malformed metadata must raise ValueError
before any kernel reads or writes out of bounds)."""
from typing import Any, cast
import pytest
import torch
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
GDNRecoverSSMCommitContext,
gdn_recoverssm_verify,
)
from vllm.third_party.flash_linear_attention.ops.fused_sigmoid_gating import (
fused_sigmoid_gating_delta_rule_update,
)
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
H, HV, K, V, T = 16, 48, 128, 128, 4
QLENS = (4, 4, 3)
def _inputs(nb: int = 10) -> dict:
torch.manual_seed(0)
dev = "cuda"
tot = sum(QLENS)
qsl = torch.tensor([0, 4, 8, 11], dtype=torch.int32, device=dev)
return dict(
A_log=(torch.rand(HV, device=dev) * 2 - 1).float(),
a=torch.randn(tot, HV, device=dev, dtype=torch.bfloat16),
b=torch.randn(tot, HV, device=dev, dtype=torch.bfloat16),
dt_bias=(torch.randn(HV, device=dev) * 0.5).bfloat16(),
q=torch.randn(1, tot, H, K, device=dev, dtype=torch.bfloat16),
k=torch.randn(1, tot, H, K, device=dev, dtype=torch.bfloat16),
v=torch.randn(1, tot, HV, V, device=dev, dtype=torch.bfloat16),
checkpoint_state=(torch.randn(nb, HV, V, K, device=dev) * 0.05).float(),
replay_cache=torch.zeros(nb, HV, T, V + K + 1, device=dev),
query_start_loc=qsl,
state_indices=torch.tensor([2, 5, 7], dtype=torch.int32, device=dev),
spec_query_len=T,
)
def _verify(inp: dict, **override):
d = {**inp, **override}
return gdn_recoverssm_verify(
d["A_log"],
d["a"],
d["b"],
d["dt_bias"],
d["q"],
d["k"],
d["v"],
checkpoint_state=d["checkpoint_state"],
replay_cache=d["replay_cache"],
query_start_loc=d["query_start_loc"],
state_indices=d["state_indices"],
spec_query_len=d["spec_query_len"],
out=d.get("out"),
)
def _native(inp: dict):
return fused_sigmoid_gating_delta_rule_update(
A_log=inp["A_log"],
a=inp["a"],
b=inp["b"],
dt_bias=inp["dt_bias"],
q=inp["q"],
k=inp["k"],
v=inp["v"],
initial_state=inp["checkpoint_state"].clone(),
inplace_final_state=False,
cu_seqlens=inp["query_start_loc"].long(),
ssm_state_indices=inp["state_indices"].long(),
use_qk_l2norm_in_kernel=True,
)
def _context(inp: dict, ckpt: torch.Tensor) -> GDNRecoverSSMCommitContext:
nb = ckpt.shape[0]
conv = [torch.zeros(nb, 8, 3 + T - 1, device="cuda", dtype=torch.bfloat16)]
return GDNRecoverSSMCommitContext.from_tensors(
conv, [ckpt], [inp["replay_cache"]], spec_query_len=T, max_num_reqs=8
)
def test_verify_matches_native_and_keeps_the_checkpoint():
inp = _inputs()
before = inp["checkpoint_state"].clone()
out = _verify(inp)
ref, _ = _native(inp)
torch.testing.assert_close(
out.float(), ref.reshape(out.shape).float(), rtol=0, atol=0
)
assert torch.equal(inp["checkpoint_state"], before)
@pytest.mark.parametrize("accepted", [1, 2, 3, 4])
def test_commit_matches_native_per_token_state(accepted):
inp = _inputs()
_verify(inp)
_, states = _native(inp)
ckpt = inp["checkpoint_state"].clone()
ctx = _context(inp, ckpt)
acc = torch.tensor(
[min(accepted, q) for q in QLENS], dtype=torch.int32, device="cuda"
)
ctx.commit(acc, inp["state_indices"], inp["query_start_loc"])
for i, blk in enumerate(inp["state_indices"].tolist()):
tok = int(inp["query_start_loc"][i]) + int(acc[i]) - 1
torch.testing.assert_close(ckpt[blk], states[tok], rtol=1e-5, atol=1e-6)
others = [b for b in range(ckpt.shape[0]) if b not in inp["state_indices"].tolist()]
assert torch.equal(ckpt[others], inp["checkpoint_state"][others])
@pytest.mark.parametrize(
"case", ["short_qsl", "capacity", "hv_lt_h", "a_log", "out", "replay", "device"]
)
def test_verify_rejects_malformed_inputs(case):
inp = _inputs()
over, match = {
"short_qsl": (
dict(query_start_loc=inp["query_start_loc"][:-1]),
"query metadata",
),
"capacity": (
dict(
state_indices=inp["state_indices"][:2],
query_start_loc=inp["query_start_loc"][:3],
),
"activation capacity",
),
"hv_lt_h": (
dict(
v=inp["v"][:, :, :8].contiguous(),
a=inp["a"][:, :8].contiguous(),
b=inp["b"][:, :8].contiguous(),
),
"positive multiple",
),
"a_log": (dict(A_log=torch.zeros(HV + 1, device="cuda")), "A_log"),
"out": (
dict(
out=torch.empty(1, 11, HV, V + 1, device="cuda", dtype=torch.bfloat16)
),
"output shape",
),
"replay": (
dict(replay_cache=torch.zeros(10, HV, T, V + K, device="cuda")),
"replay buffer",
),
"device": (dict(A_log=inp["A_log"].cpu()), "same device"),
}[case]
with pytest.raises(ValueError, match=match):
_verify(inp, **over)
def test_commit_rejects_partial_or_short_metadata():
inp = _inputs()
ctx = _context(inp, inp["checkpoint_state"].clone())
acc = torch.ones(3, dtype=torch.int32, device="cuda")
bt = torch.zeros(3, 4, dtype=torch.int32, device="cuda")
with pytest.raises(ValueError, match="align metadata is incomplete"):
ctx.commit(
acc,
inp["state_indices"],
inp["query_start_loc"],
block_table=bt,
mamba_block_size=16,
)
with pytest.raises(ValueError, match="commit metadata"):
ctx.commit(acc, inp["state_indices"], inp["query_start_loc"][:-1])
with pytest.raises(ValueError, match="request mapping"):
ctx.commit(
acc,
inp["state_indices"],
inp["query_start_loc"],
request_indices=torch.zeros(2, dtype=torch.int32, device="cuda"),
)
def test_context_rejects_inconsistent_layers():
inp = _inputs()
ck = inp["checkpoint_state"]
conv = torch.zeros(ck.shape[0], 8, 3 + T - 1, device="cuda", dtype=torch.bfloat16)
with pytest.raises(ValueError, match="at least one layer"):
GDNRecoverSSMCommitContext.from_tensors(
[], [], [], spec_query_len=T, max_num_reqs=8
)
with pytest.raises(ValueError, match="differ"):
GDNRecoverSSMCommitContext.from_tensors(
[conv], [ck, ck], [inp["replay_cache"]], spec_query_len=T, max_num_reqs=8
)
with pytest.raises(ValueError, match="block count"):
GDNRecoverSSMCommitContext.from_tensors(
[conv[:5]], [ck], [inp["replay_cache"]], spec_query_len=T, max_num_reqs=8
)
# ------------------------------------------------------------------ review round 2:
# mapping, align, extreme gates
from vllm.v1.attention.backends.recoverssm_metadata import ( # noqa: E402
recoverssm_request_indices,
recoverssm_spec_rows,
)
# (A_log range, a scale, b offset): drive g = -exp(A_log) * softplus(a + dt_bias)
# and beta = sigmoid(b)
GATES = {
"typical": (0.0, 1.0, 0.0),
"g_near_0": (-12.0, 1.0, 0.0),
"g_very_negative": (2.5, 4.0, 0.0),
"beta_near_0": (0.0, 1.0, -12.0),
"beta_near_1": (0.0, 1.0, 12.0),
}
def _make(qlens, nb=16, seed=0, gate="typical"):
g = torch.Generator(device="cuda").manual_seed(seed)
rnd = lambda *s: torch.randn(*s, device="cuda", generator=g) # noqa: E731
a_log_shift, ascale, boff = GATES[gate]
tot = sum(qlens)
qsl = torch.tensor(
[0, *torch.tensor(qlens).cumsum(0).tolist()], dtype=torch.int32, device="cuda"
)
return dict(
A_log=(
torch.rand(HV, device="cuda", generator=g) * 2 - 1 + a_log_shift
).float(),
a=(rnd(tot, HV) * ascale).bfloat16(),
b=(rnd(tot, HV) + boff).bfloat16(),
dt_bias=(rnd(HV) * 0.5).bfloat16(),
q=rnd(1, tot, H, K).bfloat16(),
k=rnd(1, tot, H, K).bfloat16(),
v=rnd(1, tot, HV, V).bfloat16(),
checkpoint_state=(rnd(nb, HV, V, K) * 0.05).float(),
replay_cache=torch.zeros(nb, HV, T, V + K + 1, device="cuda"),
query_start_loc=qsl,
spec_query_len=T,
)
def test_spec_rows_mixed_batch():
# rows: 0 spec K=3 | 1 prefill | 2 draft-less decode | 3 spec K=1 | 4 padded
# zero-query | 5,6 beyond num_reqs
is_prefilling = torch.tensor([False, True, False, False, False, False, False])
qsl = torch.tensor([0, 4, 104, 105, 107, 107], dtype=torch.int32)
drafts_in = torch.tensor(
[3, -1, -1, 1, -1, 2, 0], dtype=torch.int32
) # 5, 6: garbage padding
drafts, rows = recoverssm_spec_rows(is_prefilling, qsl, drafts_in, num_reqs=5)
assert rows.tolist() == [0, 2, 3]
assert drafts.tolist() == [1, -1, 1, 1, -1, -1, -1]
idx = recoverssm_request_indices(rows, 3, torch.device("cuda"))
assert idx is not None and idx.tolist() == [0, 2, 3]
assert (
recoverssm_request_indices(torch.tensor([0, 1, 2]), 3, torch.device("cuda"))
is None
)
with pytest.raises(ValueError, match="spec rows classified"):
recoverssm_request_indices(rows, 2, torch.device("cuda"))
def test_commit_maps_noncontiguous_requests_in_align_mode():
"""Spec rows 0, 2, 3 of a 5-request batch; accepted counts, block table and
num_computed are indexed by request row. Every spec request must commit its own
state into its own block."""
qlens = [4, 1, 2] # the spec rows' windows (K=3, draft-less, K=1)
inp = _make(qlens, nb=40)
bs = 8
bt = torch.arange(1, 1 + 5 * 6, dtype=torch.int32, device="cuda").reshape(
5, 6
) # request row -> blocks
nc = torch.tensor(
[5, 0, 16, 9, 0], dtype=torch.int32, device="cuda"
) # per request row
req_rows = torch.tensor([0, 2, 3], dtype=torch.int32, device="cuda")
src = torch.stack([bt[r, int(nc[r]) // bs] for r in req_rows.tolist()]).contiguous()
inp["state_indices"] = src
_verify(inp)
_, states = _native(inp)
ckpt = inp["checkpoint_state"].clone()
ctx = _context(inp, ckpt)
acc = torch.tensor(
[3, 9, 1, 2, 9], dtype=torch.int32, device="cuda"
) # rows 1, 4 must be ignored
ctx.commit(
acc,
src,
inp["query_start_loc"],
request_indices=req_rows,
block_table=bt,
num_computed_tokens=nc,
mamba_block_size=bs,
)
for i, r in enumerate(req_rows.tolist()):
n = min(int(acc[r]), qlens[i])
c0 = int(nc[r])
final_blk = int(bt[r, max(c0 + n - 1, 0) // bs])
tok = int(inp["query_start_loc"][i]) + n - 1
torch.testing.assert_close(ckpt[final_blk], states[tok], rtol=1e-5, atol=1e-6)
# (num_computed, accepted, block size): boundary after 1 / after 2 / ending exactly on
# it / crossing and continuing / accepted = 1 without crossing / the full window
# crossing after 3. A window ending exactly on a boundary keeps its state in the block
# holding its last token: the next block may still be unallocated.
ALIGN_CASES = [
(3, 2, 4),
(2, 3, 4),
(0, 4, 4),
(6, 4, 4),
(1, 1, 4),
(5, 4, 8),
(7, 4, 8),
(4, 4, 8),
]
@pytest.mark.parametrize("nc,accepted,bs", ALIGN_CASES)
def test_align_boundary_and_final_state_match_native(nc, accepted, bs):
inp = _make([T], nb=12)
bt = torch.arange(2, 8, dtype=torch.int32, device="cuda").reshape(1, 6)
src = bt[0, nc // bs].reshape(1).contiguous()
inp["state_indices"] = src
_verify(inp)
_, states = _native(inp)
ckpt = inp["checkpoint_state"].clone()
ctx = _context(inp, ckpt)
ctx.commit(
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
src,
inp["query_start_loc"],
block_table=bt,
num_computed_tokens=torch.tensor([nc], dtype=torch.int32, device="cuda"),
mamba_block_size=bs,
)
next_boundary = (nc // bs + 1) * bs
final_blk = int(bt[0, max(nc + accepted - 1, 0) // bs])
if (
nc + accepted >= next_boundary
): # crosses: the boundary state goes into the source block
torch.testing.assert_close(
ckpt[int(src)], states[next_boundary - nc - 1], rtol=1e-5, atol=1e-6
)
torch.testing.assert_close(
ckpt[final_blk], states[accepted - 1], rtol=1e-5, atol=1e-6
)
@pytest.mark.parametrize("gate", list(GATES))
@pytest.mark.parametrize("seed", [0, 1, 2])
def test_commit_matches_native_under_extreme_gates(gate, seed):
inp = _make(list(QLENS), nb=10, seed=seed, gate=gate)
inp["state_indices"] = torch.tensor([2, 5, 7], dtype=torch.int32, device="cuda")
_verify(inp)
_, states = _native(inp)
for accepted in (1, 2, 3, 4):
ckpt = inp["checkpoint_state"].clone()
ctx = _context(inp, ckpt)
acc = torch.tensor(
[min(accepted, q) for q in QLENS], dtype=torch.int32, device="cuda"
)
ctx.commit(acc, inp["state_indices"], inp["query_start_loc"])
for i, blk in enumerate(inp["state_indices"].tolist()):
tok = int(inp["query_start_loc"][i]) + int(acc[i]) - 1
torch.testing.assert_close(ckpt[blk], states[tok], rtol=1e-5, atol=1e-6)
@pytest.mark.parametrize("batch", [1, 2, 4])
def test_verify_on_strided_views_writes_in_place(batch):
"""The layer passes views of the packed conv output (token stride = the packed
width) and the layer's output buffer; result and replay record must equal the
contiguous-copy path."""
torch.manual_seed(batch)
H, HV, K, V, SQ, NB = 16, 48, 128, 128, 4, 6
T = batch * SQ
qd, vd = H * K, HV * V
packed = torch.randn(T, 2 * qd + 2 * vd, device="cuda", dtype=torch.bfloat16)
mixed = packed[:, : 2 * qd + vd]
a = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
b = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
A_log = torch.randn(HV, device="cuda")
dt_bias = torch.randn(HV, device="cuda")
checkpoint = torch.randn(NB, HV, V, K, device="cuda") * 0.05
qsl = torch.arange(0, T + 1, SQ, device="cuda", dtype=torch.int32)
si = torch.arange(1, batch + 1, device="cuda", dtype=torch.int32)
def run(q, k, v, out=None):
replay = torch.zeros(NB, HV, SQ, V + K + 1, device="cuda")
o = gdn_recoverssm_verify(
A_log,
a,
b,
dt_bias,
q,
k,
v,
checkpoint_state=checkpoint,
replay_cache=replay,
query_start_loc=qsl,
state_indices=si,
spec_query_len=SQ,
out=out,
)
return o, replay
q, k, v = (
x.contiguous().view(1, T, -1, d)
for x, d in zip(torch.split(mixed, [qd, qd, vd], dim=-1), (K, K, V))
)
want_out, want_replay = run(q, k, v)
core = torch.zeros(T + 3, HV, V, device="cuda", dtype=torch.bfloat16)
views = (
mixed[:, :qd].view(1, T, -1, K),
mixed[:, qd : 2 * qd].view(1, T, -1, K),
mixed[:, 2 * qd :].view(1, T, -1, V),
)
got_out, got_replay = run(*views, out=core[:T].unsqueeze(0))
assert got_out.data_ptr() == core.data_ptr()
assert torch.equal(core[:T].unsqueeze(0), want_out)
assert torch.equal(got_replay, want_replay)
def test_verify_on_strided_views_with_variable_query_lengths():
"""Token-strided views with ragged windows (query lengths 4, 1, 3), as MTP
scheduling produces them."""
torch.manual_seed(7)
H, HV, K, V, SQ, NB = 16, 48, 128, 128, 4, 6
qsl_list = [0, 4, 5, 8]
T = qsl_list[-1]
qd, vd = H * K, HV * V
packed = torch.randn(T, 2 * qd + 2 * vd, device="cuda", dtype=torch.bfloat16)
mixed = packed[:, : 2 * qd + vd]
a = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
b = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
A_log = torch.randn(HV, device="cuda")
dt_bias = torch.randn(HV, device="cuda")
checkpoint = torch.randn(NB, HV, V, K, device="cuda") * 0.05
qsl = torch.tensor(qsl_list, device="cuda", dtype=torch.int32)
si = torch.tensor([1, 3, 5], device="cuda", dtype=torch.int32)
def run(q, k, v, out=None):
replay = torch.zeros(NB, HV, SQ, V + K + 1, device="cuda")
o = gdn_recoverssm_verify(
A_log,
a,
b,
dt_bias,
q,
k,
v,
checkpoint_state=checkpoint,
replay_cache=replay,
query_start_loc=qsl,
state_indices=si,
spec_query_len=SQ,
out=out,
)
return o, replay
q, k, v = (
x.contiguous().view(1, T, -1, d)
for x, d in zip(torch.split(mixed, [qd, qd, vd], dim=-1), (K, K, V))
)
want_out, want_replay = run(q, k, v)
core = torch.zeros(T + 2, HV, V, device="cuda", dtype=torch.bfloat16)
views = (
mixed[:, :qd].view(1, T, -1, K),
mixed[:, qd : 2 * qd].view(1, T, -1, K),
mixed[:, 2 * qd :].view(1, T, -1, V),
)
_, got_replay = run(*views, out=core[:T].unsqueeze(0))
assert torch.equal(core[:T].unsqueeze(0), want_out)
assert torch.equal(got_replay, want_replay)
def test_layer_verify_qkv_views_and_fallback():
"""The layer's guard: views for a [tokens, q|k|v] conv output with a contiguous
last dimension, the contiguous-copy fallback for any other layout; same values
either way."""
from types import SimpleNamespace
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
QwenGatedDeltaNetAttention as Layer,
)
H, HV, K, V, T = 4, 8, 16, 16, 5
layer = SimpleNamespace(
key_dim=H * K, value_dim=HV * V, tp_size=1, head_k_dim=K, head_v_dim=V
)
layer.rearrange_mixed_qkv = lambda x: Layer.rearrange_mixed_qkv(cast(Any, layer), x)
width = 2 * H * K + HV * V
packed = torch.randn(T, width + HV * V, dtype=torch.bfloat16) # q|k|v|z, like qkvz
strided = packed[:, :width]
q, k, v, is_view = Layer._recoverssm_verify_qkv(cast(Any, layer), strided)
assert is_view
assert q.data_ptr() == packed.data_ptr()
rq, rk, rv = Layer.rearrange_mixed_qkv(cast(Any, layer), strided)
assert torch.equal(q, rq) and torch.equal(k, rk) and torch.equal(v, rv)
# Column-major storage: last dimension not contiguous -> copies, same values.
col_major = strided.t().contiguous().t()
assert col_major.stride(-1) != 1
q2, k2, v2, is_view2 = Layer._recoverssm_verify_qkv(cast(Any, layer), col_major)
assert not is_view2
assert torch.equal(q2, rq) and torch.equal(k2, rk) and torch.equal(v2, rv)
# ------------------------------------------------------------- BF16 recurrent state
# RecoverSSM upcasts the BF16 checkpoint, carries the accepted window in FP32 and rounds
# once when it writes the committed state. The reference is the FP32 trajectory from the
# same (already BF16-rounded) checkpoint; the committed state must equal its BF16
# rounding up to one BF16 ulp, and be no further from it than the native path is.
BF16_ULP = 2**-7
def _bf16_case(qlens, nb=12, seed=0):
inp = _make(qlens, nb=nb, seed=seed)
inp["checkpoint_state"] = inp["checkpoint_state"].bfloat16()
ref = dict(inp)
ref["checkpoint_state"] = inp["checkpoint_state"].float()
return inp, ref
def test_verify_bf16_state_matches_native():
inp, _ = _bf16_case(list(QLENS))
inp["state_indices"] = torch.tensor([2, 5, 7], dtype=torch.int32, device="cuda")
before = inp["checkpoint_state"].clone()
out = _verify(inp)
ref, _ = _native(inp)
torch.testing.assert_close(
out.float(), ref.reshape(out.shape).float(), rtol=0, atol=0
)
assert torch.equal(inp["checkpoint_state"], before)
@pytest.mark.parametrize("accepted", [1, 2, 3, 4])
@pytest.mark.parametrize("seed", [0, 1])
def test_commit_bf16_state_rounds_once(accepted, seed):
inp, ref_inp = _bf16_case(list(QLENS), seed=seed)
idx = torch.tensor([2, 5, 7], dtype=torch.int32, device="cuda")
inp["state_indices"] = ref_inp["state_indices"] = idx
_verify(inp)
_, native_states = _native(inp)
_, ref_states = _native(ref_inp)
ckpt = inp["checkpoint_state"].clone()
_context(inp, ckpt).commit(
torch.tensor(
[min(accepted, q) for q in QLENS], dtype=torch.int32, device="cuda"
),
idx,
inp["query_start_loc"],
)
for i, blk in enumerate(idx.tolist()):
tok = int(inp["query_start_loc"][i]) + min(accepted, QLENS[i]) - 1
ref = ref_states[tok].float()
got = ckpt[blk].float()
torch.testing.assert_close(
got, ref.bfloat16().float(), rtol=BF16_ULP, atol=1e-5
)
err_rssm = (got - ref).abs().mean()
err_native = (native_states[tok].bfloat16().float() - ref).abs().mean()
assert err_rssm <= err_native * 1.05 + 1e-7, (err_rssm, err_native)
@pytest.mark.parametrize("nc,accepted,bs", ALIGN_CASES)
def test_align_boundary_bf16_state(nc, accepted, bs):
inp, ref_inp = _bf16_case([T])
bt = torch.arange(2, 8, dtype=torch.int32, device="cuda").reshape(1, 6)
src = bt[0, nc // bs].reshape(1).contiguous()
inp["state_indices"] = ref_inp["state_indices"] = src
_verify(inp)
_, ref_states = _native(ref_inp)
ckpt = inp["checkpoint_state"].clone()
_context(inp, ckpt).commit(
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
src,
inp["query_start_loc"],
block_table=bt,
num_computed_tokens=torch.tensor([nc], dtype=torch.int32, device="cuda"),
mamba_block_size=bs,
)
next_boundary = (nc // bs + 1) * bs
final_blk = int(bt[0, max(nc + accepted - 1, 0) // bs])
if nc + accepted >= next_boundary:
torch.testing.assert_close(
ckpt[int(src)].float(),
ref_states[next_boundary - nc - 1].float().bfloat16().float(),
rtol=BF16_ULP,
atol=1e-5,
)
torch.testing.assert_close(
ckpt[final_blk].float(),
ref_states[accepted - 1].float().bfloat16().float(),
rtol=BF16_ULP,
atol=1e-5,
)
+153
View File
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PLE short-conv RecoverSSM against the sequential reference.
Reference: the stock decode path fed the accepted tokens one at a time.
RecoverSSM: one spec-mode conv over the whole window from the checkpoint
(offset 0), then ``_PleConvCommit`` compacts the accepted suffix to the front.
The conv history is pure data movement, so the states must be bit-identical,
and the outputs of the accepted tokens must match as well.
"""
import pytest
import torch
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.models.qwen4_exp.nvidia.ops.ple import ple_conv
from vllm.v1.attention.backends.ple_recoverssm import _PleConvCommit
C, KSIZE, DIL, SQ, NB = 512, 4, 3, 4, 8
STATE_LEN = (KSIZE - 1) * DIL
WIDTH = STATE_LEN + SQ - 1
DEV = "cuda"
def _state():
if is_conv_state_dim_first():
raw = torch.zeros(NB, C, WIDTH, dtype=torch.bfloat16, device=DEV)
return raw, raw
raw = torch.zeros(NB, WIDTH, C, dtype=torch.bfloat16, device=DEV)
return raw, raw.transpose(-1, -2)
def _i32(xs):
return torch.tensor(xs, dtype=torch.int32, device=DEV)
def _setup(seed, blocks):
g = torch.Generator(device=DEV).manual_seed(seed)
w = torch.randn(C, KSIZE, generator=g, device=DEV).to(torch.bfloat16)
x = torch.randn(len(blocks) * SQ, C, generator=g, device=DEV).to(torch.bfloat16)
outer = torch.randn_like(x)
raw, view = _state()
for b in blocks:
view[b, :, :STATE_LEN] = torch.randn(C, STATE_LEN, generator=g, device=DEV).to(
torch.bfloat16
)
return w, x, outer, raw, view
def _sequential(w, x, outer, view, block, first, count):
"""Stock decode, one token at a time, on a copy of the checkpoint block."""
ref = view.clone()
res = torch.zeros(count, C, dtype=torch.bfloat16, device=DEV)
for i in range(count):
t = first + i
ple_conv(
inputs=x[t : t + 1],
residual=res[i : i + 1],
conv_state=ref,
conv_weights=w,
state_indices=_i32([block]),
outer_residual=outer[t : t + 1],
mode="decode",
dilation=DIL,
)
return ref[block, :, :STATE_LEN].clone(), res
def _spec(w, x, outer, view, blocks):
res = torch.zeros(len(blocks) * SQ, C, dtype=torch.bfloat16, device=DEV)
ple_conv(
inputs=x,
residual=res,
conv_state=view,
conv_weights=w,
state_indices=_i32(blocks),
outer_residual=outer,
mode="spec",
dilation=DIL,
query_start_loc=_i32(list(range(0, len(blocks) * SQ + 1, SQ))),
num_accepted_tokens=_i32([1] * len(blocks)),
spec_query_len=SQ,
)
return res
@pytest.mark.parametrize("accepted", [1, 2, SQ])
@pytest.mark.parametrize("seed", [0, 1])
def test_ple_commit_matches_sequential(accepted, seed):
block = 3
w, x, outer, raw, view = _setup(seed, [block])
want_state, want_out = _sequential(w, x, outer, view, block, 0, accepted)
history = torch.cat([view[block, :, :STATE_LEN], x[:accepted].T], dim=1)
got_out = _spec(w, x, outer, view, [block])
ctx = _PleConvCommit([raw], spec_query_len=SQ, max_num_reqs=NB)
ctx.commit(_i32([accepted]), _i32([block]), _i32([0, SQ]))
got_state = view[block, :, :STATE_LEN]
assert torch.equal(got_state, want_state)
assert torch.equal(got_state, history[:, -STATE_LEN:])
assert torch.equal(got_out[:accepted], want_out)
@pytest.mark.parametrize(
("num_computed", "accepted", "crosses"),
[(6, 3, True), (6, 2, True), (1, 2, False), (7, SQ, True)],
)
def test_ple_commit_align_boundary(num_computed, accepted, crosses):
block_size, src, nxt = 8, 2, 5
w, x, outer, raw, view = _setup(7, [src])
boundary_len = block_size - num_computed
want_final, _ = _sequential(w, x, outer, view, src, 0, accepted)
want_boundary, _ = _sequential(w, x, outer, view, src, 0, min(boundary_len, SQ))
_spec(w, x, outer, view, [src])
ctx = _PleConvCommit([raw], spec_query_len=SQ, max_num_reqs=NB)
ctx.commit(
_i32([accepted]),
_i32([src]),
_i32([0, SQ]),
block_table=_i32([[src, nxt]]),
num_computed_tokens=_i32([num_computed]),
mamba_block_size=block_size,
)
final_block = nxt if num_computed + accepted > block_size else src
assert torch.equal(view[final_block, :, :STATE_LEN], want_final)
if crosses:
assert torch.equal(view[src, :, :STATE_LEN], want_boundary)
def test_ple_commit_non_contiguous_request_rows():
blocks, accepted = [1, 6], [3, 1]
w, x, outer, raw, view = _setup(11, blocks)
want = [
_sequential(w, x, outer, view, b, i * SQ, a)[0]
for i, (b, a) in enumerate(zip(blocks, accepted))
]
_spec(w, x, outer, view, blocks)
# Spec rows 0 and 1 belong to requests 1 and 3 of a four-request batch.
counts = _i32([9, accepted[0], 9, accepted[1]])
ctx = _PleConvCommit([raw], spec_query_len=SQ, max_num_reqs=NB)
ctx.commit(
counts,
_i32(blocks),
_i32([0, SQ, 2 * SQ]),
request_indices=_i32([1, 3]),
)
for b, ref in zip(blocks, want):
assert torch.equal(view[b, :, :STATE_LEN], ref)