[Spec Decode][MRV2] RecoverSSM for Qwen GDN and PLE short conv (PR #58863)
Intake of open upstream PR vllm-project/vllm#58863. Active only with VLLM_USE_V2_MODEL_RUNNER=1; inert otherwise. Speculative decoding on Gated DeltaNet snapshotted the full recurrent state after every draft position and reserved one Mamba block per draft position. On a 24 GB card that is reserved VRAM we never use, plus per-step copy traffic. RecoverSSM, the protocol Kimi-K3 KDA already uses, replaces this: verify runs from a read-only checkpoint and stores a small per-token replay record [correction (V), normalized key (K), gate (1)] in fp32; commit replays only the accepted tokens and writes the recurrent state once, plus the block-boundary state in align mode. The PLE dilated short conv compacts its accepted window with the same plan. Conflict in vllm/models/qwen4_exp/nvidia/mtp.py: our get_top_tokens() work put LocalArgmaxMixin on Qwen4ExpMTP, the PR adds SupportsReplaySSM to the same base list. Kept both. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -105,7 +105,7 @@ def test_kda_recoverssm_config_state_layout():
|
||||
cache_config=SimpleNamespace(
|
||||
mamba_cache_dtype="auto",
|
||||
mamba_ssm_cache_dtype="auto",
|
||||
use_kda_recoverssm=True,
|
||||
use_recoverssm=True,
|
||||
),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=2),
|
||||
|
||||
@@ -120,7 +120,7 @@ def _make_builder(
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
vllm_config.cache_config.use_replayssm = use_recoverssm
|
||||
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
|
||||
vllm_config.cache_config.use_recoverssm = use_recoverssm
|
||||
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
|
||||
builder = builder_cls(
|
||||
kv_cache_spec=MambaSpec(
|
||||
@@ -162,7 +162,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
||||
cache_config=SimpleNamespace(
|
||||
mamba_cache_dtype="auto",
|
||||
mamba_ssm_cache_dtype="auto",
|
||||
use_kda_recoverssm=True,
|
||||
use_recoverssm=True,
|
||||
),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=2),
|
||||
@@ -195,7 +195,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
||||
),
|
||||
cache_config=SimpleNamespace(
|
||||
mamba_cache_mode="align",
|
||||
use_kda_recoverssm=True,
|
||||
use_recoverssm=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Align-mode RecoverSSM state columns at recurrent block boundaries.
|
||||
|
||||
A speculative window whose accepted tokens end exactly on a recurrent block
|
||||
boundary must keep its state in the block holding the last committed token.
|
||||
The following block can still be unallocated, so selecting it would drop the
|
||||
commit while the bookkeeping recorded the state as advanced.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import _prepare_commit_plan_kernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.worker.gpu.model_states.recoverssm import (
|
||||
_postprocess_recoverssm_align_kernel,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="The RecoverSSM kernels require a CUDA-alike device.",
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 2048
|
||||
SPEC_QUERY_LEN = 4
|
||||
BLOCKS = [100, 101, 102, 103, 104, 105]
|
||||
|
||||
# (computed before the step, committed tokens, allocated block columns,
|
||||
# final column, boundary column or None)
|
||||
CASES = [
|
||||
# 8,188 + 4 ends exactly on the 8,192 boundary; column 4 is unallocated.
|
||||
(8188, 4, 4, 3, 3),
|
||||
# The window crosses the boundary: state moves on, checkpoint stays behind.
|
||||
(8190, 4, 5, 4, 3),
|
||||
# The window stays inside column 3.
|
||||
(8180, 4, 4, 3, None),
|
||||
# The first token after a boundary belongs to the new column.
|
||||
(8192, 1, 5, 4, None),
|
||||
# Single-token commit ending on the first boundary.
|
||||
(2047, 1, 1, 0, 0),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
|
||||
)
|
||||
def test_commit_plan_keeps_boundary_state_in_last_computed_block(
|
||||
num_computed, commit_len, allocated, final_col, boundary_col
|
||||
):
|
||||
device = current_platform.device_type
|
||||
columns = [
|
||||
block if col < allocated else NULL_BLOCK_ID for col, block in enumerate(BLOCKS)
|
||||
]
|
||||
block_table = torch.tensor([columns], dtype=torch.int32, device=device)
|
||||
num_accepted = torch.tensor([commit_len], dtype=torch.int32, device=device)
|
||||
state_indices = torch.tensor(
|
||||
[BLOCKS[(num_computed - 1) // BLOCK_SIZE]], dtype=torch.int32, device=device
|
||||
)
|
||||
query_start_loc = torch.tensor(
|
||||
[0, SPEC_QUERY_LEN], dtype=torch.int32, device=device
|
||||
)
|
||||
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
|
||||
commit_lens, final_indices, boundary_indices, recovery_lens = (
|
||||
torch.full((1,), -7, dtype=torch.int32, device=device) for _ in range(4)
|
||||
)
|
||||
|
||||
_prepare_commit_plan_kernel[(1,)](
|
||||
num_accepted,
|
||||
None,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
block_table,
|
||||
computed,
|
||||
commit_lens,
|
||||
final_indices,
|
||||
boundary_indices,
|
||||
recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
BLOCK_SIZE,
|
||||
block_table.shape[1],
|
||||
num_accepted.stride(0),
|
||||
0,
|
||||
state_indices.stride(0),
|
||||
query_start_loc.stride(0),
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
computed.stride(0),
|
||||
SPEC_QUERY_LEN=SPEC_QUERY_LEN,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
assert commit_lens.item() == commit_len
|
||||
assert final_indices.item() == BLOCKS[final_col]
|
||||
expected_boundary = NULL_BLOCK_ID if boundary_col is None else BLOCKS[boundary_col]
|
||||
assert boundary_indices.item() == expected_boundary
|
||||
if boundary_col is not None:
|
||||
next_boundary = (num_computed // BLOCK_SIZE + 1) * BLOCK_SIZE
|
||||
assert recovery_lens.item() == next_boundary - num_computed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
|
||||
)
|
||||
def test_postprocess_records_the_committed_state_column(
|
||||
num_computed, commit_len, allocated, final_col, boundary_col
|
||||
):
|
||||
device = current_platform.device_type
|
||||
idx_mapping = torch.tensor([2], dtype=torch.int32, device=device)
|
||||
num_sampled = torch.tensor([commit_len], dtype=torch.int32, device=device)
|
||||
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
|
||||
state_idx = torch.full((4,), -7, dtype=torch.int32, device=device)
|
||||
num_accepted = torch.full((4,), -7, dtype=torch.int32, device=device)
|
||||
|
||||
_postprocess_recoverssm_align_kernel[(1,)](
|
||||
idx_mapping,
|
||||
num_sampled,
|
||||
None,
|
||||
computed,
|
||||
state_idx,
|
||||
num_accepted,
|
||||
MAMBA_BLOCK_SIZE=BLOCK_SIZE,
|
||||
BLOCK_TABLE_WIDTH=len(BLOCKS),
|
||||
)
|
||||
|
||||
assert state_idx[2].item() == final_col
|
||||
assert num_accepted[2].item() == 1
|
||||
assert state_idx[[0, 1, 3]].tolist() == [-7, -7, -7]
|
||||
Reference in New Issue
Block a user