[K3] support recoverssm for K3 (#51855)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-08-17 19:34:14 +08:00
committed by GitHub
co-authored by OpenAI Codex Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f27ae25473
commit 70afdedc10
19 changed files with 2235 additions and 75 deletions
+342
View File
@@ -6,6 +6,8 @@ Compares chunk_kda against a naive recurrent reference (float32).
Uses torch.rand for q/k/v to match FLA's test pattern.
"""
from types import SimpleNamespace
import pytest
import torch
import torch.nn.functional as F
@@ -22,6 +24,12 @@ from vllm.models.kimi_k3.nvidia.kda import (
is_flashkda_supported,
is_fused_kda_decode_supported,
)
from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM
from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
KDARecoverSSMCommitContext,
kda_recoverssm_verify,
)
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda,
chunk_kda_with_fused_gate,
@@ -48,6 +56,38 @@ PACKED_DECODE_IMPLS = {
}
def test_kda_recoverssm_config_state_layout():
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
dtype=torch.bfloat16,
hf_config=SimpleNamespace(
linear_attn_config={
"num_heads": 4,
"head_dim": 32,
"short_conv_kernel_size": 4,
}
),
),
cache_config=SimpleNamespace(
mamba_cache_dtype="auto",
use_kda_recoverssm=True,
),
parallel_config=SimpleNamespace(tensor_parallel_size=1),
speculative_config=SimpleNamespace(num_speculative_tokens=2),
)
assert KimiLinearForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == (
torch.bfloat16,
torch.float32,
torch.float32,
torch.bfloat16,
)
assert KimiLinearForCausalLM.get_mamba_state_shape_from_config(vllm_config)[2:] == (
(4, 3, 32),
(4, 3, 64),
)
@torch.inference_mode()
def test_gather_initial_states_correctness():
row_size = 8 * 128 * 128
@@ -535,6 +575,308 @@ def test_kda_spec_decode_correctness(
assert torch.isnan(output_storage[..., H * D :]).all()
@pytest.mark.parametrize(
(
"conv_state_dim_first",
"use_request_indices",
"lower_bound",
"align_mode",
),
[
pytest.param(False, False, None, False, id="baseline"),
pytest.param(True, True, -5.0, True, id="all-features"),
pytest.param(False, True, -5.0, False, id="request-indexed"),
pytest.param(True, False, None, True, id="aligned"),
],
)
@torch.inference_mode()
def test_kda_recoverssm_verify_and_group_commit(
monkeypatch: pytest.MonkeyPatch,
lower_bound: float | None,
use_request_indices: bool,
conv_state_dim_first: bool,
align_mode: bool,
):
monkeypatch.setattr(
recoverssm_ops,
"is_conv_state_dim_first",
lambda: conv_state_dim_first,
)
num_layers, num_seqs, query_len = 2, 2, 8
num_blocks, num_heads, dim = (7 if align_mode else 3), 4, 128
total_tokens = num_seqs * query_len
torch.manual_seed(20260808)
q, k, v, raw_g = [
torch.randn(
1,
total_tokens,
num_heads,
dim,
dtype=torch.bfloat16,
device=DEVICE,
)
for _ in range(4)
]
raw_beta = torch.randn(
1,
total_tokens,
num_heads,
dtype=torch.bfloat16,
device=DEVICE,
)
query_start_loc = torch.arange(
0,
total_tokens + 1,
query_len,
dtype=torch.int32,
device=DEVICE,
)
state_indices = torch.tensor(
[5, 6] if align_mode else [1, 2], dtype=torch.int32, device=DEVICE
)
accepted = [2, 8]
if use_request_indices:
global_num_accepted = torch.tensor(
[0, accepted[0], 0, accepted[1]],
dtype=torch.int32,
device=DEVICE,
)
request_indices = torch.tensor([1, 3], dtype=torch.int32, device=DEVICE)
else:
global_num_accepted = torch.tensor(accepted, dtype=torch.int32, device=DEVICE)
request_indices = None
block_table = None
num_computed_tokens = None
mamba_block_size = None
if align_mode:
batch_size = 4 if use_request_indices else num_seqs
block_table = torch.full((batch_size, 2), -1, dtype=torch.int32, device=DEVICE)
rows = (
request_indices
if request_indices is not None
else torch.arange(num_seqs, device=DEVICE)
)
block_table[rows] = torch.tensor(
[[1, 5], [2, 6]],
dtype=torch.int32,
device=DEVICE,
)
num_computed_tokens = torch.zeros(batch_size, dtype=torch.int32, device=DEVICE)
num_computed_tokens[rows] = 4
mamba_block_size = 8
layers = []
expected_outputs = []
expected_states = []
initial_states = []
initial_conv_states = []
history_len, conv_dim = 3, 12
for layer_idx in range(num_layers):
A_log = (
0.2 * torch.randn(num_heads, dtype=torch.float32, device=DEVICE)
+ layer_idx * 0.03
).contiguous()
dt_bias = (
0.1 * torch.randn(num_heads, dim, dtype=torch.float32, device=DEVICE)
).contiguous()
checkpoint = 0.01 * torch.randn(
num_blocks,
num_heads,
dim,
dim,
dtype=torch.float32,
device=DEVICE,
)
conv_shape = (
(num_blocks, conv_dim, history_len + query_len - 1)
if conv_state_dim_first
else (num_blocks, history_len + query_len - 1, conv_dim)
)
conv_state = torch.randn(conv_shape, dtype=torch.bfloat16, device=DEVICE)
correction_cache = torch.empty(
num_blocks,
num_heads,
query_len,
dim,
dtype=torch.float32,
device=DEVICE,
)
kg_cache = torch.empty(
num_blocks,
num_heads,
query_len,
2 * dim,
dtype=torch.bfloat16,
device=DEVICE,
)
layer = SimpleNamespace(
kv_cache=(
conv_state,
checkpoint,
correction_cache,
kg_cache,
),
A_log=A_log,
dt_bias=dt_bias,
local_num_heads=num_heads,
head_dim=dim,
gate_lower_bound=lower_bound,
)
layers.append(layer)
initial_states.append(checkpoint.clone())
initial_conv_states.append(conv_state.clone())
actual_output = kda_recoverssm_verify(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
checkpoint_state=checkpoint,
correction_cache=correction_cache,
kg_cache=kg_cache,
query_start_loc=query_start_loc,
state_indices=state_indices,
spec_query_len=query_len,
)
normalized_q = q.float() * torch.rsqrt(
q.float().square().sum(dim=-1, keepdim=True) + 1e-6
)
normalized_k = k.float() * torch.rsqrt(
k.float().square().sum(dim=-1, keepdim=True) + 1e-6
)
gate_input = raw_g.float() + dt_bias.view(1, 1, num_heads, dim)
if lower_bound is None:
gate = -A_log.exp().view(1, 1, num_heads, 1) * F.softplus(gate_input)
else:
gate = lower_bound * torch.sigmoid(
A_log.exp().view(1, 1, num_heads, 1) * gate_input
)
beta = raw_beta.float().sigmoid()
reference_output = []
committed_states = checkpoint.clone()
for seq_idx, commit_len in enumerate(accepted):
start = seq_idx * query_len
end = start + query_len
output, _ = naive_recurrent_kda(
normalized_q[:, start:end],
normalized_k[:, start:end],
v[:, start:end],
gate[:, start:end],
beta[:, start:end],
initial_state=checkpoint[state_indices[seq_idx]].transpose(-1, -2),
)
reference_output.append(output)
_, committed_state = naive_recurrent_kda(
normalized_q[:, start : start + commit_len],
normalized_k[:, start : start + commit_len],
v[:, start : start + commit_len],
gate[:, start : start + commit_len],
beta[:, start : start + commit_len],
initial_state=checkpoint[state_indices[seq_idx]].transpose(-1, -2),
output_final_state=True,
)
assert committed_state is not None
final_block = state_indices[seq_idx]
if align_mode:
assert block_table is not None
row = request_indices[seq_idx] if use_request_indices else seq_idx
final_block = block_table[row, (4 + commit_len) // 8]
committed_states[final_block] = committed_state.transpose(-1, -2)
if align_mode and 4 + commit_len >= 8:
_, boundary_state = naive_recurrent_kda(
normalized_q[:, start : start + 4],
normalized_k[:, start : start + 4],
v[:, start : start + 4],
gate[:, start : start + 4],
beta[:, start : start + 4],
initial_state=checkpoint[state_indices[seq_idx]].transpose(-1, -2),
output_final_state=True,
)
assert boundary_state is not None
assert block_table is not None
row = request_indices[seq_idx] if use_request_indices else seq_idx
committed_states[block_table[row, 0]] = boundary_state.transpose(-1, -2)
expected_outputs.append(torch.cat(reference_output, dim=1))
expected_states.append(committed_states)
torch.testing.assert_close(checkpoint, initial_states[-1])
torch.testing.assert_close(
actual_output,
expected_outputs[-1],
atol=3e-2,
rtol=3e-2,
)
context = KDARecoverSSMCommitContext.create(
layers,
spec_query_len=query_len,
max_num_reqs=global_num_accepted.shape[0],
)
context.commit(
global_num_accepted,
state_indices,
query_start_loc,
request_indices=request_indices,
block_table=block_table,
num_computed_tokens=num_computed_tokens,
mamba_block_size=mamba_block_size,
)
for layer_idx, layer in enumerate(layers):
torch.testing.assert_close(
layer.kv_cache[1],
expected_states[layer_idx],
atol=3e-3,
rtol=3e-3,
)
for seq_idx, commit_len in enumerate(accepted):
block = state_indices[seq_idx]
if align_mode:
assert block_table is not None
row = request_indices[seq_idx] if use_request_indices else seq_idx
block = block_table[row, (4 + commit_len) // 8]
source_block = state_indices[seq_idx] if align_mode else block
if conv_state_dim_first:
actual_conv = layer.kv_cache[0][block, :, :history_len]
expected_conv = initial_conv_states[layer_idx][
source_block,
:,
commit_len - 1 : commit_len - 1 + history_len,
]
else:
actual_conv = layer.kv_cache[0][block, :history_len]
expected_conv = initial_conv_states[layer_idx][
source_block,
commit_len - 1 : commit_len - 1 + history_len,
]
torch.testing.assert_close(actual_conv, expected_conv)
if align_mode and 4 + commit_len >= 8:
assert block_table is not None
boundary_block = block_table[row, 0]
if conv_state_dim_first:
actual_boundary_conv = layer.kv_cache[0][
boundary_block, :, :history_len
]
expected_boundary_conv = initial_conv_states[layer_idx][
state_indices[seq_idx], :, 3 : 3 + history_len
]
else:
actual_boundary_conv = layer.kv_cache[0][
boundary_block, :history_len
]
expected_boundary_conv = initial_conv_states[layer_idx][
state_indices[seq_idx], 3 : 3 + history_len
]
torch.testing.assert_close(actual_boundary_conv, expected_boundary_conv)
@pytest.mark.parametrize(
("num_heads", "num_seqs", "lower_bound", "fuse_output_norm"),
[
+145 -4
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import fields
from unittest.mock import Mock
import pytest
import torch
@@ -26,6 +27,9 @@ from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.recoverssm_metadata import (
RecoverSSMPostprocessMetadata,
)
from vllm.v1.attention.backends.utils import (
NULL_BLOCK_ID,
mamba_get_block_table_tensor,
@@ -44,8 +48,12 @@ PRUNED_METADATA_FIELDS = {
}
def _assert_matches_shared_gdn(reference, actual: KimiK3KDAMetadata):
for field in fields(KimiK3KDAMetadata):
def _assert_matches_shared_gdn(
reference: GDNAttentionMetadata, actual: KimiK3KDAMetadata
):
assert actual.recoverssm_commit is None
assert actual.recoverssm_context is None
for field in fields(GDNAttentionMetadata):
actual_value = getattr(actual, field.name)
expected_value = getattr(reference, field.name)
if field.name in PRUNED_METADATA_FIELDS:
@@ -78,6 +86,7 @@ def _make_builder(
full_cuda_graph: bool,
device: torch.device = DEVICE,
mamba_cache_mode: str = "none",
use_recoverssm: bool = False,
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
@@ -92,17 +101,24 @@ def _make_builder(
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
return builder_cls(
vllm_config.cache_config.use_replayssm = use_recoverssm
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
builder = builder_cls(
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=num_speculative_tokens,
mamba_cache_mode=mamba_cache_mode,
num_speculative_blocks=(0 if use_recoverssm else num_speculative_tokens),
),
layer_names=["layer.0"],
vllm_config=vllm_config,
device=device,
)
if use_recoverssm:
assert isinstance(builder, KimiK3KDAMetadataBuilder)
builder.recoverssm_context = Mock()
return builder
@pytest.mark.parametrize(
@@ -244,6 +260,99 @@ def test_mixed_regular_and_spec_decode_excludes_request_padding():
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
def test_recoverssm_spec_uses_one_state_slot_and_current_window(
mamba_cache_mode: str,
):
if mamba_cache_mode == "align" and not torch.cuda.is_available():
pytest.skip("align metadata construction requires CUDA")
device = torch.device("cuda") if mamba_cache_mode == "align" else DEVICE
batch = BatchSpec(seq_lens=[100, 65, 20], query_lens=[1, 1, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device
).replace(is_prefilling=torch.tensor([True, True, False]))
builder = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
device=device,
mamba_cache_mode=mamba_cache_mode,
use_recoverssm=True,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
context = builder.recoverssm_context
assert context is not None
actual = builder.build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
num_accepted_tokens=torch.tensor([3, 2, 2], dtype=torch.int32, device=device),
)
assert actual.spec_state_indices_tensor is not None
assert actual.spec_state_indices_tensor.shape == (1, 1)
torch.testing.assert_close(
actual.num_accepted_tokens,
torch.ones(1, dtype=torch.int32, device=device),
)
commit_metadata = actual.recoverssm_commit
assert commit_metadata is not None
torch.testing.assert_close(
commit_metadata.request_indices,
torch.tensor([2], dtype=torch.int32, device=device),
)
assert actual.recoverssm_context is context
num_accepted_tokens = torch.tensor([3, 2, 1], dtype=torch.int32, device=device)
postprocess = actual.commit_recoverssm_state(num_accepted_tokens)
if mamba_cache_mode == "none":
assert commit_metadata.align is None
assert postprocess is None
else:
assert isinstance(postprocess, RecoverSSMPostprocessMetadata)
assert postprocess.num_spec_decodes == 1
assert postprocess.request_indices is commit_metadata.request_indices
assert postprocess.block_table is common_attn_metadata.block_table_tensor
assert (
postprocess.num_computed_tokens
is common_attn_metadata.compute_num_computed_tokens()
)
assert postprocess.block_size == BLOCK_SIZE
args = context.commit.call_args.args
assert args[0] is num_accepted_tokens
torch.testing.assert_close(args[1], commit_metadata.state_indices[:, 0])
torch.testing.assert_close(args[2], commit_metadata.query_start_loc)
def test_recoverssm_distinguishes_draftless_decode_from_one_token_prefill():
batch = BatchSpec(seq_lens=[40, 30], query_lens=[1, 1])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([False, True]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
use_recoverssm=True,
).build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.full((2,), -1, dtype=torch.int32),
num_accepted_tokens=torch.ones(2, dtype=torch.int32),
)
assert actual.num_spec_decodes == 1
assert actual.num_decodes == 0
assert actual.num_prefills == 1
assert actual.spec_state_indices_tensor is not None
assert actual.spec_state_indices_tensor.shape == (1, 1)
torch.testing.assert_close(
actual.spec_query_start_loc,
torch.tensor([0, 1], dtype=torch.int32),
)
@pytest.mark.parametrize(
("seq_len", "expected_has_initial_state"),
[
@@ -307,6 +416,38 @@ def test_kimi_k3_kda_cudagraph_capture_matches_shared_gdn():
_assert_matches_shared_gdn(reference, actual)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_recoverssm_spec_cudagraph_stages_one_checkpoint_per_request():
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device
).replace(is_prefilling=torch.tensor([False, False]))
builder = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=True,
device=device,
use_recoverssm=True,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
assert builder.spec_state_indices_tensor.shape == (
builder.vllm_config.scheduler_config.max_num_seqs,
1,
)
actual = builder.build_for_cudagraph_capture(common_attn_metadata)
assert actual.spec_state_indices_tensor is not None
assert actual.spec_state_indices_tensor.shape == (batch.batch_size, 1)
assert actual.num_accepted_tokens is not None
torch.testing.assert_close(
actual.num_accepted_tokens,
torch.ones(batch.batch_size, dtype=torch.int32, device=device),
)
assert actual.recoverssm_commit is not None
assert actual.recoverssm_commit.request_indices is None
def test_kimi_k3_kda_backend_uses_private_metadata_builder():
assert KimiK3KDAAttentionBackend.get_builder_cls() is KimiK3KDAMetadataBuilder
assert KimiK3KDAAttentionBackend.is_ssm()
+3 -1
View File
@@ -143,8 +143,10 @@ def test_registry_is_pp(model_arch, is_pp, init_cuda):
@pytest.mark.parametrize(
"model_arch,supported",
[
# ReplaySSM is opt-in per model; only Nemotron-H sets the flag today.
# ReplaySSM is opt-in per model.
("NemotronHForCausalLM", True),
("KimiLinearForCausalLM", not current_platform.is_rocm()),
("KimiK3ForConditionalGeneration", not current_platform.is_rocm()),
("Mamba2ForCausalLM", False),
("Zamba2ForCausalLM", False),
],