[K3] support recoverssm for K3 (#51855)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f27ae25473
commit
70afdedc10
@@ -6,6 +6,8 @@ Compares chunk_kda against a naive recurrent reference (float32).
|
||||
Uses torch.rand for q/k/v to match FLA's test pattern.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -22,6 +24,12 @@ from vllm.models.kimi_k3.nvidia.kda import (
|
||||
is_flashkda_supported,
|
||||
is_fused_kda_decode_supported,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM
|
||||
from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
|
||||
KDARecoverSSMCommitContext,
|
||||
kda_recoverssm_verify,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda,
|
||||
chunk_kda_with_fused_gate,
|
||||
@@ -48,6 +56,38 @@ PACKED_DECODE_IMPLS = {
|
||||
}
|
||||
|
||||
|
||||
def test_kda_recoverssm_config_state_layout():
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.bfloat16,
|
||||
hf_config=SimpleNamespace(
|
||||
linear_attn_config={
|
||||
"num_heads": 4,
|
||||
"head_dim": 32,
|
||||
"short_conv_kernel_size": 4,
|
||||
}
|
||||
),
|
||||
),
|
||||
cache_config=SimpleNamespace(
|
||||
mamba_cache_dtype="auto",
|
||||
use_kda_recoverssm=True,
|
||||
),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=2),
|
||||
)
|
||||
|
||||
assert KimiLinearForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == (
|
||||
torch.bfloat16,
|
||||
torch.float32,
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
)
|
||||
assert KimiLinearForCausalLM.get_mamba_state_shape_from_config(vllm_config)[2:] == (
|
||||
(4, 3, 32),
|
||||
(4, 3, 64),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gather_initial_states_correctness():
|
||||
row_size = 8 * 128 * 128
|
||||
@@ -535,6 +575,308 @@ def test_kda_spec_decode_correctness(
|
||||
assert torch.isnan(output_storage[..., H * D :]).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"conv_state_dim_first",
|
||||
"use_request_indices",
|
||||
"lower_bound",
|
||||
"align_mode",
|
||||
),
|
||||
[
|
||||
pytest.param(False, False, None, False, id="baseline"),
|
||||
pytest.param(True, True, -5.0, True, id="all-features"),
|
||||
pytest.param(False, True, -5.0, False, id="request-indexed"),
|
||||
pytest.param(True, False, None, True, id="aligned"),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_kda_recoverssm_verify_and_group_commit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
lower_bound: float | None,
|
||||
use_request_indices: bool,
|
||||
conv_state_dim_first: bool,
|
||||
align_mode: bool,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
recoverssm_ops,
|
||||
"is_conv_state_dim_first",
|
||||
lambda: conv_state_dim_first,
|
||||
)
|
||||
num_layers, num_seqs, query_len = 2, 2, 8
|
||||
num_blocks, num_heads, dim = (7 if align_mode else 3), 4, 128
|
||||
total_tokens = num_seqs * query_len
|
||||
torch.manual_seed(20260808)
|
||||
|
||||
q, k, v, raw_g = [
|
||||
torch.randn(
|
||||
1,
|
||||
total_tokens,
|
||||
num_heads,
|
||||
dim,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
for _ in range(4)
|
||||
]
|
||||
raw_beta = torch.randn(
|
||||
1,
|
||||
total_tokens,
|
||||
num_heads,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
query_start_loc = torch.arange(
|
||||
0,
|
||||
total_tokens + 1,
|
||||
query_len,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_indices = torch.tensor(
|
||||
[5, 6] if align_mode else [1, 2], dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
accepted = [2, 8]
|
||||
if use_request_indices:
|
||||
global_num_accepted = torch.tensor(
|
||||
[0, accepted[0], 0, accepted[1]],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
request_indices = torch.tensor([1, 3], dtype=torch.int32, device=DEVICE)
|
||||
else:
|
||||
global_num_accepted = torch.tensor(accepted, dtype=torch.int32, device=DEVICE)
|
||||
request_indices = None
|
||||
|
||||
block_table = None
|
||||
num_computed_tokens = None
|
||||
mamba_block_size = None
|
||||
if align_mode:
|
||||
batch_size = 4 if use_request_indices else num_seqs
|
||||
block_table = torch.full((batch_size, 2), -1, dtype=torch.int32, device=DEVICE)
|
||||
rows = (
|
||||
request_indices
|
||||
if request_indices is not None
|
||||
else torch.arange(num_seqs, device=DEVICE)
|
||||
)
|
||||
block_table[rows] = torch.tensor(
|
||||
[[1, 5], [2, 6]],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
num_computed_tokens = torch.zeros(batch_size, dtype=torch.int32, device=DEVICE)
|
||||
num_computed_tokens[rows] = 4
|
||||
mamba_block_size = 8
|
||||
|
||||
layers = []
|
||||
expected_outputs = []
|
||||
expected_states = []
|
||||
initial_states = []
|
||||
initial_conv_states = []
|
||||
history_len, conv_dim = 3, 12
|
||||
for layer_idx in range(num_layers):
|
||||
A_log = (
|
||||
0.2 * torch.randn(num_heads, dtype=torch.float32, device=DEVICE)
|
||||
+ layer_idx * 0.03
|
||||
).contiguous()
|
||||
dt_bias = (
|
||||
0.1 * torch.randn(num_heads, dim, dtype=torch.float32, device=DEVICE)
|
||||
).contiguous()
|
||||
checkpoint = 0.01 * torch.randn(
|
||||
num_blocks,
|
||||
num_heads,
|
||||
dim,
|
||||
dim,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
conv_shape = (
|
||||
(num_blocks, conv_dim, history_len + query_len - 1)
|
||||
if conv_state_dim_first
|
||||
else (num_blocks, history_len + query_len - 1, conv_dim)
|
||||
)
|
||||
conv_state = torch.randn(conv_shape, dtype=torch.bfloat16, device=DEVICE)
|
||||
correction_cache = torch.empty(
|
||||
num_blocks,
|
||||
num_heads,
|
||||
query_len,
|
||||
dim,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
kg_cache = torch.empty(
|
||||
num_blocks,
|
||||
num_heads,
|
||||
query_len,
|
||||
2 * dim,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
kv_cache=(
|
||||
conv_state,
|
||||
checkpoint,
|
||||
correction_cache,
|
||||
kg_cache,
|
||||
),
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
local_num_heads=num_heads,
|
||||
head_dim=dim,
|
||||
gate_lower_bound=lower_bound,
|
||||
)
|
||||
layers.append(layer)
|
||||
initial_states.append(checkpoint.clone())
|
||||
initial_conv_states.append(conv_state.clone())
|
||||
|
||||
actual_output = kda_recoverssm_verify(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
checkpoint_state=checkpoint,
|
||||
correction_cache=correction_cache,
|
||||
kg_cache=kg_cache,
|
||||
query_start_loc=query_start_loc,
|
||||
state_indices=state_indices,
|
||||
spec_query_len=query_len,
|
||||
)
|
||||
|
||||
normalized_q = q.float() * torch.rsqrt(
|
||||
q.float().square().sum(dim=-1, keepdim=True) + 1e-6
|
||||
)
|
||||
normalized_k = k.float() * torch.rsqrt(
|
||||
k.float().square().sum(dim=-1, keepdim=True) + 1e-6
|
||||
)
|
||||
gate_input = raw_g.float() + dt_bias.view(1, 1, num_heads, dim)
|
||||
if lower_bound is None:
|
||||
gate = -A_log.exp().view(1, 1, num_heads, 1) * F.softplus(gate_input)
|
||||
else:
|
||||
gate = lower_bound * torch.sigmoid(
|
||||
A_log.exp().view(1, 1, num_heads, 1) * gate_input
|
||||
)
|
||||
beta = raw_beta.float().sigmoid()
|
||||
|
||||
reference_output = []
|
||||
committed_states = checkpoint.clone()
|
||||
for seq_idx, commit_len in enumerate(accepted):
|
||||
start = seq_idx * query_len
|
||||
end = start + query_len
|
||||
output, _ = naive_recurrent_kda(
|
||||
normalized_q[:, start:end],
|
||||
normalized_k[:, start:end],
|
||||
v[:, start:end],
|
||||
gate[:, start:end],
|
||||
beta[:, start:end],
|
||||
initial_state=checkpoint[state_indices[seq_idx]].transpose(-1, -2),
|
||||
)
|
||||
reference_output.append(output)
|
||||
_, committed_state = naive_recurrent_kda(
|
||||
normalized_q[:, start : start + commit_len],
|
||||
normalized_k[:, start : start + commit_len],
|
||||
v[:, start : start + commit_len],
|
||||
gate[:, start : start + commit_len],
|
||||
beta[:, start : start + commit_len],
|
||||
initial_state=checkpoint[state_indices[seq_idx]].transpose(-1, -2),
|
||||
output_final_state=True,
|
||||
)
|
||||
assert committed_state is not None
|
||||
final_block = state_indices[seq_idx]
|
||||
if align_mode:
|
||||
assert block_table is not None
|
||||
row = request_indices[seq_idx] if use_request_indices else seq_idx
|
||||
final_block = block_table[row, (4 + commit_len) // 8]
|
||||
committed_states[final_block] = committed_state.transpose(-1, -2)
|
||||
if align_mode and 4 + commit_len >= 8:
|
||||
_, boundary_state = naive_recurrent_kda(
|
||||
normalized_q[:, start : start + 4],
|
||||
normalized_k[:, start : start + 4],
|
||||
v[:, start : start + 4],
|
||||
gate[:, start : start + 4],
|
||||
beta[:, start : start + 4],
|
||||
initial_state=checkpoint[state_indices[seq_idx]].transpose(-1, -2),
|
||||
output_final_state=True,
|
||||
)
|
||||
assert boundary_state is not None
|
||||
assert block_table is not None
|
||||
row = request_indices[seq_idx] if use_request_indices else seq_idx
|
||||
committed_states[block_table[row, 0]] = boundary_state.transpose(-1, -2)
|
||||
expected_outputs.append(torch.cat(reference_output, dim=1))
|
||||
expected_states.append(committed_states)
|
||||
torch.testing.assert_close(checkpoint, initial_states[-1])
|
||||
torch.testing.assert_close(
|
||||
actual_output,
|
||||
expected_outputs[-1],
|
||||
atol=3e-2,
|
||||
rtol=3e-2,
|
||||
)
|
||||
|
||||
context = KDARecoverSSMCommitContext.create(
|
||||
layers,
|
||||
spec_query_len=query_len,
|
||||
max_num_reqs=global_num_accepted.shape[0],
|
||||
)
|
||||
context.commit(
|
||||
global_num_accepted,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
request_indices=request_indices,
|
||||
block_table=block_table,
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
mamba_block_size=mamba_block_size,
|
||||
)
|
||||
|
||||
for layer_idx, layer in enumerate(layers):
|
||||
torch.testing.assert_close(
|
||||
layer.kv_cache[1],
|
||||
expected_states[layer_idx],
|
||||
atol=3e-3,
|
||||
rtol=3e-3,
|
||||
)
|
||||
for seq_idx, commit_len in enumerate(accepted):
|
||||
block = state_indices[seq_idx]
|
||||
if align_mode:
|
||||
assert block_table is not None
|
||||
row = request_indices[seq_idx] if use_request_indices else seq_idx
|
||||
block = block_table[row, (4 + commit_len) // 8]
|
||||
source_block = state_indices[seq_idx] if align_mode else block
|
||||
if conv_state_dim_first:
|
||||
actual_conv = layer.kv_cache[0][block, :, :history_len]
|
||||
expected_conv = initial_conv_states[layer_idx][
|
||||
source_block,
|
||||
:,
|
||||
commit_len - 1 : commit_len - 1 + history_len,
|
||||
]
|
||||
else:
|
||||
actual_conv = layer.kv_cache[0][block, :history_len]
|
||||
expected_conv = initial_conv_states[layer_idx][
|
||||
source_block,
|
||||
commit_len - 1 : commit_len - 1 + history_len,
|
||||
]
|
||||
torch.testing.assert_close(actual_conv, expected_conv)
|
||||
if align_mode and 4 + commit_len >= 8:
|
||||
assert block_table is not None
|
||||
boundary_block = block_table[row, 0]
|
||||
if conv_state_dim_first:
|
||||
actual_boundary_conv = layer.kv_cache[0][
|
||||
boundary_block, :, :history_len
|
||||
]
|
||||
expected_boundary_conv = initial_conv_states[layer_idx][
|
||||
state_indices[seq_idx], :, 3 : 3 + history_len
|
||||
]
|
||||
else:
|
||||
actual_boundary_conv = layer.kv_cache[0][
|
||||
boundary_block, :history_len
|
||||
]
|
||||
expected_boundary_conv = initial_conv_states[layer_idx][
|
||||
state_indices[seq_idx], 3 : 3 + history_len
|
||||
]
|
||||
torch.testing.assert_close(actual_boundary_conv, expected_boundary_conv)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_heads", "num_seqs", "lower_bound", "fuse_output_norm"),
|
||||
[
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import fields
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -26,6 +27,9 @@ from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMPostprocessMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
NULL_BLOCK_ID,
|
||||
mamba_get_block_table_tensor,
|
||||
@@ -44,8 +48,12 @@ PRUNED_METADATA_FIELDS = {
|
||||
}
|
||||
|
||||
|
||||
def _assert_matches_shared_gdn(reference, actual: KimiK3KDAMetadata):
|
||||
for field in fields(KimiK3KDAMetadata):
|
||||
def _assert_matches_shared_gdn(
|
||||
reference: GDNAttentionMetadata, actual: KimiK3KDAMetadata
|
||||
):
|
||||
assert actual.recoverssm_commit is None
|
||||
assert actual.recoverssm_context is None
|
||||
for field in fields(GDNAttentionMetadata):
|
||||
actual_value = getattr(actual, field.name)
|
||||
expected_value = getattr(reference, field.name)
|
||||
if field.name in PRUNED_METADATA_FIELDS:
|
||||
@@ -78,6 +86,7 @@ def _make_builder(
|
||||
full_cuda_graph: bool,
|
||||
device: torch.device = DEVICE,
|
||||
mamba_cache_mode: str = "none",
|
||||
use_recoverssm: bool = False,
|
||||
) -> AttentionMetadataBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
@@ -92,17 +101,24 @@ def _make_builder(
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
return builder_cls(
|
||||
vllm_config.cache_config.use_replayssm = use_recoverssm
|
||||
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
|
||||
builder = builder_cls(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=num_speculative_tokens,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_speculative_blocks=(0 if use_recoverssm else num_speculative_tokens),
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
if use_recoverssm:
|
||||
assert isinstance(builder, KimiK3KDAMetadataBuilder)
|
||||
builder.recoverssm_context = Mock()
|
||||
return builder
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -244,6 +260,99 @@ def test_mixed_regular_and_spec_decode_excludes_request_padding():
|
||||
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
def test_recoverssm_spec_uses_one_state_slot_and_current_window(
|
||||
mamba_cache_mode: str,
|
||||
):
|
||||
if mamba_cache_mode == "align" and not torch.cuda.is_available():
|
||||
pytest.skip("align metadata construction requires CUDA")
|
||||
device = torch.device("cuda") if mamba_cache_mode == "align" else DEVICE
|
||||
batch = BatchSpec(seq_lens=[100, 65, 20], query_lens=[1, 1, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device
|
||||
).replace(is_prefilling=torch.tensor([True, True, False]))
|
||||
builder = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
device=device,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
use_recoverssm=True,
|
||||
)
|
||||
assert isinstance(builder, KimiK3KDAMetadataBuilder)
|
||||
context = builder.recoverssm_context
|
||||
assert context is not None
|
||||
actual = builder.build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.tensor([3, 2, 2], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
assert actual.spec_state_indices_tensor is not None
|
||||
assert actual.spec_state_indices_tensor.shape == (1, 1)
|
||||
torch.testing.assert_close(
|
||||
actual.num_accepted_tokens,
|
||||
torch.ones(1, dtype=torch.int32, device=device),
|
||||
)
|
||||
commit_metadata = actual.recoverssm_commit
|
||||
assert commit_metadata is not None
|
||||
torch.testing.assert_close(
|
||||
commit_metadata.request_indices,
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert actual.recoverssm_context is context
|
||||
num_accepted_tokens = torch.tensor([3, 2, 1], dtype=torch.int32, device=device)
|
||||
|
||||
postprocess = actual.commit_recoverssm_state(num_accepted_tokens)
|
||||
|
||||
if mamba_cache_mode == "none":
|
||||
assert commit_metadata.align is None
|
||||
assert postprocess is None
|
||||
else:
|
||||
assert isinstance(postprocess, RecoverSSMPostprocessMetadata)
|
||||
assert postprocess.num_spec_decodes == 1
|
||||
assert postprocess.request_indices is commit_metadata.request_indices
|
||||
assert postprocess.block_table is common_attn_metadata.block_table_tensor
|
||||
assert (
|
||||
postprocess.num_computed_tokens
|
||||
is common_attn_metadata.compute_num_computed_tokens()
|
||||
)
|
||||
assert postprocess.block_size == BLOCK_SIZE
|
||||
args = context.commit.call_args.args
|
||||
assert args[0] is num_accepted_tokens
|
||||
torch.testing.assert_close(args[1], commit_metadata.state_indices[:, 0])
|
||||
torch.testing.assert_close(args[2], commit_metadata.query_start_loc)
|
||||
|
||||
|
||||
def test_recoverssm_distinguishes_draftless_decode_from_one_token_prefill():
|
||||
batch = BatchSpec(seq_lens=[40, 30], query_lens=[1, 1])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([False, True]))
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
use_recoverssm=True,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.full((2,), -1, dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(2, dtype=torch.int32),
|
||||
)
|
||||
|
||||
assert actual.num_spec_decodes == 1
|
||||
assert actual.num_decodes == 0
|
||||
assert actual.num_prefills == 1
|
||||
assert actual.spec_state_indices_tensor is not None
|
||||
assert actual.spec_state_indices_tensor.shape == (1, 1)
|
||||
torch.testing.assert_close(
|
||||
actual.spec_query_start_loc,
|
||||
torch.tensor([0, 1], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seq_len", "expected_has_initial_state"),
|
||||
[
|
||||
@@ -307,6 +416,38 @@ def test_kimi_k3_kda_cudagraph_capture_matches_shared_gdn():
|
||||
_assert_matches_shared_gdn(reference, actual)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_recoverssm_spec_cudagraph_stages_one_checkpoint_per_request():
|
||||
device = torch.device("cuda")
|
||||
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device
|
||||
).replace(is_prefilling=torch.tensor([False, False]))
|
||||
builder = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=True,
|
||||
device=device,
|
||||
use_recoverssm=True,
|
||||
)
|
||||
assert isinstance(builder, KimiK3KDAMetadataBuilder)
|
||||
assert builder.spec_state_indices_tensor.shape == (
|
||||
builder.vllm_config.scheduler_config.max_num_seqs,
|
||||
1,
|
||||
)
|
||||
actual = builder.build_for_cudagraph_capture(common_attn_metadata)
|
||||
|
||||
assert actual.spec_state_indices_tensor is not None
|
||||
assert actual.spec_state_indices_tensor.shape == (batch.batch_size, 1)
|
||||
assert actual.num_accepted_tokens is not None
|
||||
torch.testing.assert_close(
|
||||
actual.num_accepted_tokens,
|
||||
torch.ones(batch.batch_size, dtype=torch.int32, device=device),
|
||||
)
|
||||
assert actual.recoverssm_commit is not None
|
||||
assert actual.recoverssm_commit.request_indices is None
|
||||
|
||||
|
||||
def test_kimi_k3_kda_backend_uses_private_metadata_builder():
|
||||
assert KimiK3KDAAttentionBackend.get_builder_cls() is KimiK3KDAMetadataBuilder
|
||||
assert KimiK3KDAAttentionBackend.is_ssm()
|
||||
|
||||
@@ -143,8 +143,10 @@ def test_registry_is_pp(model_arch, is_pp, init_cuda):
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
# ReplaySSM is opt-in per model; only Nemotron-H sets the flag today.
|
||||
# ReplaySSM is opt-in per model.
|
||||
("NemotronHForCausalLM", True),
|
||||
("KimiLinearForCausalLM", not current_platform.is_rocm()),
|
||||
("KimiK3ForConditionalGeneration", not current_platform.is_rocm()),
|
||||
("Mamba2ForCausalLM", False),
|
||||
("Zamba2ForCausalLM", False),
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user