[Kimi-K3] Add FlashInfer speculative KDA backend (#54255)

Signed-off-by: Duncan Moss <dmoss@nvidia.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Duncan Moss
2026-10-01 02:28:50 +00:00
committed by GitHub
co-authored by Roger Wang mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Thien Tran
parent edaf3e7b61
commit 766b4ae54c
3 changed files with 383 additions and 9 deletions
+241
View File
@@ -30,13 +30,17 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import (
)
from vllm.models.kimi_k3.nvidia import kda as nvidia_kda
from vllm.models.kimi_k3.nvidia.kda import (
KimiK3DeltaAttention,
_flashinfer_kda_prefill,
_flashkda_prefill,
is_flashinfer_fused_kda_decode_supported,
is_flashinfer_fused_kda_spec_decode_supported,
is_flashinfer_recurrent_kda_prefill_supported,
is_flashkda_supported,
is_fused_kda_decode_supported,
resolve_kda_spec_decode_backend,
)
from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata
from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM
from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
@@ -90,6 +94,25 @@ def test_kda_warmup_skips_missing_metadata(monkeypatch):
assert layer._forward(empty, empty, empty, empty, empty) is None
def test_resolve_kda_spec_decode_backend(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
lambda *args: True,
)
args = (12, 128, 4, 6, torch.bfloat16, torch.bfloat16, torch.float32, -5.0)
assert resolve_kda_spec_decode_backend("auto", *args, False) == "flashinfer"
assert resolve_kda_spec_decode_backend("native", *args, False) == "native"
assert resolve_kda_spec_decode_backend("flashinfer", *args, False) == "flashinfer"
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
lambda *args: False,
)
assert resolve_kda_spec_decode_backend("auto", *args, False) == "native"
with pytest.raises(RuntimeError, match="packed_fused_kda_decode"):
resolve_kda_spec_decode_backend("flashinfer", *args, False)
def test_kda_recoverssm_config_state_layout():
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
@@ -1182,6 +1205,224 @@ def test_fused_kda_decode_correctness(
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
@torch.inference_mode()
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16])
def test_flashinfer_fused_kda_spec_decode_integration(
monkeypatch: pytest.MonkeyPatch,
state_dtype: torch.dtype,
):
if not is_flashinfer_fused_kda_spec_decode_supported(
num_heads=12,
head_dim=128,
conv_width=4,
num_spec=6,
input_dtype=torch.bfloat16,
conv_state_dtype=torch.bfloat16,
recurrent_state_dtype=state_dtype,
lower_bound=-5.0,
use_recoverssm=False,
):
pytest.skip("FlashInfer packed fused KDA decode is not supported")
generator = torch.Generator(device=DEVICE).manual_seed(20260820)
H, D, W, T = 12, 128, 4, 7
hidden_size = H * D
num_rows = 10
num_slots = 16
x = torch.randn(
num_rows,
3 * hidden_size,
dtype=torch.bfloat16,
device=DEVICE,
generator=generator,
)
conv_weight = 0.1 * torch.randn(
3 * hidden_size,
W,
dtype=torch.float32,
device=DEVICE,
generator=generator,
)
fused_weight = conv_weight.view(3, hidden_size, W).transpose(1, 2).contiguous()
raw_gate = torch.randn(
1, num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
raw_beta = torch.randn(
1, num_rows, H, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
output_gate = torch.randn(
num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
A_log = torch.log(torch.arange(1, H + 1, dtype=torch.float32, device=DEVICE))
dt_bias = torch.empty(hidden_size, dtype=torch.float32, device=DEVICE).uniform_(
-7.0, -4.0, generator=generator
)
norm_weight = torch.randn(
D, dtype=torch.float32, device=DEVICE, generator=generator
)
norm_eps = 1e-5
state_indices = torch.tensor(
[
list(range(1, T + 1)),
list(range(T + 1, 2 * T + 1)),
[0] * T,
],
dtype=torch.int32,
device=DEVICE,
)
query_start_loc = torch.tensor([0, 7, 10, 10], dtype=torch.int32, device=DEVICE)
num_accepted_tokens = torch.tensor([1, 4, 1], dtype=torch.int32, device=DEVICE)
conv_seed = (
0.5
* torch.randn(
num_slots,
T + W - 2,
3 * hidden_size,
dtype=torch.bfloat16,
device=DEVICE,
generator=generator,
)
).transpose(1, 2)
state_seed = (
0.5
* torch.randn(
num_slots,
H,
D,
D,
dtype=torch.float32,
device=DEVICE,
generator=generator,
)
).to(state_dtype)
metadata = KimiK3KDAMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=0,
num_decode_tokens=0,
num_spec_decodes=2,
num_spec_decode_tokens=num_rows,
num_actual_tokens=num_rows,
spec_query_start_loc=query_start_loc,
spec_state_indices_tensor=state_indices,
num_accepted_tokens=num_accepted_tokens,
)
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.get_forward_context",
lambda: SimpleNamespace(attn_metadata={"test.layer": metadata}),
)
def clone_strided(tensor: torch.Tensor) -> torch.Tensor:
clone = torch.empty_strided(
tensor.shape, tensor.stride(), dtype=tensor.dtype, device=tensor.device
)
clone.copy_(tensor)
return clone
def composed_reference(
conv_state: torch.Tensor,
recurrent_state: torch.Tensor,
) -> torch.Tensor:
mixed_qkv = causal_conv1d_update(
x,
conv_state,
conv_weight,
bias=None,
activation="silu",
conv_state_indices=state_indices[:, 0],
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc,
max_query_len=T,
validate_data=False,
out=torch.empty_like(x),
)
q, k, v = mixed_qkv.view(num_rows, 3, H, D).unbind(1)
recurrent_out, _ = fused_recurrent_kda_nvidia(
q=q.unsqueeze(0),
k=k.unsqueeze(0),
v=v.unsqueeze(0),
raw_g=raw_gate,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=-5.0,
initial_state=recurrent_state,
cu_seqlens=query_start_loc,
ssm_state_indices=state_indices,
num_accepted_tokens=num_accepted_tokens,
)
recurrent_float = recurrent_out.float()
return (
recurrent_float
* torch.rsqrt(
recurrent_float.square().mean(dim=-1, keepdim=True) + norm_eps
)
* norm_weight
* output_gate.float().sigmoid().unsqueeze(0)
).to(torch.bfloat16)
def run_flashinfer(
conv_state: torch.Tensor,
recurrent_state: torch.Tensor,
output: torch.Tensor,
) -> torch.Tensor:
layer = SimpleNamespace(
prefix="test.layer",
kv_cache=(conv_state.transpose(-1, -2), recurrent_state),
kda_spec_decode_backend="flashinfer",
decode_conv1d_weight=fused_weight,
decode_norm_weight=norm_weight,
gate_lower_bound=-5.0,
A_log=A_log,
dt_bias=dt_bias,
o_norm=SimpleNamespace(eps=norm_eps),
)
KimiK3DeltaAttention._forward(
layer,
mixed_qkv=x,
g1=raw_gate,
g2=output_gate,
beta=raw_beta,
core_attn_out=output,
)
return output
expected_conv = clone_strided(conv_seed)
expected_state = state_seed.clone()
expected = composed_reference(expected_conv, expected_state)
actual_conv = clone_strided(conv_seed)
actual_state = state_seed.clone()
output = torch.empty_like(expected)
actual = run_flashinfer(actual_conv, actual_state, output)
assert actual.data_ptr() == output.data_ptr()
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=3e-2)
assert torch.count_nonzero(actual[:, :7]) > 0
torch.testing.assert_close(actual_state[0], state_seed[0], atol=0, rtol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = run_flashinfer(actual_conv, actual_state, output)
actual_conv.copy_(conv_seed)
actual_state.copy_(state_seed)
query_start_loc.copy_(torch.tensor([0, 6, 10, 10], device=DEVICE))
num_accepted_tokens.copy_(torch.tensor([3, 2, 1], device=DEVICE))
expected_conv.copy_(conv_seed)
expected_state.copy_(state_seed)
expected = composed_reference(expected_conv, expected_state)
graph.replay()
torch.accelerator.synchronize()
assert captured.data_ptr() == output.data_ptr()
torch.testing.assert_close(output, expected, atol=2e-2, rtol=3e-2)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
def test_fused_kda_decode_rejects_speculative_conv_state():
assert not is_fused_kda_decode_supported(
num_heads=12,