[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
@@ -0,0 +1,221 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
import pytest
import torch
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
)
_DTYPE = torch.bfloat16
_NUM_TOKENS = 3
_NUM_HEADS = 4
_BLOCK_SIZE = 8
_POSITIONS = (1, 7, 13)
_SLOTS = (0, 3, 9)
def _randn(*shape: int) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=_DTYPE) * 0.2
def _rope_cache(max_position: int = 32) -> torch.Tensor:
inv_freq = 1.0 / (
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
)
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
freqs = torch.outer(positions, inv_freq)
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
def _apply_gptj_rope(
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
) -> torch.Tensor:
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
for _ in range(x.ndim - 2):
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
x1 = x[..., ::2].float()
x2 = x[..., 1::2].float()
out1 = x1 * cos.float() - x2 * sin.float()
out2 = x2 * cos.float() + x1 * sin.float()
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
torch.testing.assert_close(
actual.float(),
expected.to(torch.float8_e4m3fn).float(),
atol=0.03125,
rtol=0.15,
)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(0)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
k_pe = _randn(_NUM_TOKENS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
q_expected = q.clone()
q_expected[..., 128:] = _apply_gptj_rope(
q_expected[..., 128:], positions, cos_sin_cache
)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
k_expected = torch.cat(
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = q.clone()
k_actual = fused_mla_key_concat_kv_cache_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
q,
k_nope,
k_pe,
kv_c,
v,
cache,
slots,
one,
one,
one,
one,
positions,
cos_sin_cache,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(k_actual, k_expected)
_assert_fp8_close(v_actual, v)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = q.clone()
k_actual = fused_mla_key_concat_ds_mla_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(1)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope,
q_pe,
kv_c,
k_pe,
cache,
slots,
q_scale_inv=one,
cache_scale_inv=one,
**kwargs,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@torch.inference_mode()
def test_decode_epilogue_preserves_nope_path() -> None:
torch.manual_seed(2)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots
)
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)
+40
View File
@@ -197,6 +197,46 @@ def test_silu_and_mul_with_clamp(
opcheck(torch.ops._C.silu_and_mul_with_clamp, (out_buf, x, swiglu_limit))
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
@torch.inference_mode()
def test_masked_situ_and_mul(
default_vllm_config,
linear_beta: float,
dtype: torch.dtype,
) -> None:
"""Masked SITU computes valid expert rows and preserves padded zeros."""
device = CUDA_DEVICES[0]
num_experts, max_num_tokens, d = 4, 7, 512
beta = 1.5
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
torch.ops._C.masked_situ_and_mul(
output, input, expert_num_tokens, beta, linear_beta
)
gate, up = input.float().chunk(2, dim=-1)
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
if linear_beta > 0:
up = linear_beta * torch.tanh(up / linear_beta)
expected = (expected * up).to(dtype)
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
torch.testing.assert_close(
output[expert, :num_tokens],
expected[expert, :num_tokens],
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
opcheck(
torch.ops._C.masked_situ_and_mul,
(output, input, expert_num_tokens, beta, linear_beta),
)
@pytest.mark.parametrize(
"activation",
[
@@ -13,7 +13,7 @@ from __future__ import annotations
import pytest
import torch
from vllm.models.deepseek_v4.common.ops import fused_q_kv_rmsnorm
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
@@ -7,7 +7,9 @@ matching the eager triton kernel output."""
import pytest
import torch
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.third_party.flash_linear_attention.ops.fused_norm_gate import (
FusedRMSNormGated,
)
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.bfloat16]
+266
View File
@@ -23,6 +23,53 @@ from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
def _run_single_group_topk(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
return fused_grouped_topk(
hidden_states=torch.empty(
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
),
gating_output=logits,
topk=topk,
renormalize=renormalize,
e_score_correction_bias=bias,
num_expert_group=1,
topk_group=1,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
)
def _single_group_reference(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
if scoring_func == "sigmoid":
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
else:
scores = torch.softmax(logits, dim=-1).float()
indices = torch.argsort(
scores + bias.float(), dim=-1, descending=True, stable=True
)[:, :topk]
values = scores.gather(1, indices)
if renormalize:
values /= values.sum(dim=-1, keepdim=True) + 1e-20
values *= routed_scaling_factor
return values, indices.to(torch.int32)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@@ -101,3 +148,222 @@ def test_grouped_topk(
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
)
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
def test_grouped_topk_single_group_large_batch():
set_random_seed(0)
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
actual_values, actual_ids = _run_single_group_topk(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,input_dtype,bias_dtype",
[
(512, 9, torch.bfloat16, torch.float32),
(512, 16, torch.float16, torch.float16),
(513, 9, torch.float32, torch.bfloat16),
(513, 16, torch.bfloat16, torch.float32),
(895, 9, torch.float16, torch.bfloat16),
(896, 16, torch.float32, torch.float16),
(897, 9, torch.bfloat16, torch.bfloat16),
(897, 16, torch.float16, torch.float32),
(1024, 9, torch.float32, torch.bfloat16),
(1024, 16, torch.bfloat16, torch.float16),
],
)
@pytest.mark.parametrize(
"scoring_func,renormalize,routed_scaling_factor",
[
("sigmoid", True, 1.0),
("sigmoid", False, 2.5),
("softmax", True, 2.5),
("softmax", False, 1.0),
],
)
def test_grouped_topk_single_group_tiers(
num_experts: int,
topk: int,
input_dtype: torch.dtype,
bias_dtype: torch.dtype,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float,
):
set_random_seed(7)
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,scoring_func",
[
(128, 8, "sigmoid"),
(129, 8, "sigmoid"),
(257, 8, "sigmoid"),
(385, 8, "sigmoid"),
(512, 9, "sigmoid"),
(513, 9, "sigmoid"),
(769, 9, "sigmoid"),
(897, 16, "sigmoid"),
(1024, 16, "sigmoid"),
(128, 4, "softmax"),
(128, 5, "softmax"),
(129, 8, "softmax"),
(161, 8, "softmax"),
(256, 9, "softmax"),
(257, 8, "softmax"),
(512, 9, "softmax"),
(512, 17, "softmax"),
(512, 23, "softmax"),
(513, 8, "softmax"),
(577, 9, "softmax"),
(769, 9, "softmax"),
(897, 9, "softmax"),
(1024, 16, "softmax"),
],
)
def test_grouped_topk_single_group_capacity_tiers(
num_experts: int,
topk: int,
scoring_func: str,
):
set_random_seed(11)
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
def test_grouped_topk_single_group_stable_ties(num_experts: int):
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=True,
routed_scaling_factor=2.5,
)
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
@pytest.mark.parametrize("num_finite", [0, 15])
@pytest.mark.parametrize("renormalize", [False, True])
def test_grouped_topk_single_group_nonfinite_scores(
num_experts: int, num_finite: int, renormalize: bool
):
logits = torch.full(
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
)
if num_finite:
logits[0, :num_finite] = torch.arange(
num_finite, dtype=torch.bfloat16, device="cuda"
)
logits[0, num_finite] = torch.inf
logits[0, num_finite + 1] = -torch.inf
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=renormalize,
routed_scaling_factor=2.5,
)
if num_finite == 0:
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
if renormalize:
expected_values = torch.full(
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
)
else:
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
else:
expected_ids = torch.cat(
(
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
)
)[None]
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
if renormalize:
finite_values /= finite_values.sum()
finite_values *= 2.5
expected_values = torch.cat(
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
)[None]
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@@ -269,6 +269,7 @@ def test_moe_align_block_size_with_expert_map(
if (experts[k] in local_experts) or not mask_inactive_experts
else -1
)
topk_ids[0, 0] = -1
actual_sorted_ids, actual_expert_ids, actual_num_tokens = moe_align_block_size(
topk_ids=topk_ids,
+649
View File
@@ -0,0 +1,649 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch)."""
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
import regex as re
import torch
from torch import nn
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig
from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
# Keyed by local (N, K): (cute token counts, dsv3 token counts). 1536x7168 is
# the unified shared_gate_up_proj/mla_g_proj entry (dsv3 M1..16).
EXPECTED_SELECTIONS = {
(1536, 128): (set(), set(range(1, 17))),
(3072, 128): (set(), set(range(1, 17))),
(1536, 7168): (set(), set(range(1, 17))),
(3072, 7168): (set(range(1, 6)), set()),
(2112, 7168): (set(), set(range(1, 17))),
(2304, 1536): (set(), set(range(1, 17))),
(4608, 1536): (set(), set(range(1, 17))),
(3584, 7168): ({1}, set(range(2, 9))),
(6288, 7168): (set(range(1, 5)), set()),
(12448, 7168): (set(range(1, 4)), set()),
(7168, 768): (set(), set(range(1, 17))),
(7168, 1536): ({1}, set()),
(7168, 3072): ({1, 2}, set()),
(7168, 3584): ({1, 2}, set()),
(7168, 4224): ({1}, set()),
(7168, 8448): (set(range(1, 4)), set()),
(8448, 7168): ({1, 2}, set()),
(16896, 7168): ({1, 2}, set()),
(20480, 7168): (set(range(1, 5)), set()),
(40960, 7168): (set(range(1, 5)), set()),
# TP16.
(3216, 7168): (set(range(1, 6)), set(range(9, 16))),
(768, 7168): (set(range(1, 5)), set(range(5, 17))),
(1152, 1536): ({1}, set(range(2, 17))),
(768, 128): (set(), set(range(1, 17))),
(7168, 384): (set(), set(range(1, 9))),
(4224, 7168): (set(range(1, 4)), set(range(4, 9))),
(10240, 7168): (set(range(1, 5)), set()),
}
CUTE_CASES = [
(spec.n, spec.k, num_tokens)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, _ in spec.cute_configs
]
RESIDUAL_CUTE_CASES = [
(spec.n, spec.k, num_tokens)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, _ in spec.residual_configs
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
(3072, 7168, 3): (128, 2, 1, 8),
(3072, 7168, 4): (64, 2, 2, 8),
(3072, 7168, 5): (128, 3, 1, 8),
(3584, 7168, 1): (224, 2, 4, 8),
(6288, 7168, 1): (224, 3, 4, 8),
(6288, 7168, 2): (64, 3, 2, 8),
(6288, 7168, 3): (32, 3, 4, 8),
(6288, 7168, 4): (128, 6, 1, 8),
(12448, 7168, 1): (224, 4, 2, 8),
(12448, 7168, 2): (64, 4, 2, 8),
(12448, 7168, 3): (64, 2, 2, 8),
(7168, 1536, 1): (96, 4, 2, 8),
(7168, 3072, 1): (96, 2, 4, 8),
(7168, 3072, 2): (32, 4, 4, 8),
(7168, 3584, 1): (224, 4, 2, 8),
(7168, 3584, 2): (64, 4, 2, 8),
(7168, 4224, 1): (96, 4, 2, 4),
(7168, 8448, 1): (32, 4, 4, 8),
(7168, 8448, 2): (96, 4, 1, 8),
(7168, 8448, 3): (96, 4, 1, 8),
(8448, 7168, 1): (224, 3, 4, 8),
(8448, 7168, 2): (32, 4, 4, 8),
(16896, 7168, 1): (224, 6, 4, 8),
(16896, 7168, 2): (32, 4, 4, 8),
(20480, 7168, 1): (224, 4, 2, 8),
(20480, 7168, 2): (64, 4, 2, 8),
(20480, 7168, 3): (64, 2, 2, 8),
(20480, 7168, 4): (64, 4, 1, 8),
(40960, 7168, 1): (128, 4, 2, 8),
(40960, 7168, 2): (64, 4, 2, 8),
(40960, 7168, 3): (64, 2, 2, 8),
(40960, 7168, 4): (64, 4, 1, 8),
# TP16.
(3216, 7168, 1): (224, 3, 4, 8),
(3216, 7168, 2): (128, 4, 2, 8),
(3216, 7168, 3): (128, 2, 1, 8),
(3216, 7168, 4): (64, 2, 2, 8),
(3216, 7168, 5): (128, 3, 1, 8),
(768, 7168, 1): (224, 2, 4, 8),
(768, 7168, 2): (224, 2, 2, 8),
(768, 7168, 3): (224, 2, 2, 8),
(768, 7168, 4): (224, 2, 2, 8),
(1152, 1536, 1): (192, 3, 4, 8),
(4224, 7168, 1): (224, 3, 4, 8),
(4224, 7168, 2): (128, 2, 1, 8),
(4224, 7168, 3): (64, 2, 2, 8),
(10240, 7168, 1): (224, 4, 2, 8),
(10240, 7168, 2): (32, 2, 4, 8),
(10240, 7168, 3): (64, 4, 1, 8),
(10240, 7168, 4): (64, 4, 1, 8),
}
EXPECTED_RESIDUAL_CUTE_CONFIGS = {
(7168, 3584, 1): (64, 4, 2, 8),
(7168, 3584, 2): (64, 7, 2, 8),
(7168, 3584, 3): (64, 2, 1, 8),
(7168, 3584, 4): (64, 2, 1, 8),
}
def _config_tuple(config) -> tuple[int, int, int, int]:
return (
config.block_size,
config.outputs_per_block,
config.k_unroll,
config.vector_width,
)
def test_table_is_keyed_by_shape() -> None:
for (n, k), spec in k3_gemm.KIMI_K3_PROJECTIONS.items():
assert (spec.n, spec.k) == (n, k)
def test_every_dsv3_routed_shape_is_instantiated() -> None:
"""dsv3_fused_a_gemm specializes on (K, N); an unlisted shape raises.
The table routes by shape while the kernel is built per shape, so a missing
instantiation only shows up at the token counts that route to dsv3. Checking
it here needs no GPU, which is the point -- a GPU-only check is exactly what
let (3216, 7168) ship without its DISPATCH_DSV3_SHAPE(7168, 3216).
"""
source = (
Path(__file__).resolve().parents[2]
/ "csrc"
/ "libtorch_stable"
/ "dsv3_fused_a_gemm.cu"
).read_text(encoding="utf-8")
# Benchmark-only shapes live behind VLLM_K3_BENCH_SHAPES and are not built
# by default, so they must not count as available.
production_macros = source.split("#ifdef VLLM_K3_BENCH_SHAPES")[0]
explicit = source.split("#undef DISPATCH_DSV3_SHAPE")[1].split(
"#ifdef VLLM_K3_BENCH_SHAPES"
)[0]
compiled = {
(int(hd_in), int(hd_out))
for hd_in, hd_out in re.findall(
r"DISPATCH_DSV3_SHAPE\((\d+),\s*(\d+)\)", production_macros
)
} | {
(int(hd_in), int(hd_out))
for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit)
}
assert compiled, "failed to parse the dispatch list"
missing = sorted(
(spec.n, spec.k)
for spec in KIMI_K3_PROJECTIONS.values()
if spec.dsv3_tokens and (spec.k, spec.n) not in compiled
)
assert not missing, (
f"routed to dsv3 with no instantiation: {missing}; add "
"DISPATCH_DSV3_SHAPE(K, N) for each"
)
def test_packed_row_major_rejects_single_row_slice() -> None:
packed = torch.empty(1, 128)
sliced = torch.empty(1, 144)[:, :128]
assert packed.is_contiguous()
assert sliced.is_contiguous()
assert k3_gemm._is_packed_row_major(packed)
assert not k3_gemm._is_packed_row_major(sliced)
def test_cute_configs_match_measured_table() -> None:
actual = {
(spec.n, spec.k, num_tokens): _config_tuple(config)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, config in spec.cute_configs
}
assert actual == EXPECTED_CUTE_CONFIGS
def test_residual_cute_configs_match_measured_table() -> None:
actual = {
(spec.n, spec.k, num_tokens): _config_tuple(config)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, config in spec.residual_configs
}
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
def test_sm103_selector_table(key: tuple[int, int]) -> None:
n, k = key
cute_tokens, dsv3_tokens = EXPECTED_SELECTIONS[key]
for num_tokens in range(1, 17):
backend = k3_gemm.select_kimi_k3_backend(num_tokens, n, k)
if num_tokens in cute_tokens:
assert backend == "cute"
elif num_tokens in dsv3_tokens:
assert backend == "dsv3_fused_a"
else:
assert backend is None
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
def test_selector_requires_supported_shape_and_tokens(key: tuple[int, int]) -> None:
n, k = key
assert k3_gemm.select_kimi_k3_backend(0, n, k) is None
assert k3_gemm.select_kimi_k3_backend(17, n, k) is None
assert k3_gemm.select_kimi_k3_backend(1, n + 1, k) is None
assert k3_gemm.select_kimi_k3_backend(1, n, k + 1) is None
def test_unlisted_shape_and_unselected_tokens_fall_back() -> None:
# Shape absent from the table.
assert k3_gemm.select_kimi_k3_backend(1, 1000, 1000) is None
# o_proj (7168,1536) is CuTe M1 only; M2+ falls back.
assert k3_gemm.select_kimi_k3_backend(2, 7168, 1536) is None
@pytest.mark.parametrize("num_tokens", range(1, 17))
def test_sm103_residual_selector_table(num_tokens: int) -> None:
backend = k3_gemm.select_kimi_k3_backend(num_tokens, 7168, 3584, has_residual=True)
assert backend == ("cute" if num_tokens <= 4 else None)
def test_build_plan_matches_selector() -> None:
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values():
plan = k3_gemm._build_plan(spec)
for num_tokens in range(1, 17):
backend = k3_gemm.select_kimi_k3_backend(num_tokens, spec.n, spec.k)
if backend is None:
assert num_tokens not in plan
else:
assert plan[num_tokens][0] == backend
def test_installation_is_shape_specific_and_unquantized(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class FakeLinear(nn.Module):
def __init__(self, quant_method: object, n: int, k: int) -> None:
super().__init__()
self.quant_method = quant_method
self.weight = torch.empty(n, k)
class FakeHead(nn.Module):
def __init__(self, n: int, k: int) -> None:
super().__init__()
self.quant_method = k3_gemm.UnquantizedEmbeddingMethod()
self.weight = torch.empty(n, k)
root = nn.Module()
# dsv3-only shape (no cute warmup contribution).
root.dsv3_only = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 2304, 1536)
# quantized: must be left untouched.
quantized_method = object()
root.quantized = FakeLinear(quantized_method, 6288, 7168)
# cute shape.
root.cute = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 6288, 7168)
# cute + residual shape.
root.residual = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 7168, 3584)
# shape absent from the table: must be left untouched.
root.unlisted = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 1234, 5678)
root.lm_head = FakeHead(20480, 7168)
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
monkeypatch.setattr(k3_gemm, "ParallelLMHead", FakeHead)
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: True)
warmup_configs: set[SkinnyGemmConfig] = set()
residual_warmup_configs: set[SkinnyGemmConfig] = set()
monkeypatch.setattr(k3_gemm.shape_dynamic_skinny_gemm, "is_available", lambda: True)
def request_warmup_configs(dtype, configs, *, has_residual=False):
target = residual_warmup_configs if has_residual else warmup_configs
target.update(configs)
monkeypatch.setattr(
k3_gemm.shape_dynamic_skinny_gemm,
"request_warmup_configs",
request_warmup_configs,
)
k3_gemm.enable_kimi_k3_low_latency_gemm(root, torch.bfloat16)
assert isinstance(root.dsv3_only.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
assert isinstance(root.cute.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
assert isinstance(root.residual.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
assert root.quantized.quant_method is quantized_method
assert type(root.unlisted.quant_method) is k3_gemm.UnquantizedLinearMethod
assert isinstance(
root.lm_head.quant_method, k3_gemm.KimiK3LowLatencyEmbeddingMethod
)
# Warmup covers only the installed modules' local (N, K).
assert warmup_configs == {
config
for key in ((6288, 7168), (7168, 3584), (20480, 7168))
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[key].cute_configs
}
assert residual_warmup_configs == {
config
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)].residual_configs
}
@pytest.mark.parametrize(
"dtype,platform_enabled",
[(torch.float16, True), (torch.bfloat16, False)],
)
def test_installation_requires_bf16_sm103(
monkeypatch: pytest.MonkeyPatch,
dtype: torch.dtype,
platform_enabled: bool,
) -> None:
class FakeLinear(nn.Module):
def __init__(self) -> None:
super().__init__()
self.quant_method = k3_gemm.UnquantizedLinearMethod()
self.weight = torch.empty(2304, 1536)
root = nn.Module()
root.projection = FakeLinear()
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: platform_enabled)
k3_gemm.enable_kimi_k3_low_latency_gemm(root, dtype)
assert type(root.projection.quant_method) is k3_gemm.UnquantizedLinearMethod
def _require_sm103_and_dsv3() -> None:
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
pytest.skip("Kimi-K3 production selection requires SM103")
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
pytest.skip("dsv3_fused_a_gemm was not built")
def _require_sm103_and_cute() -> None:
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
pytest.skip("Kimi-K3 production selection requires SM103")
if not k3_gemm.shape_dynamic_skinny_gemm.is_available():
pytest.skip("CuTe DSL is not available")
@pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES)
def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None:
_require_sm103_and_cute()
torch.manual_seed(42)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
output = k3_gemm.try_low_latency_gemm(x, weight)
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def _dsv3_probe_tokens(tokens: frozenset[int]) -> set[int]:
"""Extremes, plus both sides of the kernel's num_tokens<=8 tile_n branch."""
if not tokens:
return set()
return {min(tokens), max(tokens)} | ({8, 9} & set(tokens))
# Derived from the table rather than hand-listed, so a shape routed to dsv3
# cannot be added without being exercised here.
DSV3_CASES = sorted(
(num_tokens, spec.n, spec.k)
for spec in KIMI_K3_PROJECTIONS.values()
for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens)
)
@pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES)
def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
_require_sm103_and_dsv3()
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
assert num_tokens in spec.dsv3_tokens
torch.manual_seed(42)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
output = k3_gemm.try_low_latency_gemm(x, weight)
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_nonpacked_single_token_dsv3_falls_back() -> None:
_require_sm103_and_dsv3()
n, k = 1536, 128
storage = torch.randn(1, k + 16, dtype=torch.bfloat16, device="cuda")
x = storage[:, :k]
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
method = k3_gemm.KimiK3LowLatencyLinearMethod(
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
)
assert x.is_contiguous()
assert x.stride() == (k + 16, 1)
assert not k3_gemm._runtime_ok(x, weight) # strict guard rejects the view
output = method.apply(SimpleNamespace(weight=weight), x)
reference = torch.nn.functional.linear(x, weight)
torch.testing.assert_close(output, reference)
def test_selected_kernels_cuda_graph_capture() -> None:
_require_sm103_and_cute()
_require_sm103_and_dsv3()
cute_spec = k3_gemm.KIMI_K3_PROJECTIONS[(6288, 7168)]
dsv3_spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
cute_x = torch.randn(1, cute_spec.k, dtype=torch.bfloat16, device="cuda")
cute_weight = torch.randn(
cute_spec.n, cute_spec.k, dtype=torch.bfloat16, device="cuda"
)
dsv3_x = torch.randn(1, dsv3_spec.k, dtype=torch.bfloat16, device="cuda")
dsv3_weight = torch.randn(
dsv3_spec.n, dsv3_spec.k, dtype=torch.bfloat16, device="cuda"
)
k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
cute_output = k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
dsv3_output = k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
graph.replay()
torch.accelerator.synchronize()
assert cute_output is not None
assert dsv3_output is not None
for output, activation, weight in (
(cute_output, cute_x, cute_weight),
(dsv3_output, dsv3_x, dsv3_weight),
):
reference = torch.nn.functional.linear(activation, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
@pytest.mark.parametrize("num_tokens", [1, 8, 9, 16])
def test_dsv3_cuda_graph_capture_tile_branches(num_tokens: int) -> None:
"""Capture DSV3 across the num_tokens<=8 vs >8 tile_n branch."""
_require_sm103_and_dsv3()
spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
k3_gemm.try_low_latency_gemm(x, weight)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = k3_gemm.try_low_latency_gemm(x, weight)
graph.replay()
torch.accelerator.synchronize()
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
@pytest.mark.parametrize("n,k,num_tokens", RESIDUAL_CUTE_CASES)
def test_cute_residual_epilogue(n: int, k: int, num_tokens: int) -> None:
_require_sm103_and_cute()
torch.manual_seed(42 + num_tokens)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
config = spec.residual_config(num_tokens)
assert config is not None
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
reference = x.float() @ weight.float().t() + residual.float()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.flatten(), dim=0
).item()
assert cosine > 0.999
@pytest.mark.parametrize("num_tokens", range(1, 17))
def test_cute_residual_epilogue_all_supported_token_counts(num_tokens: int) -> None:
_require_sm103_and_cute()
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
ShapeDynamicSkinnyGemm,
)
n, k = 64, 512
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
config = ShapeDynamicSkinnyGemm._config(num_tokens, n, k)
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
reference = x.float() @ weight.float().t() + residual.float()
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
@pytest.mark.parametrize("num_tokens", range(1, 5))
def test_cute_residual_epilogue_cuda_graph_capture(num_tokens: int) -> None:
_require_sm103_and_cute()
spec = k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)]
config = spec.residual_config(num_tokens)
assert config is not None
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(num_tokens, spec.n, dtype=torch.bfloat16, device="cuda")
k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
graph.replay()
torch.accelerator.synchronize()
reference = x.float() @ weight.float().t() + residual.float()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.flatten(), dim=0
).item()
assert cosine > 0.999
class _SkinnyGemmSpy:
"""Wraps the skinny-GEMM singleton to record whether CuTe was invoked."""
def __init__(self, real: Any) -> None:
self._real = real
self.calls: list[int] = []
def __call__(self, a, b, config=None, residual=None):
self.calls.append(a.shape[0])
return self._real(a, b, config, residual)
def is_available(self) -> bool:
return self._real.is_available()
@pytest.mark.parametrize("num_tokens", [1, 2, 3, 4])
def test_latent_moe_production_layout_residual(
monkeypatch: pytest.MonkeyPatch,
num_tokens: int,
) -> None:
"""The real Latent-MoE residual is a non-packed slice of a cat buffer.
The strict packed-row-major guard rejects such a slice at every token count
(a size-1 leading dim reads as contiguous but its stride is not packed), so
the CuTe residual epilogue never fires for this production layout and the
method falls back to addmm. Output is correct regardless of the path.
"""
_require_sm103_and_cute()
latent_dim, shared_dim = 3584, 7168 # routed_expert_up_proj K, N
torch.manual_seed(7 + num_tokens)
buf = torch.randn(
num_tokens, latent_dim + shared_dim, dtype=torch.bfloat16, device="cuda"
)
latent = buf[:, :latent_dim] # non-contiguous view (row stride = full width)
residual = buf[:, latent_dim:] # non-contiguous view
weight = torch.randn(shared_dim, latent_dim, dtype=torch.bfloat16, device="cuda")
spec = k3_gemm.KIMI_K3_PROJECTIONS[(shared_dim, latent_dim)]
method = k3_gemm.KimiK3LowLatencyLinearMethod(
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
)
spy = _SkinnyGemmSpy(k3_gemm.shape_dynamic_skinny_gemm)
monkeypatch.setattr(k3_gemm, "shape_dynamic_skinny_gemm", spy)
layer = SimpleNamespace(weight=weight)
output = method.apply_with_residual(layer, latent, residual)
reference = latent.float() @ weight.float().t() + residual.float()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.flatten(), dim=0
).item()
assert cosine > 0.999 # correct regardless of the path taken
assert not spy.calls, (
"non-packed buf-slice residual must fall back to addmm at every M"
)
def test_residual_dispatch_falls_back_to_addmm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fallback = torch.randn(2, 3)
residual = torch.randn(2, 3)
x = torch.randn(2, 4)
weight = torch.randn(3, 4)
monkeypatch.setattr(torch, "addmm", lambda *args: fallback)
# CPU tensors fail the runtime check, forcing the addmm fallback.
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
output = method.apply_with_residual(SimpleNamespace(weight=weight), x, residual)
assert output is fallback
def test_fallback_preserves_default_method(monkeypatch: pytest.MonkeyPatch) -> None:
fallback = torch.empty(2, 8)
monkeypatch.setattr(
k3_gemm.UnquantizedLinearMethod,
"apply",
lambda *args: fallback,
)
# 1-D input fails the runtime check, forcing the base-method fallback.
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
output = method.apply(
SimpleNamespace(weight=torch.empty(0)),
torch.empty(0),
)
assert output is fallback
-226
View File
@@ -1,226 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Precision tests for vllm's chunk_kda Triton operator.
Compares chunk_kda against a naive recurrent reference (float32).
Uses torch.rand for q/k/v to match FLA's test pattern.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm.third_party.flash_linear_attention.ops.kda import (
chunk_kda,
chunk_kda_with_fused_gate,
fused_kda_gate,
)
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
DEVICE = "cuda"
def naive_recurrent_kda(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
scale: float | None = None,
initial_state: torch.Tensor | None = None,
output_final_state: bool = False,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
dtype = v.dtype
B, T, H, K = q.shape
V = v.shape[-1]
if scale is None:
scale = K**-0.5
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
q = q * scale
S = k.new_zeros(B, H, K, V).to(q)
if initial_state is not None:
S += initial_state
o = torch.zeros_like(v)
for i in range(T):
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
S = S * g_i[..., None].exp()
S = S + torch.einsum(
"bhk,bhv->bhkv",
b_i[..., None] * k_i,
v_i - (k_i[..., None] * S).sum(-2),
)
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
if not output_final_state:
S = None
return o.to(dtype), S
def assert_close(
name: str,
ref: torch.Tensor,
tri: torch.Tensor,
ratio: float,
err_atol: float = 1e-6,
):
"""RMSE-based relative error comparison."""
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
rel_err = rmse_diff / (rmse_base + 1e-8)
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
if abs_err <= err_atol:
return
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
assert rel_err < ratio, (
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
)
@pytest.mark.parametrize(
("H", "D", "cu_seqlens", "dtype"),
[
pytest.param(
*test,
id="H{}-D{}-cu{}-{}".format(*test),
)
for test in [
(32, 128, [0, 64], torch.float16),
(32, 128, [0, 1024], torch.float16),
(32, 128, [0, 15], torch.float16),
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
(64, 128, [0, 256, 500, 1000], torch.float16),
(32, 128, [0, 8192], torch.float16),
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
]
],
)
@torch.inference_mode()
def test_chunk_kda(
H: int,
D: int,
cu_seqlens: list[int],
dtype: torch.dtype,
):
T = cu_seqlens[-1]
torch.manual_seed(42)
B = 1
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
N = len(cu_seqlens) - 1
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
dtype
)
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
ref_outputs = []
ref_states = []
for i in range(N):
s, e = cu_seqlens[i], cu_seqlens[i + 1]
q_i = l2norm_fwd(q[:, s:e].contiguous())
k_i = l2norm_fwd(k[:, s:e].contiguous())
o_i, ht_i = naive_recurrent_kda(
q_i,
k_i,
v[:, s:e],
g[:, s:e],
beta[:, s:e],
initial_state=h0[i],
output_final_state=True,
)
ref_outputs.append(o_i)
ref_states.append(ht_i)
ref_o = torch.cat(ref_outputs, dim=1)
ref_ht = torch.cat(ref_states, dim=0)
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
tri_o, tri_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=g.clone(),
beta=beta.clone(),
initial_state=h0.transpose(-1, -2).contiguous().clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
assert_close("o", ref_o, tri_o, 0.005)
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
@pytest.mark.parametrize(
("cu_seqlens", "dtype"),
[
([0, 64], torch.float16),
([0, 15, 100, 300], torch.bfloat16),
],
)
@torch.inference_mode()
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
cu_seqlens: list[int],
dtype: torch.dtype,
):
H, D = 8, 64
T = cu_seqlens[-1]
N = len(cu_seqlens) - 1
torch.manual_seed(123)
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
beta = torch.rand(1, T, H, dtype=dtype, device=DEVICE).sigmoid()
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
dt_bias = (
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
).contiguous()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
initial_state = h0.transpose(-1, -2).contiguous()
gate = fused_kda_gate(
raw_g.reshape(T, H * D),
A_log,
D,
g_bias=dt_bias,
).unsqueeze(0)
old_o, old_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=gate,
beta=beta.clone(),
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
new_o, new_ht = chunk_kda_with_fused_gate(
q=q.clone(),
k=k.clone(),
v=v.clone(),
raw_g=raw_g,
beta=beta.clone(),
A_log=A_log,
g_bias=dt_bias,
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)