[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -0,0 +1,221 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
|
||||
)
|
||||
|
||||
_DTYPE = torch.bfloat16
|
||||
_NUM_TOKENS = 3
|
||||
_NUM_HEADS = 4
|
||||
_BLOCK_SIZE = 8
|
||||
_POSITIONS = (1, 7, 13)
|
||||
_SLOTS = (0, 3, 9)
|
||||
|
||||
|
||||
def _randn(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=_DTYPE) * 0.2
|
||||
|
||||
|
||||
def _rope_cache(max_position: int = 32) -> torch.Tensor:
|
||||
inv_freq = 1.0 / (
|
||||
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
|
||||
)
|
||||
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
|
||||
freqs = torch.outer(positions, inv_freq)
|
||||
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
|
||||
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
|
||||
|
||||
|
||||
def _apply_gptj_rope(
|
||||
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
|
||||
for _ in range(x.ndim - 2):
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
x1 = x[..., ::2].float()
|
||||
x2 = x[..., 1::2].float()
|
||||
out1 = x1 * cos.float() - x2 * sin.float()
|
||||
out2 = x2 * cos.float() + x1 * sin.float()
|
||||
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
|
||||
|
||||
|
||||
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
|
||||
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
|
||||
|
||||
|
||||
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
torch.testing.assert_close(
|
||||
actual.float(),
|
||||
expected.to(torch.float8_e4m3fn).float(),
|
||||
atol=0.03125,
|
||||
rtol=0.15,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(0)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
|
||||
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
|
||||
q_expected = q.clone()
|
||||
q_expected[..., 128:] = _apply_gptj_rope(
|
||||
q_expected[..., 128:], positions, cos_sin_cache
|
||||
)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
k_expected = torch.cat(
|
||||
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
|
||||
)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_kv_cache_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
v,
|
||||
cache,
|
||||
slots,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(k_actual, k_expected)
|
||||
_assert_fp8_close(v_actual, v)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_ds_mla_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(1)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
|
||||
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c,
|
||||
k_pe,
|
||||
cache,
|
||||
slots,
|
||||
q_scale_inv=one,
|
||||
cache_scale_inv=one,
|
||||
**kwargs,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_preserves_nope_path() -> None:
|
||||
torch.manual_seed(2)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
@@ -197,6 +197,46 @@ def test_silu_and_mul_with_clamp(
|
||||
opcheck(torch.ops._C.silu_and_mul_with_clamp, (out_buf, x, swiglu_limit))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_masked_situ_and_mul(
|
||||
default_vllm_config,
|
||||
linear_beta: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Masked SITU computes valid expert rows and preserves padded zeros."""
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts, max_num_tokens, d = 4, 7, 512
|
||||
beta = 1.5
|
||||
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
|
||||
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
|
||||
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
|
||||
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
output, input, expert_num_tokens, beta, linear_beta
|
||||
)
|
||||
|
||||
gate, up = input.float().chunk(2, dim=-1)
|
||||
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
if linear_beta > 0:
|
||||
up = linear_beta * torch.tanh(up / linear_beta)
|
||||
expected = (expected * up).to(dtype)
|
||||
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
|
||||
torch.testing.assert_close(
|
||||
output[expert, :num_tokens],
|
||||
expected[expert, :num_tokens],
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.masked_situ_and_mul,
|
||||
(output, input, expert_num_tokens, beta, linear_beta),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
|
||||
@@ -7,7 +7,9 @@ matching the eager triton kernel output."""
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.third_party.flash_linear_attention.ops.fused_norm_gate import (
|
||||
FusedRMSNormGated,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.bfloat16]
|
||||
|
||||
@@ -23,6 +23,53 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
def _run_single_group_topk(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return fused_grouped_topk(
|
||||
hidden_states=torch.empty(
|
||||
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
|
||||
),
|
||||
gating_output=logits,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
e_score_correction_bias=bias,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
|
||||
def _single_group_reference(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if scoring_func == "sigmoid":
|
||||
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
|
||||
else:
|
||||
scores = torch.softmax(logits, dim=-1).float()
|
||||
indices = torch.argsort(
|
||||
scores + bias.float(), dim=-1, descending=True, stable=True
|
||||
)[:, :topk]
|
||||
values = scores.gather(1, indices)
|
||||
if renormalize:
|
||||
values /= values.sum(dim=-1, keepdim=True) + 1e-20
|
||||
values *= routed_scaling_factor
|
||||
return values, indices.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@@ -101,3 +148,222 @@ def test_grouped_topk(
|
||||
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
def test_grouped_topk_single_group_large_batch():
|
||||
set_random_seed(0)
|
||||
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,input_dtype,bias_dtype",
|
||||
[
|
||||
(512, 9, torch.bfloat16, torch.float32),
|
||||
(512, 16, torch.float16, torch.float16),
|
||||
(513, 9, torch.float32, torch.bfloat16),
|
||||
(513, 16, torch.bfloat16, torch.float32),
|
||||
(895, 9, torch.float16, torch.bfloat16),
|
||||
(896, 16, torch.float32, torch.float16),
|
||||
(897, 9, torch.bfloat16, torch.bfloat16),
|
||||
(897, 16, torch.float16, torch.float32),
|
||||
(1024, 9, torch.float32, torch.bfloat16),
|
||||
(1024, 16, torch.bfloat16, torch.float16),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"scoring_func,renormalize,routed_scaling_factor",
|
||||
[
|
||||
("sigmoid", True, 1.0),
|
||||
("sigmoid", False, 2.5),
|
||||
("softmax", True, 2.5),
|
||||
("softmax", False, 1.0),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
input_dtype: torch.dtype,
|
||||
bias_dtype: torch.dtype,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
):
|
||||
set_random_seed(7)
|
||||
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,scoring_func",
|
||||
[
|
||||
(128, 8, "sigmoid"),
|
||||
(129, 8, "sigmoid"),
|
||||
(257, 8, "sigmoid"),
|
||||
(385, 8, "sigmoid"),
|
||||
(512, 9, "sigmoid"),
|
||||
(513, 9, "sigmoid"),
|
||||
(769, 9, "sigmoid"),
|
||||
(897, 16, "sigmoid"),
|
||||
(1024, 16, "sigmoid"),
|
||||
(128, 4, "softmax"),
|
||||
(128, 5, "softmax"),
|
||||
(129, 8, "softmax"),
|
||||
(161, 8, "softmax"),
|
||||
(256, 9, "softmax"),
|
||||
(257, 8, "softmax"),
|
||||
(512, 9, "softmax"),
|
||||
(512, 17, "softmax"),
|
||||
(512, 23, "softmax"),
|
||||
(513, 8, "softmax"),
|
||||
(577, 9, "softmax"),
|
||||
(769, 9, "softmax"),
|
||||
(897, 9, "softmax"),
|
||||
(1024, 16, "softmax"),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_capacity_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
scoring_func: str,
|
||||
):
|
||||
set_random_seed(11)
|
||||
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
def test_grouped_topk_single_group_stable_ties(num_experts: int):
|
||||
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
@pytest.mark.parametrize("num_finite", [0, 15])
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
def test_grouped_topk_single_group_nonfinite_scores(
|
||||
num_experts: int, num_finite: int, renormalize: bool
|
||||
):
|
||||
logits = torch.full(
|
||||
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
if num_finite:
|
||||
logits[0, :num_finite] = torch.arange(
|
||||
num_finite, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
logits[0, num_finite] = torch.inf
|
||||
logits[0, num_finite + 1] = -torch.inf
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
if num_finite == 0:
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
if renormalize:
|
||||
expected_values = torch.full(
|
||||
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
else:
|
||||
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
|
||||
else:
|
||||
expected_ids = torch.cat(
|
||||
(
|
||||
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
)[None]
|
||||
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
|
||||
if renormalize:
|
||||
finite_values /= finite_values.sum()
|
||||
finite_values *= 2.5
|
||||
expected_values = torch.cat(
|
||||
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
|
||||
)[None]
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
@@ -269,6 +269,7 @@ def test_moe_align_block_size_with_expert_map(
|
||||
if (experts[k] in local_experts) or not mask_inactive_experts
|
||||
else -1
|
||||
)
|
||||
topk_ids[0, 0] = -1
|
||||
|
||||
actual_sorted_ids, actual_expert_ids, actual_num_tokens = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
|
||||
@@ -0,0 +1,649 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch)."""
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig
|
||||
from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm
|
||||
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
|
||||
|
||||
# Keyed by local (N, K): (cute token counts, dsv3 token counts). 1536x7168 is
|
||||
# the unified shared_gate_up_proj/mla_g_proj entry (dsv3 M1..16).
|
||||
EXPECTED_SELECTIONS = {
|
||||
(1536, 128): (set(), set(range(1, 17))),
|
||||
(3072, 128): (set(), set(range(1, 17))),
|
||||
(1536, 7168): (set(), set(range(1, 17))),
|
||||
(3072, 7168): (set(range(1, 6)), set()),
|
||||
(2112, 7168): (set(), set(range(1, 17))),
|
||||
(2304, 1536): (set(), set(range(1, 17))),
|
||||
(4608, 1536): (set(), set(range(1, 17))),
|
||||
(3584, 7168): ({1}, set(range(2, 9))),
|
||||
(6288, 7168): (set(range(1, 5)), set()),
|
||||
(12448, 7168): (set(range(1, 4)), set()),
|
||||
(7168, 768): (set(), set(range(1, 17))),
|
||||
(7168, 1536): ({1}, set()),
|
||||
(7168, 3072): ({1, 2}, set()),
|
||||
(7168, 3584): ({1, 2}, set()),
|
||||
(7168, 4224): ({1}, set()),
|
||||
(7168, 8448): (set(range(1, 4)), set()),
|
||||
(8448, 7168): ({1, 2}, set()),
|
||||
(16896, 7168): ({1, 2}, set()),
|
||||
(20480, 7168): (set(range(1, 5)), set()),
|
||||
(40960, 7168): (set(range(1, 5)), set()),
|
||||
# TP16.
|
||||
(3216, 7168): (set(range(1, 6)), set(range(9, 16))),
|
||||
(768, 7168): (set(range(1, 5)), set(range(5, 17))),
|
||||
(1152, 1536): ({1}, set(range(2, 17))),
|
||||
(768, 128): (set(), set(range(1, 17))),
|
||||
(7168, 384): (set(), set(range(1, 9))),
|
||||
(4224, 7168): (set(range(1, 4)), set(range(4, 9))),
|
||||
(10240, 7168): (set(range(1, 5)), set()),
|
||||
}
|
||||
|
||||
CUTE_CASES = [
|
||||
(spec.n, spec.k, num_tokens)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, _ in spec.cute_configs
|
||||
]
|
||||
|
||||
RESIDUAL_CUTE_CASES = [
|
||||
(spec.n, spec.k, num_tokens)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, _ in spec.residual_configs
|
||||
]
|
||||
|
||||
EXPECTED_CUTE_CONFIGS = {
|
||||
(3072, 7168, 1): (224, 3, 4, 8),
|
||||
(3072, 7168, 2): (128, 3, 2, 8),
|
||||
(3072, 7168, 3): (128, 2, 1, 8),
|
||||
(3072, 7168, 4): (64, 2, 2, 8),
|
||||
(3072, 7168, 5): (128, 3, 1, 8),
|
||||
(3584, 7168, 1): (224, 2, 4, 8),
|
||||
(6288, 7168, 1): (224, 3, 4, 8),
|
||||
(6288, 7168, 2): (64, 3, 2, 8),
|
||||
(6288, 7168, 3): (32, 3, 4, 8),
|
||||
(6288, 7168, 4): (128, 6, 1, 8),
|
||||
(12448, 7168, 1): (224, 4, 2, 8),
|
||||
(12448, 7168, 2): (64, 4, 2, 8),
|
||||
(12448, 7168, 3): (64, 2, 2, 8),
|
||||
(7168, 1536, 1): (96, 4, 2, 8),
|
||||
(7168, 3072, 1): (96, 2, 4, 8),
|
||||
(7168, 3072, 2): (32, 4, 4, 8),
|
||||
(7168, 3584, 1): (224, 4, 2, 8),
|
||||
(7168, 3584, 2): (64, 4, 2, 8),
|
||||
(7168, 4224, 1): (96, 4, 2, 4),
|
||||
(7168, 8448, 1): (32, 4, 4, 8),
|
||||
(7168, 8448, 2): (96, 4, 1, 8),
|
||||
(7168, 8448, 3): (96, 4, 1, 8),
|
||||
(8448, 7168, 1): (224, 3, 4, 8),
|
||||
(8448, 7168, 2): (32, 4, 4, 8),
|
||||
(16896, 7168, 1): (224, 6, 4, 8),
|
||||
(16896, 7168, 2): (32, 4, 4, 8),
|
||||
(20480, 7168, 1): (224, 4, 2, 8),
|
||||
(20480, 7168, 2): (64, 4, 2, 8),
|
||||
(20480, 7168, 3): (64, 2, 2, 8),
|
||||
(20480, 7168, 4): (64, 4, 1, 8),
|
||||
(40960, 7168, 1): (128, 4, 2, 8),
|
||||
(40960, 7168, 2): (64, 4, 2, 8),
|
||||
(40960, 7168, 3): (64, 2, 2, 8),
|
||||
(40960, 7168, 4): (64, 4, 1, 8),
|
||||
# TP16.
|
||||
(3216, 7168, 1): (224, 3, 4, 8),
|
||||
(3216, 7168, 2): (128, 4, 2, 8),
|
||||
(3216, 7168, 3): (128, 2, 1, 8),
|
||||
(3216, 7168, 4): (64, 2, 2, 8),
|
||||
(3216, 7168, 5): (128, 3, 1, 8),
|
||||
(768, 7168, 1): (224, 2, 4, 8),
|
||||
(768, 7168, 2): (224, 2, 2, 8),
|
||||
(768, 7168, 3): (224, 2, 2, 8),
|
||||
(768, 7168, 4): (224, 2, 2, 8),
|
||||
(1152, 1536, 1): (192, 3, 4, 8),
|
||||
(4224, 7168, 1): (224, 3, 4, 8),
|
||||
(4224, 7168, 2): (128, 2, 1, 8),
|
||||
(4224, 7168, 3): (64, 2, 2, 8),
|
||||
(10240, 7168, 1): (224, 4, 2, 8),
|
||||
(10240, 7168, 2): (32, 2, 4, 8),
|
||||
(10240, 7168, 3): (64, 4, 1, 8),
|
||||
(10240, 7168, 4): (64, 4, 1, 8),
|
||||
}
|
||||
|
||||
EXPECTED_RESIDUAL_CUTE_CONFIGS = {
|
||||
(7168, 3584, 1): (64, 4, 2, 8),
|
||||
(7168, 3584, 2): (64, 7, 2, 8),
|
||||
(7168, 3584, 3): (64, 2, 1, 8),
|
||||
(7168, 3584, 4): (64, 2, 1, 8),
|
||||
}
|
||||
|
||||
|
||||
def _config_tuple(config) -> tuple[int, int, int, int]:
|
||||
return (
|
||||
config.block_size,
|
||||
config.outputs_per_block,
|
||||
config.k_unroll,
|
||||
config.vector_width,
|
||||
)
|
||||
|
||||
|
||||
def test_table_is_keyed_by_shape() -> None:
|
||||
for (n, k), spec in k3_gemm.KIMI_K3_PROJECTIONS.items():
|
||||
assert (spec.n, spec.k) == (n, k)
|
||||
|
||||
|
||||
def test_every_dsv3_routed_shape_is_instantiated() -> None:
|
||||
"""dsv3_fused_a_gemm specializes on (K, N); an unlisted shape raises.
|
||||
|
||||
The table routes by shape while the kernel is built per shape, so a missing
|
||||
instantiation only shows up at the token counts that route to dsv3. Checking
|
||||
it here needs no GPU, which is the point -- a GPU-only check is exactly what
|
||||
let (3216, 7168) ship without its DISPATCH_DSV3_SHAPE(7168, 3216).
|
||||
"""
|
||||
source = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "csrc"
|
||||
/ "libtorch_stable"
|
||||
/ "dsv3_fused_a_gemm.cu"
|
||||
).read_text(encoding="utf-8")
|
||||
# Benchmark-only shapes live behind VLLM_K3_BENCH_SHAPES and are not built
|
||||
# by default, so they must not count as available.
|
||||
production_macros = source.split("#ifdef VLLM_K3_BENCH_SHAPES")[0]
|
||||
explicit = source.split("#undef DISPATCH_DSV3_SHAPE")[1].split(
|
||||
"#ifdef VLLM_K3_BENCH_SHAPES"
|
||||
)[0]
|
||||
compiled = {
|
||||
(int(hd_in), int(hd_out))
|
||||
for hd_in, hd_out in re.findall(
|
||||
r"DISPATCH_DSV3_SHAPE\((\d+),\s*(\d+)\)", production_macros
|
||||
)
|
||||
} | {
|
||||
(int(hd_in), int(hd_out))
|
||||
for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit)
|
||||
}
|
||||
assert compiled, "failed to parse the dispatch list"
|
||||
|
||||
missing = sorted(
|
||||
(spec.n, spec.k)
|
||||
for spec in KIMI_K3_PROJECTIONS.values()
|
||||
if spec.dsv3_tokens and (spec.k, spec.n) not in compiled
|
||||
)
|
||||
assert not missing, (
|
||||
f"routed to dsv3 with no instantiation: {missing}; add "
|
||||
"DISPATCH_DSV3_SHAPE(K, N) for each"
|
||||
)
|
||||
|
||||
|
||||
def test_packed_row_major_rejects_single_row_slice() -> None:
|
||||
packed = torch.empty(1, 128)
|
||||
sliced = torch.empty(1, 144)[:, :128]
|
||||
|
||||
assert packed.is_contiguous()
|
||||
assert sliced.is_contiguous()
|
||||
assert k3_gemm._is_packed_row_major(packed)
|
||||
assert not k3_gemm._is_packed_row_major(sliced)
|
||||
|
||||
|
||||
def test_cute_configs_match_measured_table() -> None:
|
||||
actual = {
|
||||
(spec.n, spec.k, num_tokens): _config_tuple(config)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, config in spec.cute_configs
|
||||
}
|
||||
assert actual == EXPECTED_CUTE_CONFIGS
|
||||
|
||||
|
||||
def test_residual_cute_configs_match_measured_table() -> None:
|
||||
actual = {
|
||||
(spec.n, spec.k, num_tokens): _config_tuple(config)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, config in spec.residual_configs
|
||||
}
|
||||
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
|
||||
def test_sm103_selector_table(key: tuple[int, int]) -> None:
|
||||
n, k = key
|
||||
cute_tokens, dsv3_tokens = EXPECTED_SELECTIONS[key]
|
||||
for num_tokens in range(1, 17):
|
||||
backend = k3_gemm.select_kimi_k3_backend(num_tokens, n, k)
|
||||
if num_tokens in cute_tokens:
|
||||
assert backend == "cute"
|
||||
elif num_tokens in dsv3_tokens:
|
||||
assert backend == "dsv3_fused_a"
|
||||
else:
|
||||
assert backend is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
|
||||
def test_selector_requires_supported_shape_and_tokens(key: tuple[int, int]) -> None:
|
||||
n, k = key
|
||||
assert k3_gemm.select_kimi_k3_backend(0, n, k) is None
|
||||
assert k3_gemm.select_kimi_k3_backend(17, n, k) is None
|
||||
assert k3_gemm.select_kimi_k3_backend(1, n + 1, k) is None
|
||||
assert k3_gemm.select_kimi_k3_backend(1, n, k + 1) is None
|
||||
|
||||
|
||||
def test_unlisted_shape_and_unselected_tokens_fall_back() -> None:
|
||||
# Shape absent from the table.
|
||||
assert k3_gemm.select_kimi_k3_backend(1, 1000, 1000) is None
|
||||
# o_proj (7168,1536) is CuTe M1 only; M2+ falls back.
|
||||
assert k3_gemm.select_kimi_k3_backend(2, 7168, 1536) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", range(1, 17))
|
||||
def test_sm103_residual_selector_table(num_tokens: int) -> None:
|
||||
backend = k3_gemm.select_kimi_k3_backend(num_tokens, 7168, 3584, has_residual=True)
|
||||
assert backend == ("cute" if num_tokens <= 4 else None)
|
||||
|
||||
|
||||
def test_build_plan_matches_selector() -> None:
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values():
|
||||
plan = k3_gemm._build_plan(spec)
|
||||
for num_tokens in range(1, 17):
|
||||
backend = k3_gemm.select_kimi_k3_backend(num_tokens, spec.n, spec.k)
|
||||
if backend is None:
|
||||
assert num_tokens not in plan
|
||||
else:
|
||||
assert plan[num_tokens][0] == backend
|
||||
|
||||
|
||||
def test_installation_is_shape_specific_and_unquantized(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class FakeLinear(nn.Module):
|
||||
def __init__(self, quant_method: object, n: int, k: int) -> None:
|
||||
super().__init__()
|
||||
self.quant_method = quant_method
|
||||
self.weight = torch.empty(n, k)
|
||||
|
||||
class FakeHead(nn.Module):
|
||||
def __init__(self, n: int, k: int) -> None:
|
||||
super().__init__()
|
||||
self.quant_method = k3_gemm.UnquantizedEmbeddingMethod()
|
||||
self.weight = torch.empty(n, k)
|
||||
|
||||
root = nn.Module()
|
||||
# dsv3-only shape (no cute warmup contribution).
|
||||
root.dsv3_only = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 2304, 1536)
|
||||
# quantized: must be left untouched.
|
||||
quantized_method = object()
|
||||
root.quantized = FakeLinear(quantized_method, 6288, 7168)
|
||||
# cute shape.
|
||||
root.cute = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 6288, 7168)
|
||||
# cute + residual shape.
|
||||
root.residual = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 7168, 3584)
|
||||
# shape absent from the table: must be left untouched.
|
||||
root.unlisted = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 1234, 5678)
|
||||
root.lm_head = FakeHead(20480, 7168)
|
||||
|
||||
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
|
||||
monkeypatch.setattr(k3_gemm, "ParallelLMHead", FakeHead)
|
||||
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: True)
|
||||
warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
residual_warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
monkeypatch.setattr(k3_gemm.shape_dynamic_skinny_gemm, "is_available", lambda: True)
|
||||
|
||||
def request_warmup_configs(dtype, configs, *, has_residual=False):
|
||||
target = residual_warmup_configs if has_residual else warmup_configs
|
||||
target.update(configs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
k3_gemm.shape_dynamic_skinny_gemm,
|
||||
"request_warmup_configs",
|
||||
request_warmup_configs,
|
||||
)
|
||||
|
||||
k3_gemm.enable_kimi_k3_low_latency_gemm(root, torch.bfloat16)
|
||||
|
||||
assert isinstance(root.dsv3_only.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
|
||||
assert isinstance(root.cute.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
|
||||
assert isinstance(root.residual.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
|
||||
assert root.quantized.quant_method is quantized_method
|
||||
assert type(root.unlisted.quant_method) is k3_gemm.UnquantizedLinearMethod
|
||||
assert isinstance(
|
||||
root.lm_head.quant_method, k3_gemm.KimiK3LowLatencyEmbeddingMethod
|
||||
)
|
||||
# Warmup covers only the installed modules' local (N, K).
|
||||
assert warmup_configs == {
|
||||
config
|
||||
for key in ((6288, 7168), (7168, 3584), (20480, 7168))
|
||||
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[key].cute_configs
|
||||
}
|
||||
assert residual_warmup_configs == {
|
||||
config
|
||||
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)].residual_configs
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dtype,platform_enabled",
|
||||
[(torch.float16, True), (torch.bfloat16, False)],
|
||||
)
|
||||
def test_installation_requires_bf16_sm103(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dtype: torch.dtype,
|
||||
platform_enabled: bool,
|
||||
) -> None:
|
||||
class FakeLinear(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.quant_method = k3_gemm.UnquantizedLinearMethod()
|
||||
self.weight = torch.empty(2304, 1536)
|
||||
|
||||
root = nn.Module()
|
||||
root.projection = FakeLinear()
|
||||
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
|
||||
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: platform_enabled)
|
||||
|
||||
k3_gemm.enable_kimi_k3_low_latency_gemm(root, dtype)
|
||||
|
||||
assert type(root.projection.quant_method) is k3_gemm.UnquantizedLinearMethod
|
||||
|
||||
|
||||
def _require_sm103_and_dsv3() -> None:
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
|
||||
pytest.skip("Kimi-K3 production selection requires SM103")
|
||||
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
|
||||
pytest.skip("dsv3_fused_a_gemm was not built")
|
||||
|
||||
|
||||
def _require_sm103_and_cute() -> None:
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
|
||||
pytest.skip("Kimi-K3 production selection requires SM103")
|
||||
if not k3_gemm.shape_dynamic_skinny_gemm.is_available():
|
||||
pytest.skip("CuTe DSL is not available")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES)
|
||||
def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
output = k3_gemm.try_low_latency_gemm(x, weight)
|
||||
|
||||
assert output is not None
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def _dsv3_probe_tokens(tokens: frozenset[int]) -> set[int]:
|
||||
"""Extremes, plus both sides of the kernel's num_tokens<=8 tile_n branch."""
|
||||
if not tokens:
|
||||
return set()
|
||||
return {min(tokens), max(tokens)} | ({8, 9} & set(tokens))
|
||||
|
||||
|
||||
# Derived from the table rather than hand-listed, so a shape routed to dsv3
|
||||
# cannot be added without being exercised here.
|
||||
DSV3_CASES = sorted(
|
||||
(num_tokens, spec.n, spec.k)
|
||||
for spec in KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES)
|
||||
def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
|
||||
_require_sm103_and_dsv3()
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
|
||||
assert num_tokens in spec.dsv3_tokens
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
output = k3_gemm.try_low_latency_gemm(x, weight)
|
||||
|
||||
assert output is not None
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def test_nonpacked_single_token_dsv3_falls_back() -> None:
|
||||
_require_sm103_and_dsv3()
|
||||
n, k = 1536, 128
|
||||
storage = torch.randn(1, k + 16, dtype=torch.bfloat16, device="cuda")
|
||||
x = storage[:, :k]
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod(
|
||||
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
|
||||
)
|
||||
|
||||
assert x.is_contiguous()
|
||||
assert x.stride() == (k + 16, 1)
|
||||
assert not k3_gemm._runtime_ok(x, weight) # strict guard rejects the view
|
||||
output = method.apply(SimpleNamespace(weight=weight), x)
|
||||
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
torch.testing.assert_close(output, reference)
|
||||
|
||||
|
||||
def test_selected_kernels_cuda_graph_capture() -> None:
|
||||
_require_sm103_and_cute()
|
||||
_require_sm103_and_dsv3()
|
||||
cute_spec = k3_gemm.KIMI_K3_PROJECTIONS[(6288, 7168)]
|
||||
dsv3_spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
|
||||
cute_x = torch.randn(1, cute_spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
cute_weight = torch.randn(
|
||||
cute_spec.n, cute_spec.k, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
dsv3_x = torch.randn(1, dsv3_spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
dsv3_weight = torch.randn(
|
||||
dsv3_spec.n, dsv3_spec.k, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
|
||||
k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
cute_output = k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
|
||||
dsv3_output = k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert cute_output is not None
|
||||
assert dsv3_output is not None
|
||||
for output, activation, weight in (
|
||||
(cute_output, cute_x, cute_weight),
|
||||
(dsv3_output, dsv3_x, dsv3_weight),
|
||||
):
|
||||
reference = torch.nn.functional.linear(activation, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 9, 16])
|
||||
def test_dsv3_cuda_graph_capture_tile_branches(num_tokens: int) -> None:
|
||||
"""Capture DSV3 across the num_tokens<=8 vs >8 tile_n branch."""
|
||||
_require_sm103_and_dsv3()
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
|
||||
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
k3_gemm.try_low_latency_gemm(x, weight)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = k3_gemm.try_low_latency_gemm(x, weight)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert output is not None
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens", RESIDUAL_CUTE_CASES)
|
||||
def test_cute_residual_epilogue(n: int, k: int, num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
torch.manual_seed(42 + num_tokens)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
|
||||
config = spec.residual_config(num_tokens)
|
||||
assert config is not None
|
||||
|
||||
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
|
||||
reference = x.float() @ weight.float().t() + residual.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", range(1, 17))
|
||||
def test_cute_residual_epilogue_all_supported_token_counts(num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
ShapeDynamicSkinnyGemm,
|
||||
)
|
||||
|
||||
n, k = 64, 512
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
|
||||
config = ShapeDynamicSkinnyGemm._config(num_tokens, n, k)
|
||||
|
||||
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
|
||||
reference = x.float() @ weight.float().t() + residual.float()
|
||||
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", range(1, 5))
|
||||
def test_cute_residual_epilogue_cuda_graph_capture(num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)]
|
||||
config = spec.residual_config(num_tokens)
|
||||
assert config is not None
|
||||
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(num_tokens, spec.n, dtype=torch.bfloat16, device="cuda")
|
||||
k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
reference = x.float() @ weight.float().t() + residual.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
class _SkinnyGemmSpy:
|
||||
"""Wraps the skinny-GEMM singleton to record whether CuTe was invoked."""
|
||||
|
||||
def __init__(self, real: Any) -> None:
|
||||
self._real = real
|
||||
self.calls: list[int] = []
|
||||
|
||||
def __call__(self, a, b, config=None, residual=None):
|
||||
self.calls.append(a.shape[0])
|
||||
return self._real(a, b, config, residual)
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._real.is_available()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 3, 4])
|
||||
def test_latent_moe_production_layout_residual(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
"""The real Latent-MoE residual is a non-packed slice of a cat buffer.
|
||||
|
||||
The strict packed-row-major guard rejects such a slice at every token count
|
||||
(a size-1 leading dim reads as contiguous but its stride is not packed), so
|
||||
the CuTe residual epilogue never fires for this production layout and the
|
||||
method falls back to addmm. Output is correct regardless of the path.
|
||||
"""
|
||||
_require_sm103_and_cute()
|
||||
latent_dim, shared_dim = 3584, 7168 # routed_expert_up_proj K, N
|
||||
torch.manual_seed(7 + num_tokens)
|
||||
buf = torch.randn(
|
||||
num_tokens, latent_dim + shared_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
latent = buf[:, :latent_dim] # non-contiguous view (row stride = full width)
|
||||
residual = buf[:, latent_dim:] # non-contiguous view
|
||||
weight = torch.randn(shared_dim, latent_dim, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(shared_dim, latent_dim)]
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod(
|
||||
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
|
||||
)
|
||||
spy = _SkinnyGemmSpy(k3_gemm.shape_dynamic_skinny_gemm)
|
||||
monkeypatch.setattr(k3_gemm, "shape_dynamic_skinny_gemm", spy)
|
||||
|
||||
layer = SimpleNamespace(weight=weight)
|
||||
output = method.apply_with_residual(layer, latent, residual)
|
||||
|
||||
reference = latent.float() @ weight.float().t() + residual.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999 # correct regardless of the path taken
|
||||
assert not spy.calls, (
|
||||
"non-packed buf-slice residual must fall back to addmm at every M"
|
||||
)
|
||||
|
||||
|
||||
def test_residual_dispatch_falls_back_to_addmm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fallback = torch.randn(2, 3)
|
||||
residual = torch.randn(2, 3)
|
||||
x = torch.randn(2, 4)
|
||||
weight = torch.randn(3, 4)
|
||||
monkeypatch.setattr(torch, "addmm", lambda *args: fallback)
|
||||
# CPU tensors fail the runtime check, forcing the addmm fallback.
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
|
||||
|
||||
output = method.apply_with_residual(SimpleNamespace(weight=weight), x, residual)
|
||||
|
||||
assert output is fallback
|
||||
|
||||
|
||||
def test_fallback_preserves_default_method(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
fallback = torch.empty(2, 8)
|
||||
monkeypatch.setattr(
|
||||
k3_gemm.UnquantizedLinearMethod,
|
||||
"apply",
|
||||
lambda *args: fallback,
|
||||
)
|
||||
# 1-D input fails the runtime check, forcing the base-method fallback.
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
|
||||
|
||||
output = method.apply(
|
||||
SimpleNamespace(weight=torch.empty(0)),
|
||||
torch.empty(0),
|
||||
)
|
||||
|
||||
assert output is fallback
|
||||
@@ -1,226 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Precision tests for vllm's chunk_kda Triton operator.
|
||||
|
||||
Compares chunk_kda against a naive recurrent reference (float32).
|
||||
Uses torch.rand for q/k/v to match FLA's test pattern.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import (
|
||||
chunk_kda,
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_kda_gate,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
|
||||
def naive_recurrent_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
|
||||
dtype = v.dtype
|
||||
B, T, H, K = q.shape
|
||||
V = v.shape[-1]
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
|
||||
q = q * scale
|
||||
|
||||
S = k.new_zeros(B, H, K, V).to(q)
|
||||
if initial_state is not None:
|
||||
S += initial_state
|
||||
o = torch.zeros_like(v)
|
||||
for i in range(T):
|
||||
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
|
||||
S = S * g_i[..., None].exp()
|
||||
S = S + torch.einsum(
|
||||
"bhk,bhv->bhkv",
|
||||
b_i[..., None] * k_i,
|
||||
v_i - (k_i[..., None] * S).sum(-2),
|
||||
)
|
||||
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
|
||||
if not output_final_state:
|
||||
S = None
|
||||
return o.to(dtype), S
|
||||
|
||||
|
||||
def assert_close(
|
||||
name: str,
|
||||
ref: torch.Tensor,
|
||||
tri: torch.Tensor,
|
||||
ratio: float,
|
||||
err_atol: float = 1e-6,
|
||||
):
|
||||
"""RMSE-based relative error comparison."""
|
||||
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
|
||||
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
|
||||
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
|
||||
rel_err = rmse_diff / (rmse_base + 1e-8)
|
||||
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
|
||||
if abs_err <= err_atol:
|
||||
return
|
||||
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
|
||||
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
|
||||
assert rel_err < ratio, (
|
||||
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("H", "D", "cu_seqlens", "dtype"),
|
||||
[
|
||||
pytest.param(
|
||||
*test,
|
||||
id="H{}-D{}-cu{}-{}".format(*test),
|
||||
)
|
||||
for test in [
|
||||
(32, 128, [0, 64], torch.float16),
|
||||
(32, 128, [0, 1024], torch.float16),
|
||||
(32, 128, [0, 15], torch.float16),
|
||||
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
|
||||
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
|
||||
(64, 128, [0, 256, 500, 1000], torch.float16),
|
||||
(32, 128, [0, 8192], torch.float16),
|
||||
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
|
||||
]
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda(
|
||||
H: int,
|
||||
D: int,
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
T = cu_seqlens[-1]
|
||||
torch.manual_seed(42)
|
||||
B = 1
|
||||
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
|
||||
N = len(cu_seqlens) - 1
|
||||
|
||||
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
|
||||
dtype
|
||||
)
|
||||
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
|
||||
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
|
||||
ref_outputs = []
|
||||
ref_states = []
|
||||
for i in range(N):
|
||||
s, e = cu_seqlens[i], cu_seqlens[i + 1]
|
||||
q_i = l2norm_fwd(q[:, s:e].contiguous())
|
||||
k_i = l2norm_fwd(k[:, s:e].contiguous())
|
||||
o_i, ht_i = naive_recurrent_kda(
|
||||
q_i,
|
||||
k_i,
|
||||
v[:, s:e],
|
||||
g[:, s:e],
|
||||
beta[:, s:e],
|
||||
initial_state=h0[i],
|
||||
output_final_state=True,
|
||||
)
|
||||
ref_outputs.append(o_i)
|
||||
ref_states.append(ht_i)
|
||||
ref_o = torch.cat(ref_outputs, dim=1)
|
||||
ref_ht = torch.cat(ref_states, dim=0)
|
||||
|
||||
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
|
||||
tri_o, tri_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=g.clone(),
|
||||
beta=beta.clone(),
|
||||
initial_state=h0.transpose(-1, -2).contiguous().clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
|
||||
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
|
||||
assert_close("o", ref_o, tri_o, 0.005)
|
||||
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cu_seqlens", "dtype"),
|
||||
[
|
||||
([0, 64], torch.float16),
|
||||
([0, 15, 100, 300], torch.bfloat16),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
H, D = 8, 64
|
||||
T = cu_seqlens[-1]
|
||||
N = len(cu_seqlens) - 1
|
||||
torch.manual_seed(123)
|
||||
|
||||
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
|
||||
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
beta = torch.rand(1, T, H, dtype=dtype, device=DEVICE).sigmoid()
|
||||
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
|
||||
dt_bias = (
|
||||
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
).contiguous()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
initial_state = h0.transpose(-1, -2).contiguous()
|
||||
|
||||
gate = fused_kda_gate(
|
||||
raw_g.reshape(T, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
).unsqueeze(0)
|
||||
old_o, old_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=gate,
|
||||
beta=beta.clone(),
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
new_o, new_ht = chunk_kda_with_fused_gate(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
raw_g=raw_g,
|
||||
beta=beta.clone(),
|
||||
A_log=A_log,
|
||||
g_bias=dt_bias,
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
|
||||
Reference in New Issue
Block a user