[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
@@ -9,6 +11,8 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
|
||||
from vllm.distributed.device_communicators import custom_all_reduce
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
|
||||
from vllm.distributed.parallel_state import get_tp_group, graph_capture
|
||||
|
||||
from ..utils import (
|
||||
@@ -23,6 +27,59 @@ for i, v in enumerate(test_sizes):
|
||||
test_sizes[i] -= v % 8
|
||||
|
||||
|
||||
def test_sp16_dispatches_only_to_mnnvl_lamport(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""SP16 uses the MNNVL Lamport kernels and rejects same-host dispatch."""
|
||||
comm = CustomAllreduce.__new__(CustomAllreduce)
|
||||
comm.disabled = False
|
||||
comm.world_size = 16
|
||||
comm.fully_connected = False
|
||||
comm.mnnvl_only = True
|
||||
comm._IS_CAPTURING = False
|
||||
comm.max_mnnvl_all_gather_size = 2 * 1024 * 1024
|
||||
comm.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
|
||||
comm.mnnvl_multicast_ptr = 1
|
||||
comm.mnnvl_lamport_ag_local_ptr = 1
|
||||
comm.mnnvl_lamport_ag_multicast_ptr = 1
|
||||
comm.mnnvl_lamport_ag_epoch_ptr = 1
|
||||
comm.mnnvl_lamport_rs_local_ptr = 1
|
||||
comm.mnnvl_lamport_rs_epoch_ptr = 1
|
||||
comm.mnnvl_buffer_size = 32 * 1024 * 1024
|
||||
comm._ptr = 0
|
||||
|
||||
lamport_all_gather = Mock()
|
||||
lamport_reduce_scatter = Mock()
|
||||
monkeypatch.setattr(custom_all_reduce.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
custom_all_reduce,
|
||||
"ops",
|
||||
SimpleNamespace(
|
||||
mnnvl_lamport_all_gather=lamport_all_gather,
|
||||
mnnvl_lamport_reduce_scatter=lamport_reduce_scatter,
|
||||
),
|
||||
)
|
||||
|
||||
gathered = comm.custom_all_gather(torch.empty((8, 8), dtype=torch.bfloat16))
|
||||
scattered = comm.custom_reduce_scatter(torch.empty((16, 8), dtype=torch.bfloat16))
|
||||
|
||||
assert gathered is not None
|
||||
assert scattered is not None
|
||||
lamport_all_gather.assert_called_once()
|
||||
lamport_reduce_scatter.assert_called_once()
|
||||
assert not comm.should_custom_ar(torch.empty(8, dtype=torch.bfloat16))
|
||||
assert not comm.should_custom_all_gather(torch.empty((8, 8), dtype=torch.int32))
|
||||
assert not comm.should_custom_all_gather(
|
||||
torch.empty((131073, 8), dtype=torch.bfloat16)
|
||||
)
|
||||
|
||||
comm.mnnvl_only = False
|
||||
assert not comm.should_custom_all_gather(torch.empty((8, 8), dtype=torch.bfloat16))
|
||||
assert not comm.should_custom_reduce_scatter(
|
||||
torch.empty((16, 8), dtype=torch.bfloat16)
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -80,6 +137,32 @@ def graph_allreduce(
|
||||
torch.testing.assert_close(out1, inp1)
|
||||
torch.testing.assert_close(out2, inp2)
|
||||
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
tp_rank = rank % tp_size
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
local = torch.full(
|
||||
(512, 4096), tp_rank + 1, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
reduce_input = torch.full(
|
||||
(512 * tp_size, 4096),
|
||||
tp_rank + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
gathered = fa.custom_all_gather(local)
|
||||
scattered = fa.custom_reduce_scatter(reduce_input)
|
||||
graph.replay()
|
||||
assert gathered is not None
|
||||
assert scattered is not None
|
||||
expected_gather = torch.cat(
|
||||
[torch.full_like(local, peer_rank + 1) for peer_rank in range(tp_size)]
|
||||
)
|
||||
expected_scatter = torch.full_like(local, tp_size * (tp_size + 1) // 2)
|
||||
torch.testing.assert_close(gathered, expected_gather)
|
||||
torch.testing.assert_close(scattered, expected_scatter)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def eager_allreduce(
|
||||
@@ -110,6 +193,29 @@ def eager_allreduce(
|
||||
out = fa.all_reduce(out, registered=False)
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
group = get_tp_group().device_group
|
||||
tp_rank = rank % tp_size
|
||||
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
|
||||
local = torch.full((64, 4096), tp_rank + 1, dtype=dtype, device=device)
|
||||
expected_gather = torch.empty(
|
||||
(64 * tp_size, 4096), dtype=dtype, device=device
|
||||
)
|
||||
dist.all_gather_into_tensor(expected_gather, local, group=group)
|
||||
gathered = fa.custom_all_gather(local)
|
||||
assert gathered is not None
|
||||
torch.testing.assert_close(gathered, expected_gather)
|
||||
|
||||
reduce_input = torch.full(
|
||||
(64 * tp_size, 4096), tp_rank + 1, dtype=dtype, device=device
|
||||
)
|
||||
expected_scatter = torch.empty((64, 4096), dtype=dtype, device=device)
|
||||
dist.reduce_scatter_tensor(
|
||||
expected_scatter, reduce_input.clone(), group=group
|
||||
)
|
||||
scattered = fa.custom_reduce_scatter(reduce_input)
|
||||
assert scattered is not None
|
||||
torch.testing.assert_close(scattered, expected_scatter)
|
||||
|
||||
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
|
||||
out = inp
|
||||
for _ in range(num_communication):
|
||||
@@ -130,3 +236,14 @@ def test_custom_allreduce(
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
|
||||
def test_custom_collectives_world_size_four(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
test_target,
|
||||
):
|
||||
"""Exercise the four-rank kernel specialization used by Kimi SP."""
|
||||
if torch.accelerator.device_count() < 4:
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, 4, 1, test_target)
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
|
||||
)
|
||||
|
||||
_DTYPE = torch.bfloat16
|
||||
_NUM_TOKENS = 3
|
||||
_NUM_HEADS = 4
|
||||
_BLOCK_SIZE = 8
|
||||
_POSITIONS = (1, 7, 13)
|
||||
_SLOTS = (0, 3, 9)
|
||||
|
||||
|
||||
def _randn(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=_DTYPE) * 0.2
|
||||
|
||||
|
||||
def _rope_cache(max_position: int = 32) -> torch.Tensor:
|
||||
inv_freq = 1.0 / (
|
||||
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
|
||||
)
|
||||
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
|
||||
freqs = torch.outer(positions, inv_freq)
|
||||
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
|
||||
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
|
||||
|
||||
|
||||
def _apply_gptj_rope(
|
||||
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
|
||||
for _ in range(x.ndim - 2):
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
x1 = x[..., ::2].float()
|
||||
x2 = x[..., 1::2].float()
|
||||
out1 = x1 * cos.float() - x2 * sin.float()
|
||||
out2 = x2 * cos.float() + x1 * sin.float()
|
||||
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
|
||||
|
||||
|
||||
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
|
||||
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
|
||||
|
||||
|
||||
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
torch.testing.assert_close(
|
||||
actual.float(),
|
||||
expected.to(torch.float8_e4m3fn).float(),
|
||||
atol=0.03125,
|
||||
rtol=0.15,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(0)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
|
||||
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
|
||||
|
||||
q_expected = q.clone()
|
||||
q_expected[..., 128:] = _apply_gptj_rope(
|
||||
q_expected[..., 128:], positions, cos_sin_cache
|
||||
)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
k_expected = torch.cat(
|
||||
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
|
||||
)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_kv_cache_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
q,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
v,
|
||||
cache,
|
||||
slots,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
one,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(k_actual, k_expected)
|
||||
_assert_fp8_close(v_actual, v)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = q.clone()
|
||||
k_actual = fused_mla_key_concat_ds_mla_insert(
|
||||
q_actual,
|
||||
k_nope,
|
||||
k_pe,
|
||||
kv_c,
|
||||
cache,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(k_actual, k_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
torch.manual_seed(1)
|
||||
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
cos_sin_cache = _rope_cache()
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
|
||||
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
|
||||
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
|
||||
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
|
||||
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
|
||||
|
||||
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
|
||||
if cache_kind == "bf16":
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
|
||||
)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
|
||||
elif cache_kind == "fp8":
|
||||
cache = torch.zeros(
|
||||
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
|
||||
)
|
||||
one = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope,
|
||||
q_pe,
|
||||
kv_c,
|
||||
k_pe,
|
||||
cache,
|
||||
slots,
|
||||
q_scale_inv=one,
|
||||
cache_scale_inv=one,
|
||||
**kwargs,
|
||||
)
|
||||
_assert_fp8_close(q_actual, q_expected)
|
||||
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
|
||||
else:
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
|
||||
)
|
||||
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
|
||||
torch.testing.assert_close(q_actual, q_expected)
|
||||
torch.testing.assert_close(rope_cache, k_pe_expected)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_decode_epilogue_preserves_nope_path() -> None:
|
||||
torch.manual_seed(2)
|
||||
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
|
||||
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
|
||||
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
|
||||
kv_c = _randn(_NUM_TOKENS, 512)
|
||||
k_pe = _randn(_NUM_TOKENS, 64)
|
||||
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
@@ -197,6 +197,46 @@ def test_silu_and_mul_with_clamp(
|
||||
opcheck(torch.ops._C.silu_and_mul_with_clamp, (out_buf, x, swiglu_limit))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_masked_situ_and_mul(
|
||||
default_vllm_config,
|
||||
linear_beta: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Masked SITU computes valid expert rows and preserves padded zeros."""
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts, max_num_tokens, d = 4, 7, 512
|
||||
beta = 1.5
|
||||
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
|
||||
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
|
||||
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
|
||||
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
output, input, expert_num_tokens, beta, linear_beta
|
||||
)
|
||||
|
||||
gate, up = input.float().chunk(2, dim=-1)
|
||||
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
if linear_beta > 0:
|
||||
up = linear_beta * torch.tanh(up / linear_beta)
|
||||
expected = (expected * up).to(dtype)
|
||||
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
|
||||
torch.testing.assert_close(
|
||||
output[expert, :num_tokens],
|
||||
expected[expert, :num_tokens],
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.masked_situ_and_mul,
|
||||
(output, input, expert_num_tokens, beta, linear_beta),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
|
||||
@@ -7,7 +7,9 @@ matching the eager triton kernel output."""
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.third_party.flash_linear_attention.ops.fused_norm_gate import (
|
||||
FusedRMSNormGated,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.bfloat16]
|
||||
|
||||
@@ -23,6 +23,53 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
def _run_single_group_topk(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return fused_grouped_topk(
|
||||
hidden_states=torch.empty(
|
||||
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
|
||||
),
|
||||
gating_output=logits,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
e_score_correction_bias=bias,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
|
||||
def _single_group_reference(
|
||||
logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if scoring_func == "sigmoid":
|
||||
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
|
||||
else:
|
||||
scores = torch.softmax(logits, dim=-1).float()
|
||||
indices = torch.argsort(
|
||||
scores + bias.float(), dim=-1, descending=True, stable=True
|
||||
)[:, :topk]
|
||||
values = scores.gather(1, indices)
|
||||
if renormalize:
|
||||
values /= values.sum(dim=-1, keepdim=True) + 1e-20
|
||||
values *= routed_scaling_factor
|
||||
return values, indices.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@@ -101,3 +148,222 @@ def test_grouped_topk(
|
||||
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
def test_grouped_topk_single_group_large_batch():
|
||||
set_random_seed(0)
|
||||
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits, bias, 16, scoring_func="sigmoid", renormalize=True
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,input_dtype,bias_dtype",
|
||||
[
|
||||
(512, 9, torch.bfloat16, torch.float32),
|
||||
(512, 16, torch.float16, torch.float16),
|
||||
(513, 9, torch.float32, torch.bfloat16),
|
||||
(513, 16, torch.bfloat16, torch.float32),
|
||||
(895, 9, torch.float16, torch.bfloat16),
|
||||
(896, 16, torch.float32, torch.float16),
|
||||
(897, 9, torch.bfloat16, torch.bfloat16),
|
||||
(897, 16, torch.float16, torch.float32),
|
||||
(1024, 9, torch.float32, torch.bfloat16),
|
||||
(1024, 16, torch.bfloat16, torch.float16),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"scoring_func,renormalize,routed_scaling_factor",
|
||||
[
|
||||
("sigmoid", True, 1.0),
|
||||
("sigmoid", False, 2.5),
|
||||
("softmax", True, 2.5),
|
||||
("softmax", False, 1.0),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
input_dtype: torch.dtype,
|
||||
bias_dtype: torch.dtype,
|
||||
scoring_func: str,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
):
|
||||
set_random_seed(7)
|
||||
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
|
||||
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts,topk,scoring_func",
|
||||
[
|
||||
(128, 8, "sigmoid"),
|
||||
(129, 8, "sigmoid"),
|
||||
(257, 8, "sigmoid"),
|
||||
(385, 8, "sigmoid"),
|
||||
(512, 9, "sigmoid"),
|
||||
(513, 9, "sigmoid"),
|
||||
(769, 9, "sigmoid"),
|
||||
(897, 16, "sigmoid"),
|
||||
(1024, 16, "sigmoid"),
|
||||
(128, 4, "softmax"),
|
||||
(128, 5, "softmax"),
|
||||
(129, 8, "softmax"),
|
||||
(161, 8, "softmax"),
|
||||
(256, 9, "softmax"),
|
||||
(257, 8, "softmax"),
|
||||
(512, 9, "softmax"),
|
||||
(512, 17, "softmax"),
|
||||
(512, 23, "softmax"),
|
||||
(513, 8, "softmax"),
|
||||
(577, 9, "softmax"),
|
||||
(769, 9, "softmax"),
|
||||
(897, 9, "softmax"),
|
||||
(1024, 16, "softmax"),
|
||||
],
|
||||
)
|
||||
def test_grouped_topk_single_group_capacity_tiers(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
scoring_func: str,
|
||||
):
|
||||
set_random_seed(11)
|
||||
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
expected_values, expected_ids = _single_group_reference(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
topk,
|
||||
scoring_func=scoring_func,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
def test_grouped_topk_single_group_stable_ties(num_experts: int):
|
||||
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
|
||||
@pytest.mark.parametrize("num_finite", [0, 15])
|
||||
@pytest.mark.parametrize("renormalize", [False, True])
|
||||
def test_grouped_topk_single_group_nonfinite_scores(
|
||||
num_experts: int, num_finite: int, renormalize: bool
|
||||
):
|
||||
logits = torch.full(
|
||||
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
if num_finite:
|
||||
logits[0, :num_finite] = torch.arange(
|
||||
num_finite, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
logits[0, num_finite] = torch.inf
|
||||
logits[0, num_finite + 1] = -torch.inf
|
||||
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
|
||||
|
||||
actual_values, actual_ids = _run_single_group_topk(
|
||||
logits,
|
||||
bias,
|
||||
16,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=2.5,
|
||||
)
|
||||
|
||||
if num_finite == 0:
|
||||
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
|
||||
if renormalize:
|
||||
expected_values = torch.full(
|
||||
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
else:
|
||||
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
|
||||
else:
|
||||
expected_ids = torch.cat(
|
||||
(
|
||||
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
)[None]
|
||||
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
|
||||
if renormalize:
|
||||
finite_values /= finite_values.sum()
|
||||
finite_values *= 2.5
|
||||
expected_values = torch.cat(
|
||||
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
|
||||
)[None]
|
||||
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
|
||||
|
||||
@@ -269,6 +269,7 @@ def test_moe_align_block_size_with_expert_map(
|
||||
if (experts[k] in local_experts) or not mask_inactive_experts
|
||||
else -1
|
||||
)
|
||||
topk_ids[0, 0] = -1
|
||||
|
||||
actual_sorted_ids, actual_expert_ids, actual_num_tokens = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
|
||||
@@ -0,0 +1,649 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch)."""
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig
|
||||
from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm
|
||||
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
|
||||
|
||||
# Keyed by local (N, K): (cute token counts, dsv3 token counts). 1536x7168 is
|
||||
# the unified shared_gate_up_proj/mla_g_proj entry (dsv3 M1..16).
|
||||
EXPECTED_SELECTIONS = {
|
||||
(1536, 128): (set(), set(range(1, 17))),
|
||||
(3072, 128): (set(), set(range(1, 17))),
|
||||
(1536, 7168): (set(), set(range(1, 17))),
|
||||
(3072, 7168): (set(range(1, 6)), set()),
|
||||
(2112, 7168): (set(), set(range(1, 17))),
|
||||
(2304, 1536): (set(), set(range(1, 17))),
|
||||
(4608, 1536): (set(), set(range(1, 17))),
|
||||
(3584, 7168): ({1}, set(range(2, 9))),
|
||||
(6288, 7168): (set(range(1, 5)), set()),
|
||||
(12448, 7168): (set(range(1, 4)), set()),
|
||||
(7168, 768): (set(), set(range(1, 17))),
|
||||
(7168, 1536): ({1}, set()),
|
||||
(7168, 3072): ({1, 2}, set()),
|
||||
(7168, 3584): ({1, 2}, set()),
|
||||
(7168, 4224): ({1}, set()),
|
||||
(7168, 8448): (set(range(1, 4)), set()),
|
||||
(8448, 7168): ({1, 2}, set()),
|
||||
(16896, 7168): ({1, 2}, set()),
|
||||
(20480, 7168): (set(range(1, 5)), set()),
|
||||
(40960, 7168): (set(range(1, 5)), set()),
|
||||
# TP16.
|
||||
(3216, 7168): (set(range(1, 6)), set(range(9, 16))),
|
||||
(768, 7168): (set(range(1, 5)), set(range(5, 17))),
|
||||
(1152, 1536): ({1}, set(range(2, 17))),
|
||||
(768, 128): (set(), set(range(1, 17))),
|
||||
(7168, 384): (set(), set(range(1, 9))),
|
||||
(4224, 7168): (set(range(1, 4)), set(range(4, 9))),
|
||||
(10240, 7168): (set(range(1, 5)), set()),
|
||||
}
|
||||
|
||||
CUTE_CASES = [
|
||||
(spec.n, spec.k, num_tokens)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, _ in spec.cute_configs
|
||||
]
|
||||
|
||||
RESIDUAL_CUTE_CASES = [
|
||||
(spec.n, spec.k, num_tokens)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, _ in spec.residual_configs
|
||||
]
|
||||
|
||||
EXPECTED_CUTE_CONFIGS = {
|
||||
(3072, 7168, 1): (224, 3, 4, 8),
|
||||
(3072, 7168, 2): (128, 3, 2, 8),
|
||||
(3072, 7168, 3): (128, 2, 1, 8),
|
||||
(3072, 7168, 4): (64, 2, 2, 8),
|
||||
(3072, 7168, 5): (128, 3, 1, 8),
|
||||
(3584, 7168, 1): (224, 2, 4, 8),
|
||||
(6288, 7168, 1): (224, 3, 4, 8),
|
||||
(6288, 7168, 2): (64, 3, 2, 8),
|
||||
(6288, 7168, 3): (32, 3, 4, 8),
|
||||
(6288, 7168, 4): (128, 6, 1, 8),
|
||||
(12448, 7168, 1): (224, 4, 2, 8),
|
||||
(12448, 7168, 2): (64, 4, 2, 8),
|
||||
(12448, 7168, 3): (64, 2, 2, 8),
|
||||
(7168, 1536, 1): (96, 4, 2, 8),
|
||||
(7168, 3072, 1): (96, 2, 4, 8),
|
||||
(7168, 3072, 2): (32, 4, 4, 8),
|
||||
(7168, 3584, 1): (224, 4, 2, 8),
|
||||
(7168, 3584, 2): (64, 4, 2, 8),
|
||||
(7168, 4224, 1): (96, 4, 2, 4),
|
||||
(7168, 8448, 1): (32, 4, 4, 8),
|
||||
(7168, 8448, 2): (96, 4, 1, 8),
|
||||
(7168, 8448, 3): (96, 4, 1, 8),
|
||||
(8448, 7168, 1): (224, 3, 4, 8),
|
||||
(8448, 7168, 2): (32, 4, 4, 8),
|
||||
(16896, 7168, 1): (224, 6, 4, 8),
|
||||
(16896, 7168, 2): (32, 4, 4, 8),
|
||||
(20480, 7168, 1): (224, 4, 2, 8),
|
||||
(20480, 7168, 2): (64, 4, 2, 8),
|
||||
(20480, 7168, 3): (64, 2, 2, 8),
|
||||
(20480, 7168, 4): (64, 4, 1, 8),
|
||||
(40960, 7168, 1): (128, 4, 2, 8),
|
||||
(40960, 7168, 2): (64, 4, 2, 8),
|
||||
(40960, 7168, 3): (64, 2, 2, 8),
|
||||
(40960, 7168, 4): (64, 4, 1, 8),
|
||||
# TP16.
|
||||
(3216, 7168, 1): (224, 3, 4, 8),
|
||||
(3216, 7168, 2): (128, 4, 2, 8),
|
||||
(3216, 7168, 3): (128, 2, 1, 8),
|
||||
(3216, 7168, 4): (64, 2, 2, 8),
|
||||
(3216, 7168, 5): (128, 3, 1, 8),
|
||||
(768, 7168, 1): (224, 2, 4, 8),
|
||||
(768, 7168, 2): (224, 2, 2, 8),
|
||||
(768, 7168, 3): (224, 2, 2, 8),
|
||||
(768, 7168, 4): (224, 2, 2, 8),
|
||||
(1152, 1536, 1): (192, 3, 4, 8),
|
||||
(4224, 7168, 1): (224, 3, 4, 8),
|
||||
(4224, 7168, 2): (128, 2, 1, 8),
|
||||
(4224, 7168, 3): (64, 2, 2, 8),
|
||||
(10240, 7168, 1): (224, 4, 2, 8),
|
||||
(10240, 7168, 2): (32, 2, 4, 8),
|
||||
(10240, 7168, 3): (64, 4, 1, 8),
|
||||
(10240, 7168, 4): (64, 4, 1, 8),
|
||||
}
|
||||
|
||||
EXPECTED_RESIDUAL_CUTE_CONFIGS = {
|
||||
(7168, 3584, 1): (64, 4, 2, 8),
|
||||
(7168, 3584, 2): (64, 7, 2, 8),
|
||||
(7168, 3584, 3): (64, 2, 1, 8),
|
||||
(7168, 3584, 4): (64, 2, 1, 8),
|
||||
}
|
||||
|
||||
|
||||
def _config_tuple(config) -> tuple[int, int, int, int]:
|
||||
return (
|
||||
config.block_size,
|
||||
config.outputs_per_block,
|
||||
config.k_unroll,
|
||||
config.vector_width,
|
||||
)
|
||||
|
||||
|
||||
def test_table_is_keyed_by_shape() -> None:
|
||||
for (n, k), spec in k3_gemm.KIMI_K3_PROJECTIONS.items():
|
||||
assert (spec.n, spec.k) == (n, k)
|
||||
|
||||
|
||||
def test_every_dsv3_routed_shape_is_instantiated() -> None:
|
||||
"""dsv3_fused_a_gemm specializes on (K, N); an unlisted shape raises.
|
||||
|
||||
The table routes by shape while the kernel is built per shape, so a missing
|
||||
instantiation only shows up at the token counts that route to dsv3. Checking
|
||||
it here needs no GPU, which is the point -- a GPU-only check is exactly what
|
||||
let (3216, 7168) ship without its DISPATCH_DSV3_SHAPE(7168, 3216).
|
||||
"""
|
||||
source = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "csrc"
|
||||
/ "libtorch_stable"
|
||||
/ "dsv3_fused_a_gemm.cu"
|
||||
).read_text(encoding="utf-8")
|
||||
# Benchmark-only shapes live behind VLLM_K3_BENCH_SHAPES and are not built
|
||||
# by default, so they must not count as available.
|
||||
production_macros = source.split("#ifdef VLLM_K3_BENCH_SHAPES")[0]
|
||||
explicit = source.split("#undef DISPATCH_DSV3_SHAPE")[1].split(
|
||||
"#ifdef VLLM_K3_BENCH_SHAPES"
|
||||
)[0]
|
||||
compiled = {
|
||||
(int(hd_in), int(hd_out))
|
||||
for hd_in, hd_out in re.findall(
|
||||
r"DISPATCH_DSV3_SHAPE\((\d+),\s*(\d+)\)", production_macros
|
||||
)
|
||||
} | {
|
||||
(int(hd_in), int(hd_out))
|
||||
for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit)
|
||||
}
|
||||
assert compiled, "failed to parse the dispatch list"
|
||||
|
||||
missing = sorted(
|
||||
(spec.n, spec.k)
|
||||
for spec in KIMI_K3_PROJECTIONS.values()
|
||||
if spec.dsv3_tokens and (spec.k, spec.n) not in compiled
|
||||
)
|
||||
assert not missing, (
|
||||
f"routed to dsv3 with no instantiation: {missing}; add "
|
||||
"DISPATCH_DSV3_SHAPE(K, N) for each"
|
||||
)
|
||||
|
||||
|
||||
def test_packed_row_major_rejects_single_row_slice() -> None:
|
||||
packed = torch.empty(1, 128)
|
||||
sliced = torch.empty(1, 144)[:, :128]
|
||||
|
||||
assert packed.is_contiguous()
|
||||
assert sliced.is_contiguous()
|
||||
assert k3_gemm._is_packed_row_major(packed)
|
||||
assert not k3_gemm._is_packed_row_major(sliced)
|
||||
|
||||
|
||||
def test_cute_configs_match_measured_table() -> None:
|
||||
actual = {
|
||||
(spec.n, spec.k, num_tokens): _config_tuple(config)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, config in spec.cute_configs
|
||||
}
|
||||
assert actual == EXPECTED_CUTE_CONFIGS
|
||||
|
||||
|
||||
def test_residual_cute_configs_match_measured_table() -> None:
|
||||
actual = {
|
||||
(spec.n, spec.k, num_tokens): _config_tuple(config)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, config in spec.residual_configs
|
||||
}
|
||||
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
|
||||
def test_sm103_selector_table(key: tuple[int, int]) -> None:
|
||||
n, k = key
|
||||
cute_tokens, dsv3_tokens = EXPECTED_SELECTIONS[key]
|
||||
for num_tokens in range(1, 17):
|
||||
backend = k3_gemm.select_kimi_k3_backend(num_tokens, n, k)
|
||||
if num_tokens in cute_tokens:
|
||||
assert backend == "cute"
|
||||
elif num_tokens in dsv3_tokens:
|
||||
assert backend == "dsv3_fused_a"
|
||||
else:
|
||||
assert backend is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
|
||||
def test_selector_requires_supported_shape_and_tokens(key: tuple[int, int]) -> None:
|
||||
n, k = key
|
||||
assert k3_gemm.select_kimi_k3_backend(0, n, k) is None
|
||||
assert k3_gemm.select_kimi_k3_backend(17, n, k) is None
|
||||
assert k3_gemm.select_kimi_k3_backend(1, n + 1, k) is None
|
||||
assert k3_gemm.select_kimi_k3_backend(1, n, k + 1) is None
|
||||
|
||||
|
||||
def test_unlisted_shape_and_unselected_tokens_fall_back() -> None:
|
||||
# Shape absent from the table.
|
||||
assert k3_gemm.select_kimi_k3_backend(1, 1000, 1000) is None
|
||||
# o_proj (7168,1536) is CuTe M1 only; M2+ falls back.
|
||||
assert k3_gemm.select_kimi_k3_backend(2, 7168, 1536) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", range(1, 17))
|
||||
def test_sm103_residual_selector_table(num_tokens: int) -> None:
|
||||
backend = k3_gemm.select_kimi_k3_backend(num_tokens, 7168, 3584, has_residual=True)
|
||||
assert backend == ("cute" if num_tokens <= 4 else None)
|
||||
|
||||
|
||||
def test_build_plan_matches_selector() -> None:
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values():
|
||||
plan = k3_gemm._build_plan(spec)
|
||||
for num_tokens in range(1, 17):
|
||||
backend = k3_gemm.select_kimi_k3_backend(num_tokens, spec.n, spec.k)
|
||||
if backend is None:
|
||||
assert num_tokens not in plan
|
||||
else:
|
||||
assert plan[num_tokens][0] == backend
|
||||
|
||||
|
||||
def test_installation_is_shape_specific_and_unquantized(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class FakeLinear(nn.Module):
|
||||
def __init__(self, quant_method: object, n: int, k: int) -> None:
|
||||
super().__init__()
|
||||
self.quant_method = quant_method
|
||||
self.weight = torch.empty(n, k)
|
||||
|
||||
class FakeHead(nn.Module):
|
||||
def __init__(self, n: int, k: int) -> None:
|
||||
super().__init__()
|
||||
self.quant_method = k3_gemm.UnquantizedEmbeddingMethod()
|
||||
self.weight = torch.empty(n, k)
|
||||
|
||||
root = nn.Module()
|
||||
# dsv3-only shape (no cute warmup contribution).
|
||||
root.dsv3_only = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 2304, 1536)
|
||||
# quantized: must be left untouched.
|
||||
quantized_method = object()
|
||||
root.quantized = FakeLinear(quantized_method, 6288, 7168)
|
||||
# cute shape.
|
||||
root.cute = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 6288, 7168)
|
||||
# cute + residual shape.
|
||||
root.residual = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 7168, 3584)
|
||||
# shape absent from the table: must be left untouched.
|
||||
root.unlisted = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 1234, 5678)
|
||||
root.lm_head = FakeHead(20480, 7168)
|
||||
|
||||
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
|
||||
monkeypatch.setattr(k3_gemm, "ParallelLMHead", FakeHead)
|
||||
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: True)
|
||||
warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
residual_warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
monkeypatch.setattr(k3_gemm.shape_dynamic_skinny_gemm, "is_available", lambda: True)
|
||||
|
||||
def request_warmup_configs(dtype, configs, *, has_residual=False):
|
||||
target = residual_warmup_configs if has_residual else warmup_configs
|
||||
target.update(configs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
k3_gemm.shape_dynamic_skinny_gemm,
|
||||
"request_warmup_configs",
|
||||
request_warmup_configs,
|
||||
)
|
||||
|
||||
k3_gemm.enable_kimi_k3_low_latency_gemm(root, torch.bfloat16)
|
||||
|
||||
assert isinstance(root.dsv3_only.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
|
||||
assert isinstance(root.cute.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
|
||||
assert isinstance(root.residual.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
|
||||
assert root.quantized.quant_method is quantized_method
|
||||
assert type(root.unlisted.quant_method) is k3_gemm.UnquantizedLinearMethod
|
||||
assert isinstance(
|
||||
root.lm_head.quant_method, k3_gemm.KimiK3LowLatencyEmbeddingMethod
|
||||
)
|
||||
# Warmup covers only the installed modules' local (N, K).
|
||||
assert warmup_configs == {
|
||||
config
|
||||
for key in ((6288, 7168), (7168, 3584), (20480, 7168))
|
||||
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[key].cute_configs
|
||||
}
|
||||
assert residual_warmup_configs == {
|
||||
config
|
||||
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)].residual_configs
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dtype,platform_enabled",
|
||||
[(torch.float16, True), (torch.bfloat16, False)],
|
||||
)
|
||||
def test_installation_requires_bf16_sm103(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dtype: torch.dtype,
|
||||
platform_enabled: bool,
|
||||
) -> None:
|
||||
class FakeLinear(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.quant_method = k3_gemm.UnquantizedLinearMethod()
|
||||
self.weight = torch.empty(2304, 1536)
|
||||
|
||||
root = nn.Module()
|
||||
root.projection = FakeLinear()
|
||||
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
|
||||
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: platform_enabled)
|
||||
|
||||
k3_gemm.enable_kimi_k3_low_latency_gemm(root, dtype)
|
||||
|
||||
assert type(root.projection.quant_method) is k3_gemm.UnquantizedLinearMethod
|
||||
|
||||
|
||||
def _require_sm103_and_dsv3() -> None:
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
|
||||
pytest.skip("Kimi-K3 production selection requires SM103")
|
||||
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
|
||||
pytest.skip("dsv3_fused_a_gemm was not built")
|
||||
|
||||
|
||||
def _require_sm103_and_cute() -> None:
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
|
||||
pytest.skip("Kimi-K3 production selection requires SM103")
|
||||
if not k3_gemm.shape_dynamic_skinny_gemm.is_available():
|
||||
pytest.skip("CuTe DSL is not available")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES)
|
||||
def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
output = k3_gemm.try_low_latency_gemm(x, weight)
|
||||
|
||||
assert output is not None
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def _dsv3_probe_tokens(tokens: frozenset[int]) -> set[int]:
|
||||
"""Extremes, plus both sides of the kernel's num_tokens<=8 tile_n branch."""
|
||||
if not tokens:
|
||||
return set()
|
||||
return {min(tokens), max(tokens)} | ({8, 9} & set(tokens))
|
||||
|
||||
|
||||
# Derived from the table rather than hand-listed, so a shape routed to dsv3
|
||||
# cannot be added without being exercised here.
|
||||
DSV3_CASES = sorted(
|
||||
(num_tokens, spec.n, spec.k)
|
||||
for spec in KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES)
|
||||
def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
|
||||
_require_sm103_and_dsv3()
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
|
||||
assert num_tokens in spec.dsv3_tokens
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
output = k3_gemm.try_low_latency_gemm(x, weight)
|
||||
|
||||
assert output is not None
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def test_nonpacked_single_token_dsv3_falls_back() -> None:
|
||||
_require_sm103_and_dsv3()
|
||||
n, k = 1536, 128
|
||||
storage = torch.randn(1, k + 16, dtype=torch.bfloat16, device="cuda")
|
||||
x = storage[:, :k]
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod(
|
||||
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
|
||||
)
|
||||
|
||||
assert x.is_contiguous()
|
||||
assert x.stride() == (k + 16, 1)
|
||||
assert not k3_gemm._runtime_ok(x, weight) # strict guard rejects the view
|
||||
output = method.apply(SimpleNamespace(weight=weight), x)
|
||||
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
torch.testing.assert_close(output, reference)
|
||||
|
||||
|
||||
def test_selected_kernels_cuda_graph_capture() -> None:
|
||||
_require_sm103_and_cute()
|
||||
_require_sm103_and_dsv3()
|
||||
cute_spec = k3_gemm.KIMI_K3_PROJECTIONS[(6288, 7168)]
|
||||
dsv3_spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
|
||||
cute_x = torch.randn(1, cute_spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
cute_weight = torch.randn(
|
||||
cute_spec.n, cute_spec.k, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
dsv3_x = torch.randn(1, dsv3_spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
dsv3_weight = torch.randn(
|
||||
dsv3_spec.n, dsv3_spec.k, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
|
||||
k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
cute_output = k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
|
||||
dsv3_output = k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert cute_output is not None
|
||||
assert dsv3_output is not None
|
||||
for output, activation, weight in (
|
||||
(cute_output, cute_x, cute_weight),
|
||||
(dsv3_output, dsv3_x, dsv3_weight),
|
||||
):
|
||||
reference = torch.nn.functional.linear(activation, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 9, 16])
|
||||
def test_dsv3_cuda_graph_capture_tile_branches(num_tokens: int) -> None:
|
||||
"""Capture DSV3 across the num_tokens<=8 vs >8 tile_n branch."""
|
||||
_require_sm103_and_dsv3()
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
|
||||
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
k3_gemm.try_low_latency_gemm(x, weight)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = k3_gemm.try_low_latency_gemm(x, weight)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert output is not None
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens", RESIDUAL_CUTE_CASES)
|
||||
def test_cute_residual_epilogue(n: int, k: int, num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
torch.manual_seed(42 + num_tokens)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
|
||||
config = spec.residual_config(num_tokens)
|
||||
assert config is not None
|
||||
|
||||
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
|
||||
reference = x.float() @ weight.float().t() + residual.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", range(1, 17))
|
||||
def test_cute_residual_epilogue_all_supported_token_counts(num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
ShapeDynamicSkinnyGemm,
|
||||
)
|
||||
|
||||
n, k = 64, 512
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
|
||||
config = ShapeDynamicSkinnyGemm._config(num_tokens, n, k)
|
||||
|
||||
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
|
||||
reference = x.float() @ weight.float().t() + residual.float()
|
||||
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", range(1, 5))
|
||||
def test_cute_residual_epilogue_cuda_graph_capture(num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)]
|
||||
config = spec.residual_config(num_tokens)
|
||||
assert config is not None
|
||||
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(num_tokens, spec.n, dtype=torch.bfloat16, device="cuda")
|
||||
k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
reference = x.float() @ weight.float().t() + residual.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
class _SkinnyGemmSpy:
|
||||
"""Wraps the skinny-GEMM singleton to record whether CuTe was invoked."""
|
||||
|
||||
def __init__(self, real: Any) -> None:
|
||||
self._real = real
|
||||
self.calls: list[int] = []
|
||||
|
||||
def __call__(self, a, b, config=None, residual=None):
|
||||
self.calls.append(a.shape[0])
|
||||
return self._real(a, b, config, residual)
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._real.is_available()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 3, 4])
|
||||
def test_latent_moe_production_layout_residual(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
"""The real Latent-MoE residual is a non-packed slice of a cat buffer.
|
||||
|
||||
The strict packed-row-major guard rejects such a slice at every token count
|
||||
(a size-1 leading dim reads as contiguous but its stride is not packed), so
|
||||
the CuTe residual epilogue never fires for this production layout and the
|
||||
method falls back to addmm. Output is correct regardless of the path.
|
||||
"""
|
||||
_require_sm103_and_cute()
|
||||
latent_dim, shared_dim = 3584, 7168 # routed_expert_up_proj K, N
|
||||
torch.manual_seed(7 + num_tokens)
|
||||
buf = torch.randn(
|
||||
num_tokens, latent_dim + shared_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
latent = buf[:, :latent_dim] # non-contiguous view (row stride = full width)
|
||||
residual = buf[:, latent_dim:] # non-contiguous view
|
||||
weight = torch.randn(shared_dim, latent_dim, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS[(shared_dim, latent_dim)]
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod(
|
||||
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
|
||||
)
|
||||
spy = _SkinnyGemmSpy(k3_gemm.shape_dynamic_skinny_gemm)
|
||||
monkeypatch.setattr(k3_gemm, "shape_dynamic_skinny_gemm", spy)
|
||||
|
||||
layer = SimpleNamespace(weight=weight)
|
||||
output = method.apply_with_residual(layer, latent, residual)
|
||||
|
||||
reference = latent.float() @ weight.float().t() + residual.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999 # correct regardless of the path taken
|
||||
assert not spy.calls, (
|
||||
"non-packed buf-slice residual must fall back to addmm at every M"
|
||||
)
|
||||
|
||||
|
||||
def test_residual_dispatch_falls_back_to_addmm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fallback = torch.randn(2, 3)
|
||||
residual = torch.randn(2, 3)
|
||||
x = torch.randn(2, 4)
|
||||
weight = torch.randn(3, 4)
|
||||
monkeypatch.setattr(torch, "addmm", lambda *args: fallback)
|
||||
# CPU tensors fail the runtime check, forcing the addmm fallback.
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
|
||||
|
||||
output = method.apply_with_residual(SimpleNamespace(weight=weight), x, residual)
|
||||
|
||||
assert output is fallback
|
||||
|
||||
|
||||
def test_fallback_preserves_default_method(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
fallback = torch.empty(2, 8)
|
||||
monkeypatch.setattr(
|
||||
k3_gemm.UnquantizedLinearMethod,
|
||||
"apply",
|
||||
lambda *args: fallback,
|
||||
)
|
||||
# 1-D input fails the runtime check, forcing the base-method fallback.
|
||||
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
|
||||
|
||||
output = method.apply(
|
||||
SimpleNamespace(weight=torch.empty(0)),
|
||||
torch.empty(0),
|
||||
)
|
||||
|
||||
assert output is fallback
|
||||
@@ -1,226 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Precision tests for vllm's chunk_kda Triton operator.
|
||||
|
||||
Compares chunk_kda against a naive recurrent reference (float32).
|
||||
Uses torch.rand for q/k/v to match FLA's test pattern.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import (
|
||||
chunk_kda,
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_kda_gate,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
|
||||
def naive_recurrent_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
|
||||
dtype = v.dtype
|
||||
B, T, H, K = q.shape
|
||||
V = v.shape[-1]
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
|
||||
q = q * scale
|
||||
|
||||
S = k.new_zeros(B, H, K, V).to(q)
|
||||
if initial_state is not None:
|
||||
S += initial_state
|
||||
o = torch.zeros_like(v)
|
||||
for i in range(T):
|
||||
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
|
||||
S = S * g_i[..., None].exp()
|
||||
S = S + torch.einsum(
|
||||
"bhk,bhv->bhkv",
|
||||
b_i[..., None] * k_i,
|
||||
v_i - (k_i[..., None] * S).sum(-2),
|
||||
)
|
||||
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
|
||||
if not output_final_state:
|
||||
S = None
|
||||
return o.to(dtype), S
|
||||
|
||||
|
||||
def assert_close(
|
||||
name: str,
|
||||
ref: torch.Tensor,
|
||||
tri: torch.Tensor,
|
||||
ratio: float,
|
||||
err_atol: float = 1e-6,
|
||||
):
|
||||
"""RMSE-based relative error comparison."""
|
||||
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
|
||||
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
|
||||
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
|
||||
rel_err = rmse_diff / (rmse_base + 1e-8)
|
||||
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
|
||||
if abs_err <= err_atol:
|
||||
return
|
||||
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
|
||||
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
|
||||
assert rel_err < ratio, (
|
||||
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("H", "D", "cu_seqlens", "dtype"),
|
||||
[
|
||||
pytest.param(
|
||||
*test,
|
||||
id="H{}-D{}-cu{}-{}".format(*test),
|
||||
)
|
||||
for test in [
|
||||
(32, 128, [0, 64], torch.float16),
|
||||
(32, 128, [0, 1024], torch.float16),
|
||||
(32, 128, [0, 15], torch.float16),
|
||||
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
|
||||
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
|
||||
(64, 128, [0, 256, 500, 1000], torch.float16),
|
||||
(32, 128, [0, 8192], torch.float16),
|
||||
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
|
||||
]
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda(
|
||||
H: int,
|
||||
D: int,
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
T = cu_seqlens[-1]
|
||||
torch.manual_seed(42)
|
||||
B = 1
|
||||
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
|
||||
N = len(cu_seqlens) - 1
|
||||
|
||||
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
|
||||
dtype
|
||||
)
|
||||
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
|
||||
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
|
||||
ref_outputs = []
|
||||
ref_states = []
|
||||
for i in range(N):
|
||||
s, e = cu_seqlens[i], cu_seqlens[i + 1]
|
||||
q_i = l2norm_fwd(q[:, s:e].contiguous())
|
||||
k_i = l2norm_fwd(k[:, s:e].contiguous())
|
||||
o_i, ht_i = naive_recurrent_kda(
|
||||
q_i,
|
||||
k_i,
|
||||
v[:, s:e],
|
||||
g[:, s:e],
|
||||
beta[:, s:e],
|
||||
initial_state=h0[i],
|
||||
output_final_state=True,
|
||||
)
|
||||
ref_outputs.append(o_i)
|
||||
ref_states.append(ht_i)
|
||||
ref_o = torch.cat(ref_outputs, dim=1)
|
||||
ref_ht = torch.cat(ref_states, dim=0)
|
||||
|
||||
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
|
||||
tri_o, tri_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=g.clone(),
|
||||
beta=beta.clone(),
|
||||
initial_state=h0.transpose(-1, -2).contiguous().clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
|
||||
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
|
||||
assert_close("o", ref_o, tri_o, 0.005)
|
||||
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cu_seqlens", "dtype"),
|
||||
[
|
||||
([0, 64], torch.float16),
|
||||
([0, 15, 100, 300], torch.bfloat16),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
H, D = 8, 64
|
||||
T = cu_seqlens[-1]
|
||||
N = len(cu_seqlens) - 1
|
||||
torch.manual_seed(123)
|
||||
|
||||
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
|
||||
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
beta = torch.rand(1, T, H, dtype=dtype, device=DEVICE).sigmoid()
|
||||
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
|
||||
dt_bias = (
|
||||
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
).contiguous()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
initial_state = h0.transpose(-1, -2).contiguous()
|
||||
|
||||
gate = fused_kda_gate(
|
||||
raw_g.reshape(T, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
).unsqueeze(0)
|
||||
old_o, old_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=gate,
|
||||
beta=beta.clone(),
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
new_o, new_ht = chunk_kda_with_fused_gate(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
raw_g=raw_g,
|
||||
beta=beta.clone(),
|
||||
A_log=A_log,
|
||||
g_bias=dt_bias,
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
|
||||
@@ -5,6 +5,7 @@ import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.common.mtp import fused_mtp_input
|
||||
from vllm.models.kimi_k3.nvidia.ops import attn_res
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -191,3 +192,35 @@ def test_attn_res_without_output_norm():
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 17])
|
||||
def test_fused_mtp_input(num_tokens: int):
|
||||
positions = torch.arange(num_tokens, device="cuda")
|
||||
inputs_embeds = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=7)
|
||||
previous_hidden_states = _randn_with_row_padding(
|
||||
num_tokens, HIDDEN_SIZE, padding=11
|
||||
)
|
||||
enorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
hnorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
masked_inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
expected = torch.cat(
|
||||
(
|
||||
F.rms_norm(masked_inputs_embeds, (HIDDEN_SIZE,), enorm_weight, EPS),
|
||||
F.rms_norm(previous_hidden_states, (HIDDEN_SIZE,), hnorm_weight, EPS),
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
actual = fused_mtp_input(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
enorm_weight,
|
||||
hnorm_weight,
|
||||
EPS,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
assert actual.shape == (num_tokens, 2 * HIDDEN_SIZE)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.interfaces import supports_eagle3
|
||||
from vllm.models.kimi_k3.nvidia import model as kimi_model
|
||||
from vllm.models.kimi_k3.nvidia.model import (
|
||||
KimiK3ForConditionalGeneration,
|
||||
KimiLinearModel,
|
||||
)
|
||||
|
||||
|
||||
def _make_kimi_linear_model() -> KimiLinearModel:
|
||||
model = object.__new__(KimiLinearModel)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (2,))
|
||||
object.__setattr__(model, "use_sequence_parallel", False)
|
||||
return model
|
||||
|
||||
|
||||
def test_kimi_k3_advertises_eagle3_support():
|
||||
assert supports_eagle3(KimiK3ForConditionalGeneration)
|
||||
|
||||
|
||||
def test_kimi_k3_uses_shared_eagle3_layer_configuration():
|
||||
target = object.__new__(KimiK3ForConditionalGeneration)
|
||||
torch.nn.Module.__init__(target)
|
||||
model = _make_kimi_linear_model()
|
||||
object.__setattr__(model, "layers", [None] * 93)
|
||||
language_model = SimpleNamespace(
|
||||
embed_input_ids=lambda _: None,
|
||||
model=model,
|
||||
)
|
||||
object.__setattr__(target, "language_model", language_model)
|
||||
object.__setattr__(target, "_language_model_names", ["language_model"])
|
||||
|
||||
target.set_aux_hidden_state_layers((2, 46, 90))
|
||||
|
||||
assert model.aux_hidden_state_layers == (2, 46, 90)
|
||||
assert target.get_eagle3_default_aux_hidden_state_layers() == (
|
||||
2,
|
||||
46,
|
||||
90,
|
||||
)
|
||||
|
||||
|
||||
def test_kimi_linear_forward_extracts_standard_aux_hidden_states(monkeypatch):
|
||||
model = _make_kimi_linear_model()
|
||||
initial_hidden_states = torch.tensor([[1.0, 2.0]])
|
||||
layer_hidden_states = torch.tensor([[3.0, 4.0]])
|
||||
layer_residual = torch.tensor([[5.0, 6.0]])
|
||||
|
||||
object.__setattr__(model, "start_layer", 0)
|
||||
object.__setattr__(model, "end_layer", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"layers",
|
||||
[Mock(return_value=(layer_hidden_states, None, layer_residual))],
|
||||
)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
|
||||
object.__setattr__(model, "use_attn_res", False)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=initial_hidden_states,
|
||||
)
|
||||
|
||||
expected_layer_output = layer_hidden_states + layer_residual
|
||||
torch.testing.assert_close(output, expected_layer_output)
|
||||
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[1], expected_layer_output)
|
||||
|
||||
|
||||
def test_kimi_linear_forward_extracts_attn_res_aux_hidden_states(monkeypatch):
|
||||
model = _make_kimi_linear_model()
|
||||
initial_hidden_states = torch.tensor([[1.0, 2.0]])
|
||||
layer_hidden_states = torch.tensor([[3.0, 4.0]])
|
||||
prefix_sum = torch.tensor([[5.0, 6.0]])
|
||||
block_residual = torch.tensor([[[7.0, 8.0]]])
|
||||
final_hidden_states = torch.tensor([[9.0, 10.0]])
|
||||
|
||||
object.__setattr__(model, "start_layer", 0)
|
||||
object.__setattr__(model, "end_layer", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"layers",
|
||||
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
|
||||
)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
|
||||
object.__setattr__(model, "use_attn_res", True)
|
||||
object.__setattr__(model, "num_attn_res_blocks", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_norm",
|
||||
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
|
||||
)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_proj",
|
||||
SimpleNamespace(weight=torch.ones(1, 2)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
final_attn_res = Mock(return_value=final_hidden_states)
|
||||
monkeypatch.setattr(kimi_model, "attn_res", final_attn_res)
|
||||
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=initial_hidden_states,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, final_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[1], prefix_sum + layer_hidden_states)
|
||||
assert final_attn_res.call_args.args[2] is block_residual
|
||||
@@ -0,0 +1,757 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Precision tests for vllm's chunk_kda Triton operator.
|
||||
|
||||
Compares chunk_kda against a naive recurrent reference (float32).
|
||||
Uses torch.rand for q/k/v to match FLA's test pattern.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_update
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.kda import (
|
||||
is_flashkda_supported,
|
||||
is_fused_kda_decode_supported,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda,
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_kda_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_fwd,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gather_initial_states_correctness():
|
||||
row_size = 8 * 128 * 128
|
||||
storage = torch.randn(5, row_size + 256, dtype=torch.float32, device=DEVICE)
|
||||
state = storage[:, :row_size].view(5, 8, 128, 128)
|
||||
assert not state.is_contiguous()
|
||||
assert state[0].is_contiguous()
|
||||
indices = torch.tensor([4, 1, 3], dtype=torch.int32, device=DEVICE)
|
||||
has_initial_state = torch.tensor([True, False, True], device=DEVICE)
|
||||
|
||||
expected = state[indices].clone()
|
||||
expected[~has_initial_state] = 0
|
||||
|
||||
torch.testing.assert_close(
|
||||
gather_initial_states(state, indices, has_initial_state),
|
||||
expected,
|
||||
)
|
||||
|
||||
|
||||
def naive_recurrent_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
|
||||
dtype = v.dtype
|
||||
B, T, H, K = q.shape
|
||||
V = v.shape[-1]
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
|
||||
q = q * scale
|
||||
|
||||
S = k.new_zeros(B, H, K, V).to(q)
|
||||
if initial_state is not None:
|
||||
S += initial_state
|
||||
o = torch.zeros_like(v)
|
||||
for i in range(T):
|
||||
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
|
||||
S = S * g_i[..., None].exp()
|
||||
S = S + torch.einsum(
|
||||
"bhk,bhv->bhkv",
|
||||
b_i[..., None] * k_i,
|
||||
v_i - (k_i[..., None] * S).sum(-2),
|
||||
)
|
||||
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
|
||||
if not output_final_state:
|
||||
S = None
|
||||
return o.to(dtype), S
|
||||
|
||||
|
||||
def assert_close(
|
||||
name: str,
|
||||
ref: torch.Tensor,
|
||||
tri: torch.Tensor,
|
||||
ratio: float,
|
||||
err_atol: float = 1e-6,
|
||||
):
|
||||
"""RMSE-based relative error comparison."""
|
||||
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
|
||||
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
|
||||
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
|
||||
rel_err = rmse_diff / (rmse_base + 1e-8)
|
||||
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
|
||||
if abs_err <= err_atol:
|
||||
return
|
||||
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
|
||||
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
|
||||
assert rel_err < ratio, (
|
||||
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("H", "D", "cu_seqlens", "dtype"),
|
||||
[
|
||||
pytest.param(
|
||||
*test,
|
||||
id="H{}-D{}-cu{}-{}".format(*test),
|
||||
)
|
||||
for test in [
|
||||
(32, 128, [0, 64], torch.float16),
|
||||
(32, 128, [0, 1024], torch.float16),
|
||||
(32, 128, [0, 15], torch.float16),
|
||||
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
|
||||
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
|
||||
(64, 128, [0, 256, 500, 1000], torch.float16),
|
||||
(32, 128, [0, 8192], torch.float16),
|
||||
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
|
||||
]
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda(
|
||||
H: int,
|
||||
D: int,
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
T = cu_seqlens[-1]
|
||||
torch.manual_seed(42)
|
||||
B = 1
|
||||
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
|
||||
N = len(cu_seqlens) - 1
|
||||
|
||||
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
|
||||
dtype
|
||||
)
|
||||
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
|
||||
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
|
||||
ref_outputs = []
|
||||
ref_states = []
|
||||
for i in range(N):
|
||||
s, e = cu_seqlens[i], cu_seqlens[i + 1]
|
||||
q_i = l2norm_fwd(q[:, s:e].contiguous())
|
||||
k_i = l2norm_fwd(k[:, s:e].contiguous())
|
||||
o_i, ht_i = naive_recurrent_kda(
|
||||
q_i,
|
||||
k_i,
|
||||
v[:, s:e],
|
||||
g[:, s:e],
|
||||
beta[:, s:e],
|
||||
initial_state=h0[i],
|
||||
output_final_state=True,
|
||||
)
|
||||
ref_outputs.append(o_i)
|
||||
ref_states.append(ht_i)
|
||||
ref_o = torch.cat(ref_outputs, dim=1)
|
||||
ref_ht = torch.cat(ref_states, dim=0)
|
||||
|
||||
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
|
||||
tri_o, tri_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=g.clone(),
|
||||
beta=beta.clone(),
|
||||
initial_state=h0.transpose(-1, -2).contiguous().clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
|
||||
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
|
||||
assert_close("o", ref_o, tri_o, 0.005)
|
||||
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cu_seqlens", "dtype", "lower_bound"),
|
||||
[
|
||||
([0, 64], torch.float16, None),
|
||||
([0, 15, 100, 300], torch.bfloat16, None),
|
||||
([0, 15, 100, 300], torch.bfloat16, -3.0),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
lower_bound: float | None,
|
||||
):
|
||||
H, D = 8, 64
|
||||
T = cu_seqlens[-1]
|
||||
N = len(cu_seqlens) - 1
|
||||
torch.manual_seed(123)
|
||||
|
||||
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
|
||||
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
beta_storage = torch.randn(1, T, 2 * H + 3, dtype=dtype, device=DEVICE)
|
||||
raw_beta = beta_storage[..., 1 : 2 * H + 1 : 2]
|
||||
beta = raw_beta.float().sigmoid()
|
||||
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
|
||||
dt_bias = (
|
||||
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
).contiguous()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
initial_state = h0.transpose(-1, -2).contiguous()
|
||||
|
||||
gate = fused_kda_gate(
|
||||
raw_g.reshape(T, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
if lower_bound is not None:
|
||||
expected_gate = lower_bound * torch.sigmoid(
|
||||
A_log.exp()[None, :, None]
|
||||
* (raw_g.float().view(T, H, D) + dt_bias.view(H, D))
|
||||
)
|
||||
torch.testing.assert_close(gate, expected_gate)
|
||||
gate = gate.unsqueeze(0)
|
||||
old_o, old_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=gate,
|
||||
beta=beta,
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
new_o, new_ht = chunk_kda_with_fused_gate(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_seqs", [1, 8, 32])
|
||||
@pytest.mark.parametrize("lower_bound", [-5.0, None])
|
||||
@pytest.mark.parametrize("state_indices_stride", [1, 8])
|
||||
@torch.inference_mode()
|
||||
def test_packed_kda_decode_correctness(
|
||||
num_seqs: int,
|
||||
lower_bound: float | None,
|
||||
state_indices_stride: int,
|
||||
):
|
||||
H, D = 8, 128
|
||||
torch.manual_seed(321)
|
||||
|
||||
packed_storage = torch.randn(
|
||||
num_seqs,
|
||||
3 * H * D + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
mixed_qkv = packed_storage[:, : 3 * H * D]
|
||||
assert mixed_qkv.stride(0) == 3 * H * D + 1
|
||||
q, k, v = (
|
||||
x.contiguous().view(1, num_seqs, H, D) for x in mixed_qkv.split(H * D, dim=-1)
|
||||
)
|
||||
raw_g = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
H,
|
||||
D,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
H,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
beta = raw_beta.float().sigmoid()
|
||||
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
|
||||
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
state_storage = torch.randn(
|
||||
num_seqs + 1,
|
||||
H * D * D + 17,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state = state_storage[:, : H * D * D].view(num_seqs + 1, H, D, D)
|
||||
assert not state.is_contiguous()
|
||||
assert state.stride()[1:] == (D * D, D, 1)
|
||||
state_indices_storage = torch.zeros(
|
||||
num_seqs,
|
||||
state_indices_stride,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_indices = state_indices_storage[:, 0]
|
||||
state_indices.copy_(
|
||||
torch.arange(
|
||||
1,
|
||||
num_seqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
)
|
||||
gate = fused_kda_gate(
|
||||
raw_g.reshape(num_seqs, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
).unsqueeze(0)
|
||||
dense_state = state.clone()
|
||||
dense_out, _ = fused_recurrent_kda_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=gate,
|
||||
beta=beta,
|
||||
scale=D**-0.5,
|
||||
initial_state=dense_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=torch.arange(
|
||||
num_seqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
),
|
||||
ssm_state_indices=state_indices,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
packed_state = state
|
||||
packed_out, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=packed_state,
|
||||
state_indices=state_indices,
|
||||
)
|
||||
|
||||
assert_close("o", dense_out, packed_out, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", dense_state, packed_state, 1e-3, err_atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("H", "fuse_gate"),
|
||||
[(12, True), (12, False), (12, None), (96, None)],
|
||||
)
|
||||
@pytest.mark.parametrize("lower_bound", [-5.0, None])
|
||||
@torch.inference_mode()
|
||||
def test_kda_spec_decode_correctness(
|
||||
H: int,
|
||||
fuse_gate: bool | None,
|
||||
lower_bound: float | None,
|
||||
):
|
||||
num_seqs, query_len, D = 3, 3, 128
|
||||
T = num_seqs * query_len
|
||||
torch.manual_seed(1234)
|
||||
|
||||
qkv_storage = torch.randn(
|
||||
1,
|
||||
T,
|
||||
3 * H * D + 7,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
packed_qkv = qkv_storage[..., : 3 * H * D]
|
||||
q, k, v = (x.view(1, T, H, D) for x in packed_qkv.split(H * D, dim=-1))
|
||||
gate_storage = torch.randn(
|
||||
1,
|
||||
T,
|
||||
H * D + 5,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_g = gate_storage[..., : H * D].view(1, T, H, D)
|
||||
beta_storage = torch.randn(
|
||||
1,
|
||||
T,
|
||||
H + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta = beta_storage[..., :H]
|
||||
A_log = 0.5 * torch.randn(H, dtype=torch.float32, device=DEVICE)
|
||||
dt_bias = 0.1 * torch.randn(H, D, dtype=torch.float32, device=DEVICE)
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
T + 1,
|
||||
query_len,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_indices = torch.arange(
|
||||
1,
|
||||
T + 1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
).view(num_seqs, query_len)
|
||||
num_accepted_tokens = torch.tensor(
|
||||
[1, 2, 3],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_storage = 0.01 * torch.randn(
|
||||
T + 1,
|
||||
H * D * D + 17,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state = state_storage[:, : H * D * D].view(T + 1, H, D, D)
|
||||
output_storage = torch.full(
|
||||
(1, T, H * D + 11),
|
||||
torch.nan,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
output = output_storage[..., : H * D].view(1, T, H, D)
|
||||
|
||||
gate = fused_kda_gate(
|
||||
raw_g.contiguous().view(T, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
).unsqueeze(0)
|
||||
beta = raw_beta.float().sigmoid()
|
||||
q_norm = l2norm_fwd(q.contiguous())
|
||||
k_norm = l2norm_fwd(k.contiguous())
|
||||
expected_state = state.clone()
|
||||
expected_outputs = []
|
||||
for seq, accepted in enumerate(num_accepted_tokens.tolist()):
|
||||
recurrent_state = expected_state[state_indices[seq, accepted - 1]].transpose(
|
||||
-1, -2
|
||||
)
|
||||
start = seq * query_len
|
||||
for token in range(query_len):
|
||||
token_slice = slice(start + token, start + token + 1)
|
||||
token_output, recurrent_state = naive_recurrent_kda(
|
||||
q_norm[:, token_slice],
|
||||
k_norm[:, token_slice],
|
||||
v[:, token_slice],
|
||||
gate[:, token_slice],
|
||||
beta[:, token_slice],
|
||||
initial_state=recurrent_state,
|
||||
output_final_state=True,
|
||||
)
|
||||
assert recurrent_state is not None
|
||||
expected_outputs.append(token_output)
|
||||
expected_state[state_indices[seq, token]] = recurrent_state.transpose(
|
||||
-1, -2
|
||||
)
|
||||
expected = torch.cat(expected_outputs, dim=1)
|
||||
|
||||
actual_state = state.clone()
|
||||
actual, _ = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=actual_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=output,
|
||||
fuse_gate=fuse_gate,
|
||||
)
|
||||
|
||||
assert actual.data_ptr() == output.data_ptr()
|
||||
assert_close("o", expected, actual, 1e-3, err_atol=1e-3)
|
||||
used_states = state_indices.flatten().long()
|
||||
assert_close(
|
||||
"ht",
|
||||
expected_state[used_states],
|
||||
actual_state[used_states],
|
||||
3e-3,
|
||||
err_atol=3e-3,
|
||||
)
|
||||
assert torch.isnan(output_storage[..., H * D :]).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_heads", "num_seqs", "lower_bound", "fuse_output_norm"),
|
||||
[
|
||||
(12, 1, -5.0, True),
|
||||
(12, 4, None, False),
|
||||
(24, 4, None, False),
|
||||
(48, 1, -5.0, True),
|
||||
(96, 1, -5.0, True),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_correctness(
|
||||
num_heads: int,
|
||||
num_seqs: int,
|
||||
lower_bound: float | None,
|
||||
fuse_output_norm: bool,
|
||||
):
|
||||
D, W = 128, 4
|
||||
if not is_fused_kda_decode_supported(
|
||||
num_heads,
|
||||
D,
|
||||
W,
|
||||
num_spec=0,
|
||||
input_dtype=torch.bfloat16,
|
||||
conv_state_dtype=torch.bfloat16,
|
||||
):
|
||||
pytest.skip("Fused KDA decode is not supported on this platform")
|
||||
torch.manual_seed(967 + num_heads + num_seqs)
|
||||
dim = num_heads * D
|
||||
slots = num_seqs + 2
|
||||
packed_x_storage = torch.randn(
|
||||
num_seqs, 3 * dim + 17, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
packed_x = packed_x_storage[:, : 3 * dim]
|
||||
weight = 0.1 * torch.randn(3 * dim, W, dtype=torch.float32, device=DEVICE)
|
||||
conv_seed = 0.1 * torch.randn(
|
||||
slots,
|
||||
W - 1,
|
||||
3 * dim,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
).transpose(1, 2)
|
||||
raw_g = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
num_heads,
|
||||
D,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta_storage = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
num_heads + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta = raw_beta_storage[:, :, :num_heads]
|
||||
output_gate_storage = torch.randn(
|
||||
num_seqs,
|
||||
dim + 7,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
output_gate = output_gate_storage[:, :dim].view(num_seqs, num_heads, D)
|
||||
norm_weight = torch.randn(D, dtype=torch.float32, device=DEVICE)
|
||||
norm_eps = 1e-5
|
||||
A_log = 0.5 * torch.randn(num_heads, dtype=torch.float32, device=DEVICE)
|
||||
dt_bias = 0.1 * torch.randn(dim, dtype=torch.float32, device=DEVICE)
|
||||
state_indices = torch.arange(
|
||||
num_seqs,
|
||||
0,
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_seed = 0.01 * torch.randn(
|
||||
slots,
|
||||
num_heads,
|
||||
D,
|
||||
D,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
conv_ref = conv_seed.clone()
|
||||
state_ref = state_seed.clone()
|
||||
mixed_qkv = causal_conv1d_update(
|
||||
packed_x,
|
||||
conv_ref,
|
||||
weight,
|
||||
activation="silu",
|
||||
conv_state_indices=state_indices,
|
||||
validate_data=True,
|
||||
out=torch.empty_like(packed_x),
|
||||
)
|
||||
expected, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=state_ref,
|
||||
state_indices=state_indices,
|
||||
)
|
||||
if fuse_output_norm:
|
||||
expected_float = expected.float()
|
||||
expected = (
|
||||
expected_float
|
||||
* torch.rsqrt(expected_float.square().mean(dim=-1, keepdim=True) + norm_eps)
|
||||
* norm_weight
|
||||
* output_gate.float().sigmoid().unsqueeze(0)
|
||||
).to(expected.dtype)
|
||||
|
||||
conv_slot_elements = 3 * dim * (W - 1)
|
||||
state_slot_elements = num_heads * D * D
|
||||
conv_slot_bytes = conv_slot_elements * torch.bfloat16.itemsize
|
||||
page_bytes = conv_slot_bytes + state_slot_elements * torch.float32.itemsize
|
||||
cache_storage = torch.empty(slots * page_bytes, dtype=torch.uint8, device=DEVICE)
|
||||
conv_actual = torch.as_strided(
|
||||
cache_storage.view(torch.bfloat16),
|
||||
size=(slots, 3 * dim, W - 1),
|
||||
stride=(page_bytes // torch.bfloat16.itemsize, 1, 3 * dim),
|
||||
)
|
||||
state_actual = torch.as_strided(
|
||||
cache_storage.view(torch.float32),
|
||||
size=(slots, num_heads, D, D),
|
||||
stride=(page_bytes // torch.float32.itemsize, D * D, D, 1),
|
||||
storage_offset=conv_slot_bytes // torch.float32.itemsize,
|
||||
)
|
||||
conv_actual.copy_(conv_seed)
|
||||
state_actual.copy_(state_seed)
|
||||
fused_weight = weight.reshape(3, dim, W).transpose(1, 2).contiguous()
|
||||
actual = ops.fused_kda_decode(
|
||||
x=packed_x,
|
||||
weight=fused_weight,
|
||||
bias=None,
|
||||
conv_state=conv_actual,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
state_indices=state_indices,
|
||||
state=state_actual,
|
||||
lower_bound=lower_bound,
|
||||
output_gate=output_gate if fuse_output_norm else None,
|
||||
norm_weight=norm_weight if fuse_output_norm else None,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(conv_actual, conv_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def test_fused_kda_decode_rejects_speculative_conv_state():
|
||||
assert not is_fused_kda_decode_supported(
|
||||
num_heads=12,
|
||||
head_dim=128,
|
||||
conv_width=4,
|
||||
num_spec=2,
|
||||
input_dtype=torch.bfloat16,
|
||||
conv_state_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_flashkda_correctness():
|
||||
if not is_flashkda_supported(128, torch.bfloat16, -3.0):
|
||||
pytest.skip("FlashKDA is not supported on this platform")
|
||||
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
B, T, H, D = 1, 48, 2, 128
|
||||
torch.manual_seed(11)
|
||||
q, k, v, raw_g = [
|
||||
torch.randn(B, T, H, D, dtype=torch.bfloat16, device=DEVICE) for _ in range(4)
|
||||
]
|
||||
beta_logits = torch.randn(B, T, H, dtype=torch.bfloat16, device=DEVICE)
|
||||
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
|
||||
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
initial_state = torch.randn(2, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
cu_seqlens = torch.tensor([0, 17, T], dtype=torch.int32, device=DEVICE)
|
||||
lower_bound = -3.0
|
||||
|
||||
gate = lower_bound * torch.sigmoid(
|
||||
A_log.exp()[None, None, :, None] * (raw_g.float() + dt_bias[None, None, :, :])
|
||||
)
|
||||
beta = beta_logits.float().sigmoid()
|
||||
q_norm = l2norm_fwd(q.contiguous())
|
||||
k_norm = l2norm_fwd(k.contiguous())
|
||||
|
||||
expected_outputs = []
|
||||
expected_states = []
|
||||
for i, (start, end) in enumerate(
|
||||
zip(cu_seqlens[:-1].tolist(), cu_seqlens[1:].tolist())
|
||||
):
|
||||
output, final_state = naive_recurrent_kda(
|
||||
q_norm[:, start:end],
|
||||
k_norm[:, start:end],
|
||||
v[:, start:end],
|
||||
gate[:, start:end],
|
||||
beta[:, start:end],
|
||||
initial_state=initial_state[i].transpose(-1, -2),
|
||||
output_final_state=True,
|
||||
)
|
||||
expected_outputs.append(output)
|
||||
expected_states.append(final_state)
|
||||
expected_out = torch.cat(expected_outputs, dim=1)
|
||||
expected_state = torch.cat(expected_states).transpose(-1, -2).contiguous()
|
||||
|
||||
actual_out = torch.empty_like(v)
|
||||
actual_state = torch.empty_like(initial_state)
|
||||
workspace = torch.empty(
|
||||
torch.ops._flashkda_C.get_workspace_size(T, H, cu_seqlens.numel() - 1),
|
||||
dtype=torch.uint8,
|
||||
device=DEVICE,
|
||||
)
|
||||
torch.ops._flashkda_C.fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
raw_g,
|
||||
beta_logits,
|
||||
D**-0.5,
|
||||
actual_out,
|
||||
workspace,
|
||||
A_log,
|
||||
dt_bias,
|
||||
lower_bound,
|
||||
initial_state,
|
||||
actual_state,
|
||||
cu_seqlens,
|
||||
)
|
||||
|
||||
assert_close("o", expected_out, actual_out, 0.01)
|
||||
assert_close("ht", expected_state, actual_state, 0.01)
|
||||
@@ -0,0 +1,411 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import fields
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.models.kimi_k3.nvidia.kda_metadata import (
|
||||
KimiK3KDAAttentionBackend,
|
||||
KimiK3KDAMetadata,
|
||||
KimiK3KDAMetadataBuilder,
|
||||
_mamba_get_block_table_tensor,
|
||||
stage_spec_decode_metadata,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
NULL_BLOCK_ID,
|
||||
mamba_get_block_table_tensor,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
PRUNED_METADATA_FIELDS = {
|
||||
"chunk_indices",
|
||||
"chunk_offsets",
|
||||
"prefill_query_start_loc",
|
||||
"prefill_state_indices",
|
||||
"prefill_has_initial_state",
|
||||
"spec_sequence_masks",
|
||||
}
|
||||
|
||||
|
||||
def _assert_matches_shared_gdn(reference, actual: KimiK3KDAMetadata):
|
||||
for field in fields(KimiK3KDAMetadata):
|
||||
actual_value = getattr(actual, field.name)
|
||||
expected_value = getattr(reference, field.name)
|
||||
if field.name in PRUNED_METADATA_FIELDS:
|
||||
assert actual_value is None
|
||||
continue
|
||||
if (
|
||||
field.name in {"spec_token_indx", "non_spec_token_indx"}
|
||||
and actual.num_spec_decodes > 0
|
||||
and actual.num_prefills == 0
|
||||
and actual.num_decodes == 0
|
||||
):
|
||||
assert actual_value is None
|
||||
continue
|
||||
if isinstance(actual_value, torch.Tensor):
|
||||
torch.testing.assert_close(actual_value, expected_value)
|
||||
elif field.name == "nums_dict":
|
||||
assert (actual_value is None) == (expected_value is None)
|
||||
if actual_value is not None:
|
||||
assert actual_value[8]["tot"] == expected_value[8]["tot"]
|
||||
torch.testing.assert_close(
|
||||
actual_value[8]["nums"], expected_value[8]["nums"]
|
||||
)
|
||||
else:
|
||||
assert actual_value == expected_value
|
||||
|
||||
|
||||
def _make_builder(
|
||||
builder_cls: type[AttentionMetadataBuilder],
|
||||
num_speculative_tokens: int,
|
||||
full_cuda_graph: bool,
|
||||
device: torch.device = DEVICE,
|
||||
mamba_cache_mode: str = "none",
|
||||
) -> AttentionMetadataBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
if num_speculative_tokens:
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
return builder_cls(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=num_speculative_tokens,
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"batch",
|
||||
"num_decode_draft_tokens",
|
||||
"num_speculative_tokens",
|
||||
"full_cuda_graph",
|
||||
"is_prefilling",
|
||||
),
|
||||
[
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[50, 30], query_lens=[3, 3]),
|
||||
[2, 2],
|
||||
2,
|
||||
False,
|
||||
[False, False],
|
||||
id="pure-spec-decode",
|
||||
),
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[100, 65, 20], query_lens=[50, 1, 3]),
|
||||
[-1, -1, 2],
|
||||
2,
|
||||
False,
|
||||
[True, False, False],
|
||||
id="mixed-prefill-and-spec-decode",
|
||||
),
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
|
||||
None,
|
||||
0,
|
||||
False,
|
||||
[False, False],
|
||||
id="regular-decode",
|
||||
),
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
|
||||
[0, 0],
|
||||
2,
|
||||
False,
|
||||
[False, False],
|
||||
id="no-scheduled-draft-tokens",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_kimi_k3_kda_metadata_matches_shared_gdn(
|
||||
batch: BatchSpec,
|
||||
num_decode_draft_tokens: list[int] | None,
|
||||
num_speculative_tokens: int,
|
||||
full_cuda_graph: bool,
|
||||
is_prefilling: list[bool],
|
||||
):
|
||||
kwargs: dict[str, torch.Tensor] = {}
|
||||
if num_decode_draft_tokens is not None:
|
||||
kwargs = {
|
||||
"num_decode_draft_tokens_cpu": torch.tensor(
|
||||
num_decode_draft_tokens, dtype=torch.int32
|
||||
),
|
||||
"num_accepted_tokens": torch.ones(
|
||||
batch.batch_size, dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
}
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor(is_prefilling, dtype=torch.bool))
|
||||
reference = _make_builder(
|
||||
GDNAttentionMetadataBuilder,
|
||||
num_speculative_tokens,
|
||||
full_cuda_graph,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
**kwargs,
|
||||
)
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens,
|
||||
full_cuda_graph,
|
||||
).build(0, common_attn_metadata, **kwargs)
|
||||
|
||||
assert isinstance(actual, KimiK3KDAMetadata)
|
||||
_assert_matches_shared_gdn(reference, actual)
|
||||
|
||||
|
||||
def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():
|
||||
batch = BatchSpec(seq_lens=[100, 65, 20], query_lens=[1, 1, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([False, False, False]))
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
# The K3 layer dispatches the non-spec subgroup to packed decode whenever
|
||||
# it contains no prefill request.
|
||||
assert actual.num_decodes == 2
|
||||
assert actual.num_decode_tokens == 2
|
||||
assert actual.num_prefills == 0
|
||||
assert actual.num_prefill_tokens == 0
|
||||
assert actual.has_initial_state is None
|
||||
assert actual.nums_dict is None
|
||||
assert actual.non_spec_query_start_loc is None
|
||||
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0, 1]))
|
||||
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([2, 3, 4]))
|
||||
torch.testing.assert_close(
|
||||
actual.spec_query_start_loc,
|
||||
torch.tensor([0, 3], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_regular_and_spec_decode_excludes_request_padding():
|
||||
batch = BatchSpec(seq_lens=[16, 65, 20], query_lens=[0, 1, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([False, False, False]))
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
assert actual.num_decodes == 1
|
||||
assert actual.non_spec_state_indices_tensor is not None
|
||||
assert actual.non_spec_state_indices_tensor.shape == (1,)
|
||||
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0]))
|
||||
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seq_len", "expected_has_initial_state"),
|
||||
[
|
||||
pytest.param(1, False, id="first-token-prefill"),
|
||||
pytest.param(65, True, id="final-one-token-prefill-chunk"),
|
||||
],
|
||||
)
|
||||
def test_mixed_one_token_prefill_and_spec_decode_uses_prefill_metadata(
|
||||
seq_len: int,
|
||||
expected_has_initial_state: bool,
|
||||
):
|
||||
batch = BatchSpec(seq_lens=[seq_len, 20], query_lens=[1, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([True, False]))
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.tensor([-1, 2], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(2, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
assert actual.num_prefills == 1
|
||||
assert actual.num_prefill_tokens == 1
|
||||
assert actual.num_decodes == 0
|
||||
assert actual.num_decode_tokens == 0
|
||||
assert actual.has_initial_state is not None
|
||||
assert actual.has_initial_state.tolist() == [expected_has_initial_state]
|
||||
assert actual.non_spec_query_start_loc is not None
|
||||
torch.testing.assert_close(
|
||||
actual.non_spec_query_start_loc,
|
||||
torch.tensor([0, 1], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_kimi_k3_kda_cudagraph_capture_matches_shared_gdn():
|
||||
device = torch.device("cuda")
|
||||
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device
|
||||
).replace(is_prefilling=torch.tensor([False, False]))
|
||||
reference = _make_builder(
|
||||
GDNAttentionMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=True,
|
||||
device=device,
|
||||
).build_for_cudagraph_capture(common_attn_metadata)
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=True,
|
||||
device=device,
|
||||
).build_for_cudagraph_capture(common_attn_metadata)
|
||||
|
||||
assert isinstance(actual, KimiK3KDAMetadata)
|
||||
_assert_matches_shared_gdn(reference, actual)
|
||||
|
||||
|
||||
def test_kimi_k3_kda_backend_uses_private_metadata_builder():
|
||||
assert KimiK3KDAAttentionBackend.get_builder_cls() is KimiK3KDAMetadataBuilder
|
||||
assert KimiK3KDAAttentionBackend.is_ssm()
|
||||
assert issubclass(KimiK3KDAAttentionBackend, GDNAttentionBackend)
|
||||
assert issubclass(KimiK3KDAMetadata, GDNAttentionMetadata)
|
||||
assert issubclass(KimiK3KDAMetadataBuilder, GDNAttentionMetadataBuilder)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_stage_spec_decode_metadata_matches_pytorch():
|
||||
device = torch.device("cuda")
|
||||
num_spec_decodes = 33
|
||||
batch_size = 65
|
||||
num_state_slots = 3
|
||||
state_indices = torch.arange(
|
||||
num_spec_decodes * 32,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).reshape(num_spec_decodes, 32)[:, :num_state_slots]
|
||||
query_start_loc = (
|
||||
torch.arange(num_spec_decodes + 1, dtype=torch.int32, device=device)
|
||||
* num_state_slots
|
||||
)
|
||||
num_accepted_tokens = (
|
||||
torch.arange(num_spec_decodes, dtype=torch.int32, device=device)
|
||||
% num_state_slots
|
||||
+ 1
|
||||
)
|
||||
|
||||
staged_state_indices = torch.empty(
|
||||
(batch_size, num_state_slots), dtype=torch.int32, device=device
|
||||
)
|
||||
staged_query_start_loc = torch.empty(
|
||||
batch_size + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
staged_num_accepted_tokens = torch.empty(
|
||||
batch_size, dtype=torch.int32, device=device
|
||||
)
|
||||
stage_spec_decode_metadata(
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
num_accepted_tokens,
|
||||
staged_state_indices,
|
||||
staged_query_start_loc,
|
||||
staged_num_accepted_tokens,
|
||||
num_spec_decodes=num_spec_decodes,
|
||||
)
|
||||
|
||||
expected_state_indices = torch.full_like(staged_state_indices, NULL_BLOCK_ID)
|
||||
expected_state_indices[:num_spec_decodes] = state_indices
|
||||
expected_query_start_loc = torch.full(
|
||||
(batch_size + 1,),
|
||||
query_start_loc[-1],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
expected_query_start_loc[: num_spec_decodes + 1] = query_start_loc
|
||||
expected_num_accepted_tokens = torch.ones(
|
||||
batch_size, dtype=torch.int32, device=device
|
||||
)
|
||||
expected_num_accepted_tokens[:num_spec_decodes] = num_accepted_tokens
|
||||
|
||||
torch.testing.assert_close(staged_state_indices, expected_state_indices)
|
||||
torch.testing.assert_close(staged_query_start_loc, expected_query_start_loc)
|
||||
torch.testing.assert_close(staged_num_accepted_tokens, expected_num_accepted_tokens)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_aligned_block_table_matches_shared_gdn():
|
||||
device = torch.device("cuda")
|
||||
seq_lens = torch.tensor(
|
||||
[0, 1, 15, 16, 17, 31, 32, 33, 511, 512, 513],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).repeat(6)[:65]
|
||||
block_table_storage = torch.arange(
|
||||
seq_lens.numel() * 128,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).reshape(seq_lens.numel(), 128)
|
||||
block_table = block_table_storage[:, ::2]
|
||||
kv_cache_spec = MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
|
||||
expected = mamba_get_block_table_tensor(
|
||||
block_table,
|
||||
seq_lens,
|
||||
kv_cache_spec,
|
||||
"align",
|
||||
)
|
||||
actual = _mamba_get_block_table_tensor(
|
||||
block_table,
|
||||
seq_lens,
|
||||
kv_cache_spec,
|
||||
"align",
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import (
|
||||
init_test_distributed_environment,
|
||||
multi_gpu_test,
|
||||
multi_process_parallel,
|
||||
)
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import KimiK3LatentMoETailOp
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
LATENT_SIZE = 3584
|
||||
EPS = 0.1
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _test_latent_moe_tail_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
) -> None:
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rms_weight = 1 + 0.1 * torch.randn(
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
up_weight = (
|
||||
torch.randn(
|
||||
HIDDEN_SIZE,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
/ LATENT_SIZE**0.5
|
||||
)
|
||||
|
||||
group = get_tp_group().device_group
|
||||
op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
latent_size=LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
rms_eps=EPS,
|
||||
)
|
||||
cutedsl_warmup()
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
|
||||
torch.manual_seed(100 * iteration + rank + 1)
|
||||
routed_output = torch.randn(
|
||||
num_tokens,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).mul_(0.01)
|
||||
shared_output = torch.randn(
|
||||
num_tokens,
|
||||
HIDDEN_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
routed_reference = routed_output.clone()
|
||||
shared_reference = shared_output.clone()
|
||||
dist.all_reduce(routed_reference, group=group)
|
||||
dist.all_reduce(shared_reference, group=group)
|
||||
expected = F.linear(
|
||||
F.rms_norm(
|
||||
routed_reference,
|
||||
(LATENT_SIZE,),
|
||||
rms_weight,
|
||||
EPS,
|
||||
),
|
||||
up_weight,
|
||||
)
|
||||
expected.add_(shared_reference)
|
||||
|
||||
actual = op(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_output = op(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(graph_output, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def _run_latent_moe_tail_test(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
) -> None:
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("K3 latent-MoE tail fusion requires SM100")
|
||||
multi_process_parallel(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
1,
|
||||
_test_latent_moe_tail_worker,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_latent_moe_tail_tp8_matches_native_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_run_latent_moe_tail_test(monkeypatch, 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=16)
|
||||
def test_latent_moe_tail_tp16_matches_native_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_run_latent_moe_tail_test(monkeypatch, 16)
|
||||
@@ -0,0 +1,343 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.models.kimi_k3.nvidia import model as kimi_model
|
||||
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
|
||||
from vllm.models.kimi_k3.nvidia.ops import sequence_parallel as sp_ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class _IdentityNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
|
||||
self.variance_epsilon = 1e-5
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
):
|
||||
if residual is None:
|
||||
return hidden_states
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class _RecordingMoE(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.num_tokens = 0
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
self.num_tokens = hidden_states.shape[0]
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _Projection(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.ones(1, hidden_size),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
|
||||
class _SequenceParallelMTPBlock:
|
||||
use_sequence_parallel = True
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
):
|
||||
assert residual is None
|
||||
return hidden_states * 2, None, hidden_states * 3
|
||||
|
||||
|
||||
def _mock_sequence_parallel_collectives(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_reduce_scatter",
|
||||
lambda tensor: tensor.chunk(2, dim=0)[0],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "is_padding", "tp_rank", "expected"),
|
||||
[
|
||||
(1, None, 0, [False]),
|
||||
(1, None, 1, [True]),
|
||||
(5, None, 2, [False, True]),
|
||||
(5, None, 3, [True, True]),
|
||||
(5, [False, True, False, False, False], 0, [False, True]),
|
||||
],
|
||||
)
|
||||
def test_sp_padding_mask_marks_added_rows(
|
||||
monkeypatch,
|
||||
num_tokens: int,
|
||||
is_padding: list[bool] | None,
|
||||
tp_rank: int,
|
||||
expected: list[bool],
|
||||
):
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_world_size", lambda: 4)
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
|
||||
hidden_states = torch.empty(num_tokens, 2)
|
||||
padding = torch.tensor(is_padding) if is_padding is not None else None
|
||||
actual = sp_ops.sp_padding_mask(padding, hidden_states)
|
||||
|
||||
torch.testing.assert_close(actual, torch.tensor(expected))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data_parallel_size", "expected"),
|
||||
[
|
||||
(1, False),
|
||||
(2, True),
|
||||
],
|
||||
)
|
||||
def test_moe_sequence_parallel_requires_data_parallel(
|
||||
monkeypatch,
|
||||
data_parallel_size: int,
|
||||
expected: bool,
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=data_parallel_size,
|
||||
enable_expert_parallel=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
)
|
||||
|
||||
assert parallel_config.use_sequence_parallel_moe is expected
|
||||
|
||||
|
||||
def test_kimi_decoder_layer_keeps_moe_states_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(kimi_model.KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_attn_res = False
|
||||
layer.use_sequence_parallel = True
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.mlp = _RecordingMoE()
|
||||
layer._run_self_attn = MethodType(
|
||||
lambda self, positions, hidden_states: hidden_states,
|
||||
layer,
|
||||
)
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch)
|
||||
|
||||
positions = torch.arange(3)
|
||||
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
hidden_states = kimi_model.sp_shard(full_hidden_states)
|
||||
hidden_states, prefix_sum, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
|
||||
assert prefix_sum is None
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
hidden_states, prefix_sum, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
assert prefix_sum is None
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_kimi_attn_residual_states_stay_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(kimi_model.KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_attn_res = True
|
||||
layer.use_sequence_parallel = True
|
||||
layer.prev_valid_blocks = 0
|
||||
layer.block_write_idx = 0
|
||||
layer.is_block_write_layer = False
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.self_attention_res_norm = _IdentityNorm()
|
||||
layer.mlp_res_norm = _IdentityNorm()
|
||||
layer.self_attention_res_proj = _Projection()
|
||||
layer.mlp_res_proj = _Projection()
|
||||
layer.mlp = _RecordingMoE()
|
||||
layer._run_self_attn = MethodType(
|
||||
lambda self, positions, hidden_states: hidden_states,
|
||||
layer,
|
||||
)
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"attn_res",
|
||||
lambda prefix_sum, hidden_states, *args, **kwargs: (
|
||||
prefix_sum if hidden_states is None else prefix_sum + hidden_states
|
||||
),
|
||||
)
|
||||
|
||||
prefix_sum = kimi_model.sp_shard(torch.arange(6, dtype=torch.float32).view(3, 2))
|
||||
block_residual = torch.zeros(2, 1, 2)
|
||||
hidden_states, prefix_sum, block_residual = layer(
|
||||
positions=torch.arange(3),
|
||||
hidden_states=None,
|
||||
prefix_sum=prefix_sum,
|
||||
residual=block_residual,
|
||||
)
|
||||
|
||||
assert hidden_states.shape == prefix_sum.shape == (2, 2)
|
||||
assert block_residual.shape == (2, 1, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
|
||||
layer = object.__new__(kimi_mtp.KimiK3MultiTokenPredictorLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.enorm = _IdentityNorm()
|
||||
layer.hnorm = _IdentityNorm()
|
||||
layer.eh_proj = nn.Identity()
|
||||
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
|
||||
|
||||
final_norm = Mock(side_effect=lambda hidden_states: hidden_states + 1)
|
||||
object.__setattr__(
|
||||
layer,
|
||||
"shared_head",
|
||||
SimpleNamespace(norm=final_norm),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"fused_mtp_input",
|
||||
lambda positions, inputs_embeds, *args: inputs_embeds,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
logits_hidden_states, hidden_states = layer(
|
||||
input_ids=torch.zeros(3, dtype=torch.long),
|
||||
positions=torch.arange(3),
|
||||
previous_hidden_states=torch.zeros_like(inputs_embeds),
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
|
||||
expected_hidden_states = torch.cat(
|
||||
[sharded_states * 5, sharded_states * 5],
|
||||
dim=0,
|
||||
)[:3]
|
||||
torch.testing.assert_close(hidden_states, expected_hidden_states)
|
||||
torch.testing.assert_close(logits_hidden_states, expected_hidden_states + 1)
|
||||
final_norm.assert_called_once()
|
||||
torch.testing.assert_close(final_norm.call_args.args[0], expected_hidden_states)
|
||||
|
||||
|
||||
def test_sp_all_gather_uses_custom_kernel(monkeypatch):
|
||||
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
expected = torch.cat([hidden_states, hidden_states])
|
||||
custom_all_gather = Mock(return_value=expected)
|
||||
device_communicator = SimpleNamespace(
|
||||
custom_all_gather=custom_all_gather,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_communicator),
|
||||
)
|
||||
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", fallback)
|
||||
|
||||
output = sp_ops.sp_all_gather(hidden_states)
|
||||
|
||||
torch.testing.assert_close(output, expected)
|
||||
custom_all_gather.assert_called_once_with(hidden_states)
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
|
||||
hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
expected = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
custom_reduce_scatter = Mock(return_value=expected)
|
||||
device_communicator = SimpleNamespace(
|
||||
custom_reduce_scatter=custom_reduce_scatter,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_communicator),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_reduce_scatter", fallback)
|
||||
|
||||
output = sp_ops.sp_reduce_scatter(hidden_states)
|
||||
|
||||
torch.testing.assert_close(output, expected)
|
||||
padded = custom_reduce_scatter.call_args.args[0]
|
||||
assert padded.shape == (4, 2)
|
||||
torch.testing.assert_close(padded[:3], hidden_states)
|
||||
torch.testing.assert_close(padded[3], torch.zeros(2))
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
|
||||
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
all_gather = Mock(return_value=hidden_states)
|
||||
reduce_scatter = Mock(return_value=hidden_states)
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", all_gather)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"tensor_model_parallel_reduce_scatter",
|
||||
reduce_scatter,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(sp_ops.sp_all_gather(hidden_states), hidden_states)
|
||||
torch.testing.assert_close(sp_ops.sp_reduce_scatter(hidden_states), hidden_states)
|
||||
all_gather.assert_called_once_with(hidden_states, 0)
|
||||
reduce_scatter.assert_called_once_with(hidden_states, 0)
|
||||
@@ -1242,7 +1242,7 @@ def test_custom_inputs_models(
|
||||
create_new_process_for_each_test=True,
|
||||
),
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_single_image_models_heavy(
|
||||
tmp_path: PosixPath,
|
||||
model_type: str,
|
||||
|
||||
Reference in New Issue
Block a user