[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
+117
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import ray
@@ -9,6 +11,8 @@ import torch
import torch.distributed as dist
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.device_communicators import custom_all_reduce
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from ..utils import (
@@ -23,6 +27,59 @@ for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
def test_sp16_dispatches_only_to_mnnvl_lamport(
monkeypatch: pytest.MonkeyPatch,
):
"""SP16 uses the MNNVL Lamport kernels and rejects same-host dispatch."""
comm = CustomAllreduce.__new__(CustomAllreduce)
comm.disabled = False
comm.world_size = 16
comm.fully_connected = False
comm.mnnvl_only = True
comm._IS_CAPTURING = False
comm.max_mnnvl_all_gather_size = 2 * 1024 * 1024
comm.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
comm.mnnvl_multicast_ptr = 1
comm.mnnvl_lamport_ag_local_ptr = 1
comm.mnnvl_lamport_ag_multicast_ptr = 1
comm.mnnvl_lamport_ag_epoch_ptr = 1
comm.mnnvl_lamport_rs_local_ptr = 1
comm.mnnvl_lamport_rs_epoch_ptr = 1
comm.mnnvl_buffer_size = 32 * 1024 * 1024
comm._ptr = 0
lamport_all_gather = Mock()
lamport_reduce_scatter = Mock()
monkeypatch.setattr(custom_all_reduce.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
custom_all_reduce,
"ops",
SimpleNamespace(
mnnvl_lamport_all_gather=lamport_all_gather,
mnnvl_lamport_reduce_scatter=lamport_reduce_scatter,
),
)
gathered = comm.custom_all_gather(torch.empty((8, 8), dtype=torch.bfloat16))
scattered = comm.custom_reduce_scatter(torch.empty((16, 8), dtype=torch.bfloat16))
assert gathered is not None
assert scattered is not None
lamport_all_gather.assert_called_once()
lamport_reduce_scatter.assert_called_once()
assert not comm.should_custom_ar(torch.empty(8, dtype=torch.bfloat16))
assert not comm.should_custom_all_gather(torch.empty((8, 8), dtype=torch.int32))
assert not comm.should_custom_all_gather(
torch.empty((131073, 8), dtype=torch.bfloat16)
)
comm.mnnvl_only = False
assert not comm.should_custom_all_gather(torch.empty((8, 8), dtype=torch.bfloat16))
assert not comm.should_custom_reduce_scatter(
torch.empty((16, 8), dtype=torch.bfloat16)
)
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,
@@ -80,6 +137,32 @@ def graph_allreduce(
torch.testing.assert_close(out1, inp1)
torch.testing.assert_close(out2, inp2)
fa = get_tp_group().device_communicator.ca_comm
tp_rank = rank % tp_size
with graph_capture(device=device) as graph_capture_context:
local = torch.full(
(512, 4096), tp_rank + 1, dtype=torch.bfloat16, device=device
)
reduce_input = torch.full(
(512 * tp_size, 4096),
tp_rank + 1,
dtype=torch.bfloat16,
device=device,
)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
gathered = fa.custom_all_gather(local)
scattered = fa.custom_reduce_scatter(reduce_input)
graph.replay()
assert gathered is not None
assert scattered is not None
expected_gather = torch.cat(
[torch.full_like(local, peer_rank + 1) for peer_rank in range(tp_size)]
)
expected_scatter = torch.full_like(local, tp_size * (tp_size + 1) // 2)
torch.testing.assert_close(gathered, expected_gather)
torch.testing.assert_close(scattered, expected_scatter)
@ray.remote(num_gpus=1, max_calls=1)
def eager_allreduce(
@@ -110,6 +193,29 @@ def eager_allreduce(
out = fa.all_reduce(out, registered=False)
torch.testing.assert_close(out, inp * (tp_size**num_communication))
group = get_tp_group().device_group
tp_rank = rank % tp_size
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
local = torch.full((64, 4096), tp_rank + 1, dtype=dtype, device=device)
expected_gather = torch.empty(
(64 * tp_size, 4096), dtype=dtype, device=device
)
dist.all_gather_into_tensor(expected_gather, local, group=group)
gathered = fa.custom_all_gather(local)
assert gathered is not None
torch.testing.assert_close(gathered, expected_gather)
reduce_input = torch.full(
(64 * tp_size, 4096), tp_rank + 1, dtype=dtype, device=device
)
expected_scatter = torch.empty((64, 4096), dtype=dtype, device=device)
dist.reduce_scatter_tensor(
expected_scatter, reduce_input.clone(), group=group
)
scattered = fa.custom_reduce_scatter(reduce_input)
assert scattered is not None
torch.testing.assert_close(scattered, expected_scatter)
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
out = inp
for _ in range(num_communication):
@@ -130,3 +236,14 @@ def test_custom_allreduce(
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
def test_custom_collectives_world_size_four(
monkeypatch: pytest.MonkeyPatch,
test_target,
):
"""Exercise the four-rank kernel specialization used by Kimi SP."""
if torch.accelerator.device_count() < 4:
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, 4, 1, test_target)
@@ -0,0 +1,221 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RoPE equivalence tests for the fused Kimi-K3 MLA epilogues."""
import pytest
import torch
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
)
_DTYPE = torch.bfloat16
_NUM_TOKENS = 3
_NUM_HEADS = 4
_BLOCK_SIZE = 8
_POSITIONS = (1, 7, 13)
_SLOTS = (0, 3, 9)
def _randn(*shape: int) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=_DTYPE) * 0.2
def _rope_cache(max_position: int = 32) -> torch.Tensor:
inv_freq = 1.0 / (
50000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device="cuda") / 64)
)
positions = torch.arange(max_position, dtype=torch.float32, device="cuda")
freqs = torch.outer(positions, inv_freq)
# The fused epilogue reads the cos/sin table in fp32 (RoPE math runs in fp32).
return torch.cat((freqs.cos(), freqs.sin()), dim=-1)
def _apply_gptj_rope(
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
) -> torch.Tensor:
cos, sin = cos_sin_cache.index_select(0, positions).chunk(2, dim=-1)
for _ in range(x.ndim - 2):
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
x1 = x[..., ::2].float()
x2 = x[..., 1::2].float()
out1 = x1 * cos.float() - x2 * sin.float()
out2 = x2 * cos.float() + x1 * sin.float()
return torch.stack((out1, out2), dim=-1).flatten(-2).to(x.dtype)
def _cache_rows(cache: torch.Tensor, slots: torch.Tensor) -> torch.Tensor:
return cache.reshape(-1, cache.shape[-1]).index_select(0, slots)
def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
torch.testing.assert_close(
actual.float(),
expected.to(torch.float8_e4m3fn).float(),
atol=0.03125,
rtol=0.15,
)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(0)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
q = _randn(_NUM_TOKENS, _NUM_HEADS, 192)
k_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
k_pe = _randn(_NUM_TOKENS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
v = _randn(_NUM_TOKENS, _NUM_HEADS, 128)
q_expected = q.clone()
q_expected[..., 128:] = _apply_gptj_rope(
q_expected[..., 128:], positions, cos_sin_cache
)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
k_expected = torch.cat(
(k_nope, k_pe_expected[:, None, :].expand(-1, _NUM_HEADS, -1)), dim=-1
)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = q.clone()
k_actual = fused_mla_key_concat_kv_cache_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual, k_actual, v_actual = fused_mla_qkv_quant_kv_cache_fp8_insert(
q,
k_nope,
k_pe,
kv_c,
v,
cache,
slots,
one,
one,
one,
one,
positions,
cos_sin_cache,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(k_actual, k_expected)
_assert_fp8_close(v_actual, v)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = q.clone()
k_actual = fused_mla_key_concat_ds_mla_insert(
q_actual,
k_nope,
k_pe,
kv_c,
cache,
slots,
positions,
cos_sin_cache,
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(k_actual, k_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_decode_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
torch.manual_seed(1)
positions = torch.tensor(_POSITIONS, device="cuda", dtype=torch.int64)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
cos_sin_cache = _rope_cache()
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
q_pe_expected = _apply_gptj_rope(q_pe, positions, cos_sin_cache)
k_pe_expected = _apply_gptj_rope(k_pe, positions, cos_sin_cache)
q_expected = torch.cat((ql_nope, q_pe_expected), dim=-1)
cache_expected = torch.cat((kv_c, k_pe_expected), dim=-1)
kwargs = {"positions": positions, "cos_sin_cache": cos_sin_cache}
if cache_kind == "bf16":
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, **kwargs
)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(_cache_rows(cache, slots), cache_expected)
elif cache_kind == "fp8":
cache = torch.zeros(
2, _BLOCK_SIZE, 576, device="cuda", dtype=torch.float8_e4m3fn
)
one = torch.ones(1, device="cuda", dtype=torch.float32)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope,
q_pe,
kv_c,
k_pe,
cache,
slots,
q_scale_inv=one,
cache_scale_inv=one,
**kwargs,
)
_assert_fp8_close(q_actual, q_expected)
_assert_fp8_close(_cache_rows(cache, slots), cache_expected)
else:
cache = torch.zeros(2, _BLOCK_SIZE, 656, device="cuda", dtype=torch.uint8)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots, ds_mla=True, **kwargs
)
rope_cache = _cache_rows(cache, slots)[:, 528:656].view(_DTYPE)
torch.testing.assert_close(q_actual, q_expected)
torch.testing.assert_close(rope_cache, k_pe_expected)
@torch.inference_mode()
def test_decode_epilogue_preserves_nope_path() -> None:
torch.manual_seed(2)
slots = torch.tensor(_SLOTS, device="cuda", dtype=torch.int64)
ql_nope = _randn(_NUM_TOKENS, _NUM_HEADS, 512)
q_pe = _randn(_NUM_TOKENS, _NUM_HEADS, 64)
kv_c = _randn(_NUM_TOKENS, 512)
k_pe = _randn(_NUM_TOKENS, 64)
cache = torch.zeros(2, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots
)
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)
+40
View File
@@ -197,6 +197,46 @@ def test_silu_and_mul_with_clamp(
opcheck(torch.ops._C.silu_and_mul_with_clamp, (out_buf, x, swiglu_limit))
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
@torch.inference_mode()
def test_masked_situ_and_mul(
default_vllm_config,
linear_beta: float,
dtype: torch.dtype,
) -> None:
"""Masked SITU computes valid expert rows and preserves padded zeros."""
device = CUDA_DEVICES[0]
num_experts, max_num_tokens, d = 4, 7, 512
beta = 1.5
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
torch.ops._C.masked_situ_and_mul(
output, input, expert_num_tokens, beta, linear_beta
)
gate, up = input.float().chunk(2, dim=-1)
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
if linear_beta > 0:
up = linear_beta * torch.tanh(up / linear_beta)
expected = (expected * up).to(dtype)
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
torch.testing.assert_close(
output[expert, :num_tokens],
expected[expert, :num_tokens],
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
opcheck(
torch.ops._C.masked_situ_and_mul,
(output, input, expert_num_tokens, beta, linear_beta),
)
@pytest.mark.parametrize(
"activation",
[
@@ -13,7 +13,7 @@ from __future__ import annotations
import pytest
import torch
from vllm.models.deepseek_v4.common.ops import fused_q_kv_rmsnorm
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
@@ -7,7 +7,9 @@ matching the eager triton kernel output."""
import pytest
import torch
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.third_party.flash_linear_attention.ops.fused_norm_gate import (
FusedRMSNormGated,
)
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.bfloat16]
+266
View File
@@ -23,6 +23,53 @@ from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
def _run_single_group_topk(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
return fused_grouped_topk(
hidden_states=torch.empty(
(logits.shape[0], 0), dtype=logits.dtype, device=logits.device
),
gating_output=logits,
topk=topk,
renormalize=renormalize,
e_score_correction_bias=bias,
num_expert_group=1,
topk_group=1,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
)
def _single_group_reference(
logits: torch.Tensor,
bias: torch.Tensor,
topk: int,
*,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float = 1.0,
) -> tuple[torch.Tensor, torch.Tensor]:
if scoring_func == "sigmoid":
scores = 0.5 * torch.tanh(0.5 * logits.float()) + 0.5
else:
scores = torch.softmax(logits, dim=-1).float()
indices = torch.argsort(
scores + bias.float(), dim=-1, descending=True, stable=True
)[:, :topk]
values = scores.gather(1, indices)
if renormalize:
values /= values.sum(dim=-1, keepdim=True) + 1e-20
values *= routed_scaling_factor
return values, indices.to(torch.int32)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@@ -101,3 +148,222 @@ def test_grouped_topk(
baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0
)
torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
def test_grouped_topk_single_group_large_batch():
set_random_seed(0)
logits = torch.randn((1536, 896), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((896,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
actual_values, actual_ids = _run_single_group_topk(
logits, bias, 16, scoring_func="sigmoid", renormalize=True
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,input_dtype,bias_dtype",
[
(512, 9, torch.bfloat16, torch.float32),
(512, 16, torch.float16, torch.float16),
(513, 9, torch.float32, torch.bfloat16),
(513, 16, torch.bfloat16, torch.float32),
(895, 9, torch.float16, torch.bfloat16),
(896, 16, torch.float32, torch.float16),
(897, 9, torch.bfloat16, torch.bfloat16),
(897, 16, torch.float16, torch.float32),
(1024, 9, torch.float32, torch.bfloat16),
(1024, 16, torch.bfloat16, torch.float16),
],
)
@pytest.mark.parametrize(
"scoring_func,renormalize,routed_scaling_factor",
[
("sigmoid", True, 1.0),
("sigmoid", False, 2.5),
("softmax", True, 2.5),
("softmax", False, 1.0),
],
)
def test_grouped_topk_single_group_tiers(
num_experts: int,
topk: int,
input_dtype: torch.dtype,
bias_dtype: torch.dtype,
scoring_func: str,
renormalize: bool,
routed_scaling_factor: float,
):
set_random_seed(7)
logits = torch.randn((17, num_experts), dtype=input_dtype, device="cuda")
bias = torch.randn((num_experts,), dtype=bias_dtype, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize(
"num_experts,topk,scoring_func",
[
(128, 8, "sigmoid"),
(129, 8, "sigmoid"),
(257, 8, "sigmoid"),
(385, 8, "sigmoid"),
(512, 9, "sigmoid"),
(513, 9, "sigmoid"),
(769, 9, "sigmoid"),
(897, 16, "sigmoid"),
(1024, 16, "sigmoid"),
(128, 4, "softmax"),
(128, 5, "softmax"),
(129, 8, "softmax"),
(161, 8, "softmax"),
(256, 9, "softmax"),
(257, 8, "softmax"),
(512, 9, "softmax"),
(512, 17, "softmax"),
(512, 23, "softmax"),
(513, 8, "softmax"),
(577, 9, "softmax"),
(769, 9, "softmax"),
(897, 9, "softmax"),
(1024, 16, "softmax"),
],
)
def test_grouped_topk_single_group_capacity_tiers(
num_experts: int,
topk: int,
scoring_func: str,
):
set_random_seed(11)
logits = torch.randn((3, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
expected_values, expected_ids = _single_group_reference(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
topk,
scoring_func=scoring_func,
renormalize=True,
routed_scaling_factor=2.5,
)
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
def test_grouped_topk_single_group_stable_ties(num_experts: int):
logits = torch.zeros((1, num_experts), dtype=torch.bfloat16, device="cuda")
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=True,
routed_scaling_factor=2.5,
)
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
expected_values = torch.full((1, 16), 2.5 / 16, dtype=torch.float32, device="cuda")
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_experts", [512, 896, 1024])
@pytest.mark.parametrize("num_finite", [0, 15])
@pytest.mark.parametrize("renormalize", [False, True])
def test_grouped_topk_single_group_nonfinite_scores(
num_experts: int, num_finite: int, renormalize: bool
):
logits = torch.full(
(1, num_experts), float("nan"), dtype=torch.bfloat16, device="cuda"
)
if num_finite:
logits[0, :num_finite] = torch.arange(
num_finite, dtype=torch.bfloat16, device="cuda"
)
logits[0, num_finite] = torch.inf
logits[0, num_finite + 1] = -torch.inf
bias = torch.zeros((num_experts,), dtype=torch.float32, device="cuda")
actual_values, actual_ids = _run_single_group_topk(
logits,
bias,
16,
scoring_func="sigmoid",
renormalize=renormalize,
routed_scaling_factor=2.5,
)
if num_finite == 0:
expected_ids = torch.arange(16, dtype=torch.int32, device="cuda")[None]
if renormalize:
expected_values = torch.full(
(1, 16), 1 / 16, dtype=torch.float32, device="cuda"
)
else:
expected_values = torch.zeros((1, 16), dtype=torch.float32, device="cuda")
else:
expected_ids = torch.cat(
(
torch.arange(num_finite - 1, -1, -1, dtype=torch.int32, device="cuda"),
torch.tensor([num_finite], dtype=torch.int32, device="cuda"),
)
)[None]
finite_values = logits[0, :num_finite].float().sigmoid().flip(0)
if renormalize:
finite_values /= finite_values.sum()
finite_values *= 2.5
expected_values = torch.cat(
(finite_values, torch.zeros(1, dtype=torch.float32, device="cuda"))
)[None]
torch.testing.assert_close(actual_ids, expected_ids)
torch.testing.assert_close(actual_values, expected_values, atol=2e-5, rtol=0)
@@ -269,6 +269,7 @@ def test_moe_align_block_size_with_expert_map(
if (experts[k] in local_experts) or not mask_inactive_experts
else -1
)
topk_ids[0, 0] = -1
actual_sorted_ids, actual_expert_ids, actual_num_tokens = moe_align_block_size(
topk_ids=topk_ids,
+649
View File
@@ -0,0 +1,649 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch)."""
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
import regex as re
import torch
from torch import nn
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig
from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
# Keyed by local (N, K): (cute token counts, dsv3 token counts). 1536x7168 is
# the unified shared_gate_up_proj/mla_g_proj entry (dsv3 M1..16).
EXPECTED_SELECTIONS = {
(1536, 128): (set(), set(range(1, 17))),
(3072, 128): (set(), set(range(1, 17))),
(1536, 7168): (set(), set(range(1, 17))),
(3072, 7168): (set(range(1, 6)), set()),
(2112, 7168): (set(), set(range(1, 17))),
(2304, 1536): (set(), set(range(1, 17))),
(4608, 1536): (set(), set(range(1, 17))),
(3584, 7168): ({1}, set(range(2, 9))),
(6288, 7168): (set(range(1, 5)), set()),
(12448, 7168): (set(range(1, 4)), set()),
(7168, 768): (set(), set(range(1, 17))),
(7168, 1536): ({1}, set()),
(7168, 3072): ({1, 2}, set()),
(7168, 3584): ({1, 2}, set()),
(7168, 4224): ({1}, set()),
(7168, 8448): (set(range(1, 4)), set()),
(8448, 7168): ({1, 2}, set()),
(16896, 7168): ({1, 2}, set()),
(20480, 7168): (set(range(1, 5)), set()),
(40960, 7168): (set(range(1, 5)), set()),
# TP16.
(3216, 7168): (set(range(1, 6)), set(range(9, 16))),
(768, 7168): (set(range(1, 5)), set(range(5, 17))),
(1152, 1536): ({1}, set(range(2, 17))),
(768, 128): (set(), set(range(1, 17))),
(7168, 384): (set(), set(range(1, 9))),
(4224, 7168): (set(range(1, 4)), set(range(4, 9))),
(10240, 7168): (set(range(1, 5)), set()),
}
CUTE_CASES = [
(spec.n, spec.k, num_tokens)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, _ in spec.cute_configs
]
RESIDUAL_CUTE_CASES = [
(spec.n, spec.k, num_tokens)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, _ in spec.residual_configs
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
(3072, 7168, 3): (128, 2, 1, 8),
(3072, 7168, 4): (64, 2, 2, 8),
(3072, 7168, 5): (128, 3, 1, 8),
(3584, 7168, 1): (224, 2, 4, 8),
(6288, 7168, 1): (224, 3, 4, 8),
(6288, 7168, 2): (64, 3, 2, 8),
(6288, 7168, 3): (32, 3, 4, 8),
(6288, 7168, 4): (128, 6, 1, 8),
(12448, 7168, 1): (224, 4, 2, 8),
(12448, 7168, 2): (64, 4, 2, 8),
(12448, 7168, 3): (64, 2, 2, 8),
(7168, 1536, 1): (96, 4, 2, 8),
(7168, 3072, 1): (96, 2, 4, 8),
(7168, 3072, 2): (32, 4, 4, 8),
(7168, 3584, 1): (224, 4, 2, 8),
(7168, 3584, 2): (64, 4, 2, 8),
(7168, 4224, 1): (96, 4, 2, 4),
(7168, 8448, 1): (32, 4, 4, 8),
(7168, 8448, 2): (96, 4, 1, 8),
(7168, 8448, 3): (96, 4, 1, 8),
(8448, 7168, 1): (224, 3, 4, 8),
(8448, 7168, 2): (32, 4, 4, 8),
(16896, 7168, 1): (224, 6, 4, 8),
(16896, 7168, 2): (32, 4, 4, 8),
(20480, 7168, 1): (224, 4, 2, 8),
(20480, 7168, 2): (64, 4, 2, 8),
(20480, 7168, 3): (64, 2, 2, 8),
(20480, 7168, 4): (64, 4, 1, 8),
(40960, 7168, 1): (128, 4, 2, 8),
(40960, 7168, 2): (64, 4, 2, 8),
(40960, 7168, 3): (64, 2, 2, 8),
(40960, 7168, 4): (64, 4, 1, 8),
# TP16.
(3216, 7168, 1): (224, 3, 4, 8),
(3216, 7168, 2): (128, 4, 2, 8),
(3216, 7168, 3): (128, 2, 1, 8),
(3216, 7168, 4): (64, 2, 2, 8),
(3216, 7168, 5): (128, 3, 1, 8),
(768, 7168, 1): (224, 2, 4, 8),
(768, 7168, 2): (224, 2, 2, 8),
(768, 7168, 3): (224, 2, 2, 8),
(768, 7168, 4): (224, 2, 2, 8),
(1152, 1536, 1): (192, 3, 4, 8),
(4224, 7168, 1): (224, 3, 4, 8),
(4224, 7168, 2): (128, 2, 1, 8),
(4224, 7168, 3): (64, 2, 2, 8),
(10240, 7168, 1): (224, 4, 2, 8),
(10240, 7168, 2): (32, 2, 4, 8),
(10240, 7168, 3): (64, 4, 1, 8),
(10240, 7168, 4): (64, 4, 1, 8),
}
EXPECTED_RESIDUAL_CUTE_CONFIGS = {
(7168, 3584, 1): (64, 4, 2, 8),
(7168, 3584, 2): (64, 7, 2, 8),
(7168, 3584, 3): (64, 2, 1, 8),
(7168, 3584, 4): (64, 2, 1, 8),
}
def _config_tuple(config) -> tuple[int, int, int, int]:
return (
config.block_size,
config.outputs_per_block,
config.k_unroll,
config.vector_width,
)
def test_table_is_keyed_by_shape() -> None:
for (n, k), spec in k3_gemm.KIMI_K3_PROJECTIONS.items():
assert (spec.n, spec.k) == (n, k)
def test_every_dsv3_routed_shape_is_instantiated() -> None:
"""dsv3_fused_a_gemm specializes on (K, N); an unlisted shape raises.
The table routes by shape while the kernel is built per shape, so a missing
instantiation only shows up at the token counts that route to dsv3. Checking
it here needs no GPU, which is the point -- a GPU-only check is exactly what
let (3216, 7168) ship without its DISPATCH_DSV3_SHAPE(7168, 3216).
"""
source = (
Path(__file__).resolve().parents[2]
/ "csrc"
/ "libtorch_stable"
/ "dsv3_fused_a_gemm.cu"
).read_text(encoding="utf-8")
# Benchmark-only shapes live behind VLLM_K3_BENCH_SHAPES and are not built
# by default, so they must not count as available.
production_macros = source.split("#ifdef VLLM_K3_BENCH_SHAPES")[0]
explicit = source.split("#undef DISPATCH_DSV3_SHAPE")[1].split(
"#ifdef VLLM_K3_BENCH_SHAPES"
)[0]
compiled = {
(int(hd_in), int(hd_out))
for hd_in, hd_out in re.findall(
r"DISPATCH_DSV3_SHAPE\((\d+),\s*(\d+)\)", production_macros
)
} | {
(int(hd_in), int(hd_out))
for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit)
}
assert compiled, "failed to parse the dispatch list"
missing = sorted(
(spec.n, spec.k)
for spec in KIMI_K3_PROJECTIONS.values()
if spec.dsv3_tokens and (spec.k, spec.n) not in compiled
)
assert not missing, (
f"routed to dsv3 with no instantiation: {missing}; add "
"DISPATCH_DSV3_SHAPE(K, N) for each"
)
def test_packed_row_major_rejects_single_row_slice() -> None:
packed = torch.empty(1, 128)
sliced = torch.empty(1, 144)[:, :128]
assert packed.is_contiguous()
assert sliced.is_contiguous()
assert k3_gemm._is_packed_row_major(packed)
assert not k3_gemm._is_packed_row_major(sliced)
def test_cute_configs_match_measured_table() -> None:
actual = {
(spec.n, spec.k, num_tokens): _config_tuple(config)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, config in spec.cute_configs
}
assert actual == EXPECTED_CUTE_CONFIGS
def test_residual_cute_configs_match_measured_table() -> None:
actual = {
(spec.n, spec.k, num_tokens): _config_tuple(config)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, config in spec.residual_configs
}
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
def test_sm103_selector_table(key: tuple[int, int]) -> None:
n, k = key
cute_tokens, dsv3_tokens = EXPECTED_SELECTIONS[key]
for num_tokens in range(1, 17):
backend = k3_gemm.select_kimi_k3_backend(num_tokens, n, k)
if num_tokens in cute_tokens:
assert backend == "cute"
elif num_tokens in dsv3_tokens:
assert backend == "dsv3_fused_a"
else:
assert backend is None
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
def test_selector_requires_supported_shape_and_tokens(key: tuple[int, int]) -> None:
n, k = key
assert k3_gemm.select_kimi_k3_backend(0, n, k) is None
assert k3_gemm.select_kimi_k3_backend(17, n, k) is None
assert k3_gemm.select_kimi_k3_backend(1, n + 1, k) is None
assert k3_gemm.select_kimi_k3_backend(1, n, k + 1) is None
def test_unlisted_shape_and_unselected_tokens_fall_back() -> None:
# Shape absent from the table.
assert k3_gemm.select_kimi_k3_backend(1, 1000, 1000) is None
# o_proj (7168,1536) is CuTe M1 only; M2+ falls back.
assert k3_gemm.select_kimi_k3_backend(2, 7168, 1536) is None
@pytest.mark.parametrize("num_tokens", range(1, 17))
def test_sm103_residual_selector_table(num_tokens: int) -> None:
backend = k3_gemm.select_kimi_k3_backend(num_tokens, 7168, 3584, has_residual=True)
assert backend == ("cute" if num_tokens <= 4 else None)
def test_build_plan_matches_selector() -> None:
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values():
plan = k3_gemm._build_plan(spec)
for num_tokens in range(1, 17):
backend = k3_gemm.select_kimi_k3_backend(num_tokens, spec.n, spec.k)
if backend is None:
assert num_tokens not in plan
else:
assert plan[num_tokens][0] == backend
def test_installation_is_shape_specific_and_unquantized(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class FakeLinear(nn.Module):
def __init__(self, quant_method: object, n: int, k: int) -> None:
super().__init__()
self.quant_method = quant_method
self.weight = torch.empty(n, k)
class FakeHead(nn.Module):
def __init__(self, n: int, k: int) -> None:
super().__init__()
self.quant_method = k3_gemm.UnquantizedEmbeddingMethod()
self.weight = torch.empty(n, k)
root = nn.Module()
# dsv3-only shape (no cute warmup contribution).
root.dsv3_only = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 2304, 1536)
# quantized: must be left untouched.
quantized_method = object()
root.quantized = FakeLinear(quantized_method, 6288, 7168)
# cute shape.
root.cute = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 6288, 7168)
# cute + residual shape.
root.residual = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 7168, 3584)
# shape absent from the table: must be left untouched.
root.unlisted = FakeLinear(k3_gemm.UnquantizedLinearMethod(), 1234, 5678)
root.lm_head = FakeHead(20480, 7168)
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
monkeypatch.setattr(k3_gemm, "ParallelLMHead", FakeHead)
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: True)
warmup_configs: set[SkinnyGemmConfig] = set()
residual_warmup_configs: set[SkinnyGemmConfig] = set()
monkeypatch.setattr(k3_gemm.shape_dynamic_skinny_gemm, "is_available", lambda: True)
def request_warmup_configs(dtype, configs, *, has_residual=False):
target = residual_warmup_configs if has_residual else warmup_configs
target.update(configs)
monkeypatch.setattr(
k3_gemm.shape_dynamic_skinny_gemm,
"request_warmup_configs",
request_warmup_configs,
)
k3_gemm.enable_kimi_k3_low_latency_gemm(root, torch.bfloat16)
assert isinstance(root.dsv3_only.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
assert isinstance(root.cute.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
assert isinstance(root.residual.quant_method, k3_gemm.KimiK3LowLatencyLinearMethod)
assert root.quantized.quant_method is quantized_method
assert type(root.unlisted.quant_method) is k3_gemm.UnquantizedLinearMethod
assert isinstance(
root.lm_head.quant_method, k3_gemm.KimiK3LowLatencyEmbeddingMethod
)
# Warmup covers only the installed modules' local (N, K).
assert warmup_configs == {
config
for key in ((6288, 7168), (7168, 3584), (20480, 7168))
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[key].cute_configs
}
assert residual_warmup_configs == {
config
for _, config in k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)].residual_configs
}
@pytest.mark.parametrize(
"dtype,platform_enabled",
[(torch.float16, True), (torch.bfloat16, False)],
)
def test_installation_requires_bf16_sm103(
monkeypatch: pytest.MonkeyPatch,
dtype: torch.dtype,
platform_enabled: bool,
) -> None:
class FakeLinear(nn.Module):
def __init__(self) -> None:
super().__init__()
self.quant_method = k3_gemm.UnquantizedLinearMethod()
self.weight = torch.empty(2304, 1536)
root = nn.Module()
root.projection = FakeLinear()
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: platform_enabled)
k3_gemm.enable_kimi_k3_low_latency_gemm(root, dtype)
assert type(root.projection.quant_method) is k3_gemm.UnquantizedLinearMethod
def _require_sm103_and_dsv3() -> None:
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
pytest.skip("Kimi-K3 production selection requires SM103")
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
pytest.skip("dsv3_fused_a_gemm was not built")
def _require_sm103_and_cute() -> None:
if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 3):
pytest.skip("Kimi-K3 production selection requires SM103")
if not k3_gemm.shape_dynamic_skinny_gemm.is_available():
pytest.skip("CuTe DSL is not available")
@pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES)
def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None:
_require_sm103_and_cute()
torch.manual_seed(42)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
output = k3_gemm.try_low_latency_gemm(x, weight)
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def _dsv3_probe_tokens(tokens: frozenset[int]) -> set[int]:
"""Extremes, plus both sides of the kernel's num_tokens<=8 tile_n branch."""
if not tokens:
return set()
return {min(tokens), max(tokens)} | ({8, 9} & set(tokens))
# Derived from the table rather than hand-listed, so a shape routed to dsv3
# cannot be added without being exercised here.
DSV3_CASES = sorted(
(num_tokens, spec.n, spec.k)
for spec in KIMI_K3_PROJECTIONS.values()
for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens)
)
@pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES)
def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
_require_sm103_and_dsv3()
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
assert num_tokens in spec.dsv3_tokens
torch.manual_seed(42)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
output = k3_gemm.try_low_latency_gemm(x, weight)
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_nonpacked_single_token_dsv3_falls_back() -> None:
_require_sm103_and_dsv3()
n, k = 1536, 128
storage = torch.randn(1, k + 16, dtype=torch.bfloat16, device="cuda")
x = storage[:, :k]
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
method = k3_gemm.KimiK3LowLatencyLinearMethod(
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
)
assert x.is_contiguous()
assert x.stride() == (k + 16, 1)
assert not k3_gemm._runtime_ok(x, weight) # strict guard rejects the view
output = method.apply(SimpleNamespace(weight=weight), x)
reference = torch.nn.functional.linear(x, weight)
torch.testing.assert_close(output, reference)
def test_selected_kernels_cuda_graph_capture() -> None:
_require_sm103_and_cute()
_require_sm103_and_dsv3()
cute_spec = k3_gemm.KIMI_K3_PROJECTIONS[(6288, 7168)]
dsv3_spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
cute_x = torch.randn(1, cute_spec.k, dtype=torch.bfloat16, device="cuda")
cute_weight = torch.randn(
cute_spec.n, cute_spec.k, dtype=torch.bfloat16, device="cuda"
)
dsv3_x = torch.randn(1, dsv3_spec.k, dtype=torch.bfloat16, device="cuda")
dsv3_weight = torch.randn(
dsv3_spec.n, dsv3_spec.k, dtype=torch.bfloat16, device="cuda"
)
k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
cute_output = k3_gemm.try_low_latency_gemm(cute_x, cute_weight)
dsv3_output = k3_gemm.try_low_latency_gemm(dsv3_x, dsv3_weight)
graph.replay()
torch.accelerator.synchronize()
assert cute_output is not None
assert dsv3_output is not None
for output, activation, weight in (
(cute_output, cute_x, cute_weight),
(dsv3_output, dsv3_x, dsv3_weight),
):
reference = torch.nn.functional.linear(activation, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
@pytest.mark.parametrize("num_tokens", [1, 8, 9, 16])
def test_dsv3_cuda_graph_capture_tile_branches(num_tokens: int) -> None:
"""Capture DSV3 across the num_tokens<=8 vs >8 tile_n branch."""
_require_sm103_and_dsv3()
spec = k3_gemm.KIMI_K3_PROJECTIONS[(1536, 128)]
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
k3_gemm.try_low_latency_gemm(x, weight)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = k3_gemm.try_low_latency_gemm(x, weight)
graph.replay()
torch.accelerator.synchronize()
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
@pytest.mark.parametrize("n,k,num_tokens", RESIDUAL_CUTE_CASES)
def test_cute_residual_epilogue(n: int, k: int, num_tokens: int) -> None:
_require_sm103_and_cute()
torch.manual_seed(42 + num_tokens)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
config = spec.residual_config(num_tokens)
assert config is not None
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
reference = x.float() @ weight.float().t() + residual.float()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.flatten(), dim=0
).item()
assert cosine > 0.999
@pytest.mark.parametrize("num_tokens", range(1, 17))
def test_cute_residual_epilogue_all_supported_token_counts(num_tokens: int) -> None:
_require_sm103_and_cute()
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
ShapeDynamicSkinnyGemm,
)
n, k = 64, 512
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(num_tokens, n, dtype=torch.bfloat16, device="cuda")
config = ShapeDynamicSkinnyGemm._config(num_tokens, n, k)
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
reference = x.float() @ weight.float().t() + residual.float()
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
@pytest.mark.parametrize("num_tokens", range(1, 5))
def test_cute_residual_epilogue_cuda_graph_capture(num_tokens: int) -> None:
_require_sm103_and_cute()
spec = k3_gemm.KIMI_K3_PROJECTIONS[(7168, 3584)]
config = spec.residual_config(num_tokens)
assert config is not None
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(num_tokens, spec.n, dtype=torch.bfloat16, device="cuda")
k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = k3_gemm.shape_dynamic_skinny_gemm(x, weight, config, residual)
graph.replay()
torch.accelerator.synchronize()
reference = x.float() @ weight.float().t() + residual.float()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.flatten(), dim=0
).item()
assert cosine > 0.999
class _SkinnyGemmSpy:
"""Wraps the skinny-GEMM singleton to record whether CuTe was invoked."""
def __init__(self, real: Any) -> None:
self._real = real
self.calls: list[int] = []
def __call__(self, a, b, config=None, residual=None):
self.calls.append(a.shape[0])
return self._real(a, b, config, residual)
def is_available(self) -> bool:
return self._real.is_available()
@pytest.mark.parametrize("num_tokens", [1, 2, 3, 4])
def test_latent_moe_production_layout_residual(
monkeypatch: pytest.MonkeyPatch,
num_tokens: int,
) -> None:
"""The real Latent-MoE residual is a non-packed slice of a cat buffer.
The strict packed-row-major guard rejects such a slice at every token count
(a size-1 leading dim reads as contiguous but its stride is not packed), so
the CuTe residual epilogue never fires for this production layout and the
method falls back to addmm. Output is correct regardless of the path.
"""
_require_sm103_and_cute()
latent_dim, shared_dim = 3584, 7168 # routed_expert_up_proj K, N
torch.manual_seed(7 + num_tokens)
buf = torch.randn(
num_tokens, latent_dim + shared_dim, dtype=torch.bfloat16, device="cuda"
)
latent = buf[:, :latent_dim] # non-contiguous view (row stride = full width)
residual = buf[:, latent_dim:] # non-contiguous view
weight = torch.randn(shared_dim, latent_dim, dtype=torch.bfloat16, device="cuda")
spec = k3_gemm.KIMI_K3_PROJECTIONS[(shared_dim, latent_dim)]
method = k3_gemm.KimiK3LowLatencyLinearMethod(
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
)
spy = _SkinnyGemmSpy(k3_gemm.shape_dynamic_skinny_gemm)
monkeypatch.setattr(k3_gemm, "shape_dynamic_skinny_gemm", spy)
layer = SimpleNamespace(weight=weight)
output = method.apply_with_residual(layer, latent, residual)
reference = latent.float() @ weight.float().t() + residual.float()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.flatten(), dim=0
).item()
assert cosine > 0.999 # correct regardless of the path taken
assert not spy.calls, (
"non-packed buf-slice residual must fall back to addmm at every M"
)
def test_residual_dispatch_falls_back_to_addmm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fallback = torch.randn(2, 3)
residual = torch.randn(2, 3)
x = torch.randn(2, 4)
weight = torch.randn(3, 4)
monkeypatch.setattr(torch, "addmm", lambda *args: fallback)
# CPU tensors fail the runtime check, forcing the addmm fallback.
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
output = method.apply_with_residual(SimpleNamespace(weight=weight), x, residual)
assert output is fallback
def test_fallback_preserves_default_method(monkeypatch: pytest.MonkeyPatch) -> None:
fallback = torch.empty(2, 8)
monkeypatch.setattr(
k3_gemm.UnquantizedLinearMethod,
"apply",
lambda *args: fallback,
)
# 1-D input fails the runtime check, forcing the base-method fallback.
method = k3_gemm.KimiK3LowLatencyLinearMethod({}, {})
output = method.apply(
SimpleNamespace(weight=torch.empty(0)),
torch.empty(0),
)
assert output is fallback
-226
View File
@@ -1,226 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Precision tests for vllm's chunk_kda Triton operator.
Compares chunk_kda against a naive recurrent reference (float32).
Uses torch.rand for q/k/v to match FLA's test pattern.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm.third_party.flash_linear_attention.ops.kda import (
chunk_kda,
chunk_kda_with_fused_gate,
fused_kda_gate,
)
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
DEVICE = "cuda"
def naive_recurrent_kda(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
scale: float | None = None,
initial_state: torch.Tensor | None = None,
output_final_state: bool = False,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
dtype = v.dtype
B, T, H, K = q.shape
V = v.shape[-1]
if scale is None:
scale = K**-0.5
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
q = q * scale
S = k.new_zeros(B, H, K, V).to(q)
if initial_state is not None:
S += initial_state
o = torch.zeros_like(v)
for i in range(T):
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
S = S * g_i[..., None].exp()
S = S + torch.einsum(
"bhk,bhv->bhkv",
b_i[..., None] * k_i,
v_i - (k_i[..., None] * S).sum(-2),
)
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
if not output_final_state:
S = None
return o.to(dtype), S
def assert_close(
name: str,
ref: torch.Tensor,
tri: torch.Tensor,
ratio: float,
err_atol: float = 1e-6,
):
"""RMSE-based relative error comparison."""
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
rel_err = rmse_diff / (rmse_base + 1e-8)
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
if abs_err <= err_atol:
return
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
assert rel_err < ratio, (
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
)
@pytest.mark.parametrize(
("H", "D", "cu_seqlens", "dtype"),
[
pytest.param(
*test,
id="H{}-D{}-cu{}-{}".format(*test),
)
for test in [
(32, 128, [0, 64], torch.float16),
(32, 128, [0, 1024], torch.float16),
(32, 128, [0, 15], torch.float16),
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
(64, 128, [0, 256, 500, 1000], torch.float16),
(32, 128, [0, 8192], torch.float16),
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
]
],
)
@torch.inference_mode()
def test_chunk_kda(
H: int,
D: int,
cu_seqlens: list[int],
dtype: torch.dtype,
):
T = cu_seqlens[-1]
torch.manual_seed(42)
B = 1
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
N = len(cu_seqlens) - 1
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
dtype
)
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
ref_outputs = []
ref_states = []
for i in range(N):
s, e = cu_seqlens[i], cu_seqlens[i + 1]
q_i = l2norm_fwd(q[:, s:e].contiguous())
k_i = l2norm_fwd(k[:, s:e].contiguous())
o_i, ht_i = naive_recurrent_kda(
q_i,
k_i,
v[:, s:e],
g[:, s:e],
beta[:, s:e],
initial_state=h0[i],
output_final_state=True,
)
ref_outputs.append(o_i)
ref_states.append(ht_i)
ref_o = torch.cat(ref_outputs, dim=1)
ref_ht = torch.cat(ref_states, dim=0)
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
tri_o, tri_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=g.clone(),
beta=beta.clone(),
initial_state=h0.transpose(-1, -2).contiguous().clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
assert_close("o", ref_o, tri_o, 0.005)
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
@pytest.mark.parametrize(
("cu_seqlens", "dtype"),
[
([0, 64], torch.float16),
([0, 15, 100, 300], torch.bfloat16),
],
)
@torch.inference_mode()
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
cu_seqlens: list[int],
dtype: torch.dtype,
):
H, D = 8, 64
T = cu_seqlens[-1]
N = len(cu_seqlens) - 1
torch.manual_seed(123)
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
beta = torch.rand(1, T, H, dtype=dtype, device=DEVICE).sigmoid()
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
dt_bias = (
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
).contiguous()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
initial_state = h0.transpose(-1, -2).contiguous()
gate = fused_kda_gate(
raw_g.reshape(T, H * D),
A_log,
D,
g_bias=dt_bias,
).unsqueeze(0)
old_o, old_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=gate,
beta=beta.clone(),
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
new_o, new_ht = chunk_kda_with_fused_gate(
q=q.clone(),
k=k.clone(),
v=v.clone(),
raw_g=raw_g,
beta=beta.clone(),
A_log=A_log,
g_bias=dt_bias,
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
+33
View File
@@ -5,6 +5,7 @@ import pytest
import torch
import torch.nn.functional as F
from vllm.models.kimi_k3.common.mtp import fused_mtp_input
from vllm.models.kimi_k3.nvidia.ops import attn_res
from vllm.platforms import current_platform
@@ -191,3 +192,35 @@ def test_attn_res_without_output_norm():
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
@pytest.mark.parametrize("num_tokens", [0, 1, 17])
def test_fused_mtp_input(num_tokens: int):
positions = torch.arange(num_tokens, device="cuda")
inputs_embeds = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=7)
previous_hidden_states = _randn_with_row_padding(
num_tokens, HIDDEN_SIZE, padding=11
)
enorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
hnorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
masked_inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
expected = torch.cat(
(
F.rms_norm(masked_inputs_embeds, (HIDDEN_SIZE,), enorm_weight, EPS),
F.rms_norm(previous_hidden_states, (HIDDEN_SIZE,), hnorm_weight, EPS),
),
dim=-1,
)
actual = fused_mtp_input(
positions,
inputs_embeds,
previous_hidden_states,
enorm_weight,
hnorm_weight,
EPS,
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
assert actual.shape == (num_tokens, 2 * HIDDEN_SIZE)
assert actual.is_contiguous()
+130
View File
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import torch
from vllm.model_executor.models.interfaces import supports_eagle3
from vllm.models.kimi_k3.nvidia import model as kimi_model
from vllm.models.kimi_k3.nvidia.model import (
KimiK3ForConditionalGeneration,
KimiLinearModel,
)
def _make_kimi_linear_model() -> KimiLinearModel:
model = object.__new__(KimiLinearModel)
object.__setattr__(model, "aux_hidden_state_layers", (2,))
object.__setattr__(model, "use_sequence_parallel", False)
return model
def test_kimi_k3_advertises_eagle3_support():
assert supports_eagle3(KimiK3ForConditionalGeneration)
def test_kimi_k3_uses_shared_eagle3_layer_configuration():
target = object.__new__(KimiK3ForConditionalGeneration)
torch.nn.Module.__init__(target)
model = _make_kimi_linear_model()
object.__setattr__(model, "layers", [None] * 93)
language_model = SimpleNamespace(
embed_input_ids=lambda _: None,
model=model,
)
object.__setattr__(target, "language_model", language_model)
object.__setattr__(target, "_language_model_names", ["language_model"])
target.set_aux_hidden_state_layers((2, 46, 90))
assert model.aux_hidden_state_layers == (2, 46, 90)
assert target.get_eagle3_default_aux_hidden_state_layers() == (
2,
46,
90,
)
def test_kimi_linear_forward_extracts_standard_aux_hidden_states(monkeypatch):
model = _make_kimi_linear_model()
initial_hidden_states = torch.tensor([[1.0, 2.0]])
layer_hidden_states = torch.tensor([[3.0, 4.0]])
layer_residual = torch.tensor([[5.0, 6.0]])
object.__setattr__(model, "start_layer", 0)
object.__setattr__(model, "end_layer", 1)
object.__setattr__(
model,
"layers",
[Mock(return_value=(layer_hidden_states, None, layer_residual))],
)
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
object.__setattr__(model, "use_attn_res", False)
monkeypatch.setattr(
kimi_model,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
intermediate_tensors=None,
inputs_embeds=initial_hidden_states,
)
expected_layer_output = layer_hidden_states + layer_residual
torch.testing.assert_close(output, expected_layer_output)
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
torch.testing.assert_close(aux_hidden_states[1], expected_layer_output)
def test_kimi_linear_forward_extracts_attn_res_aux_hidden_states(monkeypatch):
model = _make_kimi_linear_model()
initial_hidden_states = torch.tensor([[1.0, 2.0]])
layer_hidden_states = torch.tensor([[3.0, 4.0]])
prefix_sum = torch.tensor([[5.0, 6.0]])
block_residual = torch.tensor([[[7.0, 8.0]]])
final_hidden_states = torch.tensor([[9.0, 10.0]])
object.__setattr__(model, "start_layer", 0)
object.__setattr__(model, "end_layer", 1)
object.__setattr__(
model,
"layers",
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
)
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
object.__setattr__(model, "use_attn_res", True)
object.__setattr__(model, "num_attn_res_blocks", 1)
object.__setattr__(
model,
"output_attn_res_norm",
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
)
object.__setattr__(
model,
"output_attn_res_proj",
SimpleNamespace(weight=torch.ones(1, 2)),
)
monkeypatch.setattr(
kimi_model,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
final_attn_res = Mock(return_value=final_hidden_states)
monkeypatch.setattr(kimi_model, "attn_res", final_attn_res)
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
intermediate_tensors=None,
inputs_embeds=initial_hidden_states,
)
torch.testing.assert_close(output, final_hidden_states)
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
torch.testing.assert_close(aux_hidden_states[1], prefix_sum + layer_hidden_states)
assert final_attn_res.call_args.args[2] is block_residual
+757
View File
@@ -0,0 +1,757 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Precision tests for vllm's chunk_kda Triton operator.
Compares chunk_kda against a naive recurrent reference (float32).
Uses torch.rand for q/k/v to match FLA's test pattern.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm import _custom_ops as ops
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_update
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
)
from vllm.models.kimi_k3.nvidia.kda import (
is_flashkda_supported,
is_fused_kda_decode_supported,
)
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda,
chunk_kda_with_fused_gate,
fused_kda_gate,
fused_recurrent_kda,
fused_recurrent_kda_fwd,
fused_recurrent_kda_packed_decode,
)
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
DEVICE = "cuda"
@torch.inference_mode()
def test_gather_initial_states_correctness():
row_size = 8 * 128 * 128
storage = torch.randn(5, row_size + 256, dtype=torch.float32, device=DEVICE)
state = storage[:, :row_size].view(5, 8, 128, 128)
assert not state.is_contiguous()
assert state[0].is_contiguous()
indices = torch.tensor([4, 1, 3], dtype=torch.int32, device=DEVICE)
has_initial_state = torch.tensor([True, False, True], device=DEVICE)
expected = state[indices].clone()
expected[~has_initial_state] = 0
torch.testing.assert_close(
gather_initial_states(state, indices, has_initial_state),
expected,
)
def naive_recurrent_kda(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
scale: float | None = None,
initial_state: torch.Tensor | None = None,
output_final_state: bool = False,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
dtype = v.dtype
B, T, H, K = q.shape
V = v.shape[-1]
if scale is None:
scale = K**-0.5
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
q = q * scale
S = k.new_zeros(B, H, K, V).to(q)
if initial_state is not None:
S += initial_state
o = torch.zeros_like(v)
for i in range(T):
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
S = S * g_i[..., None].exp()
S = S + torch.einsum(
"bhk,bhv->bhkv",
b_i[..., None] * k_i,
v_i - (k_i[..., None] * S).sum(-2),
)
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
if not output_final_state:
S = None
return o.to(dtype), S
def assert_close(
name: str,
ref: torch.Tensor,
tri: torch.Tensor,
ratio: float,
err_atol: float = 1e-6,
):
"""RMSE-based relative error comparison."""
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
rel_err = rmse_diff / (rmse_base + 1e-8)
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
if abs_err <= err_atol:
return
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
assert rel_err < ratio, (
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
)
@pytest.mark.parametrize(
("H", "D", "cu_seqlens", "dtype"),
[
pytest.param(
*test,
id="H{}-D{}-cu{}-{}".format(*test),
)
for test in [
(32, 128, [0, 64], torch.float16),
(32, 128, [0, 1024], torch.float16),
(32, 128, [0, 15], torch.float16),
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
(64, 128, [0, 256, 500, 1000], torch.float16),
(32, 128, [0, 8192], torch.float16),
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
]
],
)
@torch.inference_mode()
def test_chunk_kda(
H: int,
D: int,
cu_seqlens: list[int],
dtype: torch.dtype,
):
T = cu_seqlens[-1]
torch.manual_seed(42)
B = 1
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
N = len(cu_seqlens) - 1
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
dtype
)
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
ref_outputs = []
ref_states = []
for i in range(N):
s, e = cu_seqlens[i], cu_seqlens[i + 1]
q_i = l2norm_fwd(q[:, s:e].contiguous())
k_i = l2norm_fwd(k[:, s:e].contiguous())
o_i, ht_i = naive_recurrent_kda(
q_i,
k_i,
v[:, s:e],
g[:, s:e],
beta[:, s:e],
initial_state=h0[i],
output_final_state=True,
)
ref_outputs.append(o_i)
ref_states.append(ht_i)
ref_o = torch.cat(ref_outputs, dim=1)
ref_ht = torch.cat(ref_states, dim=0)
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
tri_o, tri_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=g.clone(),
beta=beta.clone(),
initial_state=h0.transpose(-1, -2).contiguous().clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
assert_close("o", ref_o, tri_o, 0.005)
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
@pytest.mark.parametrize(
("cu_seqlens", "dtype", "lower_bound"),
[
([0, 64], torch.float16, None),
([0, 15, 100, 300], torch.bfloat16, None),
([0, 15, 100, 300], torch.bfloat16, -3.0),
],
)
@torch.inference_mode()
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
cu_seqlens: list[int],
dtype: torch.dtype,
lower_bound: float | None,
):
H, D = 8, 64
T = cu_seqlens[-1]
N = len(cu_seqlens) - 1
torch.manual_seed(123)
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
beta_storage = torch.randn(1, T, 2 * H + 3, dtype=dtype, device=DEVICE)
raw_beta = beta_storage[..., 1 : 2 * H + 1 : 2]
beta = raw_beta.float().sigmoid()
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
dt_bias = (
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
).contiguous()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
initial_state = h0.transpose(-1, -2).contiguous()
gate = fused_kda_gate(
raw_g.reshape(T, H * D),
A_log,
D,
g_bias=dt_bias,
lower_bound=lower_bound,
)
if lower_bound is not None:
expected_gate = lower_bound * torch.sigmoid(
A_log.exp()[None, :, None]
* (raw_g.float().view(T, H, D) + dt_bias.view(H, D))
)
torch.testing.assert_close(gate, expected_gate)
gate = gate.unsqueeze(0)
old_o, old_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=gate,
beta=beta,
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
new_o, new_ht = chunk_kda_with_fused_gate(
q=q.clone(),
k=k.clone(),
v=v.clone(),
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
g_bias=dt_bias,
lower_bound=lower_bound,
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
@pytest.mark.parametrize("num_seqs", [1, 8, 32])
@pytest.mark.parametrize("lower_bound", [-5.0, None])
@pytest.mark.parametrize("state_indices_stride", [1, 8])
@torch.inference_mode()
def test_packed_kda_decode_correctness(
num_seqs: int,
lower_bound: float | None,
state_indices_stride: int,
):
H, D = 8, 128
torch.manual_seed(321)
packed_storage = torch.randn(
num_seqs,
3 * H * D + 1,
dtype=torch.bfloat16,
device=DEVICE,
)
mixed_qkv = packed_storage[:, : 3 * H * D]
assert mixed_qkv.stride(0) == 3 * H * D + 1
q, k, v = (
x.contiguous().view(1, num_seqs, H, D) for x in mixed_qkv.split(H * D, dim=-1)
)
raw_g = torch.randn(
1,
num_seqs,
H,
D,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta = torch.randn(
1,
num_seqs,
H,
dtype=torch.bfloat16,
device=DEVICE,
)
beta = raw_beta.float().sigmoid()
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
state_storage = torch.randn(
num_seqs + 1,
H * D * D + 17,
dtype=torch.float32,
device=DEVICE,
)
state = state_storage[:, : H * D * D].view(num_seqs + 1, H, D, D)
assert not state.is_contiguous()
assert state.stride()[1:] == (D * D, D, 1)
state_indices_storage = torch.zeros(
num_seqs,
state_indices_stride,
dtype=torch.int32,
device=DEVICE,
)
state_indices = state_indices_storage[:, 0]
state_indices.copy_(
torch.arange(
1,
num_seqs + 1,
dtype=torch.int32,
device=DEVICE,
)
)
gate = fused_kda_gate(
raw_g.reshape(num_seqs, H * D),
A_log,
D,
g_bias=dt_bias,
lower_bound=lower_bound,
).unsqueeze(0)
dense_state = state.clone()
dense_out, _ = fused_recurrent_kda_fwd(
q=q,
k=k,
v=v,
g=gate,
beta=beta,
scale=D**-0.5,
initial_state=dense_state,
inplace_final_state=True,
cu_seqlens=torch.arange(
num_seqs + 1,
dtype=torch.int32,
device=DEVICE,
),
ssm_state_indices=state_indices,
use_qk_l2norm_in_kernel=True,
)
packed_state = state
packed_out, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=mixed_qkv,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
initial_state=packed_state,
state_indices=state_indices,
)
assert_close("o", dense_out, packed_out, 1e-3, err_atol=1e-3)
assert_close("ht", dense_state, packed_state, 1e-3, err_atol=1e-3)
@pytest.mark.parametrize(
("H", "fuse_gate"),
[(12, True), (12, False), (12, None), (96, None)],
)
@pytest.mark.parametrize("lower_bound", [-5.0, None])
@torch.inference_mode()
def test_kda_spec_decode_correctness(
H: int,
fuse_gate: bool | None,
lower_bound: float | None,
):
num_seqs, query_len, D = 3, 3, 128
T = num_seqs * query_len
torch.manual_seed(1234)
qkv_storage = torch.randn(
1,
T,
3 * H * D + 7,
dtype=torch.bfloat16,
device=DEVICE,
)
packed_qkv = qkv_storage[..., : 3 * H * D]
q, k, v = (x.view(1, T, H, D) for x in packed_qkv.split(H * D, dim=-1))
gate_storage = torch.randn(
1,
T,
H * D + 5,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_g = gate_storage[..., : H * D].view(1, T, H, D)
beta_storage = torch.randn(
1,
T,
H + 1,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta = beta_storage[..., :H]
A_log = 0.5 * torch.randn(H, dtype=torch.float32, device=DEVICE)
dt_bias = 0.1 * torch.randn(H, D, dtype=torch.float32, device=DEVICE)
cu_seqlens = torch.arange(
0,
T + 1,
query_len,
dtype=torch.int32,
device=DEVICE,
)
state_indices = torch.arange(
1,
T + 1,
dtype=torch.int32,
device=DEVICE,
).view(num_seqs, query_len)
num_accepted_tokens = torch.tensor(
[1, 2, 3],
dtype=torch.int32,
device=DEVICE,
)
state_storage = 0.01 * torch.randn(
T + 1,
H * D * D + 17,
dtype=torch.float32,
device=DEVICE,
)
state = state_storage[:, : H * D * D].view(T + 1, H, D, D)
output_storage = torch.full(
(1, T, H * D + 11),
torch.nan,
dtype=torch.bfloat16,
device=DEVICE,
)
output = output_storage[..., : H * D].view(1, T, H, D)
gate = fused_kda_gate(
raw_g.contiguous().view(T, H * D),
A_log,
D,
g_bias=dt_bias,
lower_bound=lower_bound,
).unsqueeze(0)
beta = raw_beta.float().sigmoid()
q_norm = l2norm_fwd(q.contiguous())
k_norm = l2norm_fwd(k.contiguous())
expected_state = state.clone()
expected_outputs = []
for seq, accepted in enumerate(num_accepted_tokens.tolist()):
recurrent_state = expected_state[state_indices[seq, accepted - 1]].transpose(
-1, -2
)
start = seq * query_len
for token in range(query_len):
token_slice = slice(start + token, start + token + 1)
token_output, recurrent_state = naive_recurrent_kda(
q_norm[:, token_slice],
k_norm[:, token_slice],
v[:, token_slice],
gate[:, token_slice],
beta[:, token_slice],
initial_state=recurrent_state,
output_final_state=True,
)
assert recurrent_state is not None
expected_outputs.append(token_output)
expected_state[state_indices[seq, token]] = recurrent_state.transpose(
-1, -2
)
expected = torch.cat(expected_outputs, dim=1)
actual_state = state.clone()
actual, _ = fused_recurrent_kda(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
initial_state=actual_state,
cu_seqlens=cu_seqlens,
ssm_state_indices=state_indices,
num_accepted_tokens=num_accepted_tokens,
out=output,
fuse_gate=fuse_gate,
)
assert actual.data_ptr() == output.data_ptr()
assert_close("o", expected, actual, 1e-3, err_atol=1e-3)
used_states = state_indices.flatten().long()
assert_close(
"ht",
expected_state[used_states],
actual_state[used_states],
3e-3,
err_atol=3e-3,
)
assert torch.isnan(output_storage[..., H * D :]).all()
@pytest.mark.parametrize(
("num_heads", "num_seqs", "lower_bound", "fuse_output_norm"),
[
(12, 1, -5.0, True),
(12, 4, None, False),
(24, 4, None, False),
(48, 1, -5.0, True),
(96, 1, -5.0, True),
],
)
@torch.inference_mode()
def test_fused_kda_decode_correctness(
num_heads: int,
num_seqs: int,
lower_bound: float | None,
fuse_output_norm: bool,
):
D, W = 128, 4
if not is_fused_kda_decode_supported(
num_heads,
D,
W,
num_spec=0,
input_dtype=torch.bfloat16,
conv_state_dtype=torch.bfloat16,
):
pytest.skip("Fused KDA decode is not supported on this platform")
torch.manual_seed(967 + num_heads + num_seqs)
dim = num_heads * D
slots = num_seqs + 2
packed_x_storage = torch.randn(
num_seqs, 3 * dim + 17, dtype=torch.bfloat16, device=DEVICE
)
packed_x = packed_x_storage[:, : 3 * dim]
weight = 0.1 * torch.randn(3 * dim, W, dtype=torch.float32, device=DEVICE)
conv_seed = 0.1 * torch.randn(
slots,
W - 1,
3 * dim,
dtype=torch.bfloat16,
device=DEVICE,
).transpose(1, 2)
raw_g = torch.randn(
1,
num_seqs,
num_heads,
D,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta_storage = torch.randn(
1,
num_seqs,
num_heads + 1,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta = raw_beta_storage[:, :, :num_heads]
output_gate_storage = torch.randn(
num_seqs,
dim + 7,
dtype=torch.bfloat16,
device=DEVICE,
)
output_gate = output_gate_storage[:, :dim].view(num_seqs, num_heads, D)
norm_weight = torch.randn(D, dtype=torch.float32, device=DEVICE)
norm_eps = 1e-5
A_log = 0.5 * torch.randn(num_heads, dtype=torch.float32, device=DEVICE)
dt_bias = 0.1 * torch.randn(dim, dtype=torch.float32, device=DEVICE)
state_indices = torch.arange(
num_seqs,
0,
-1,
dtype=torch.int32,
device=DEVICE,
)
state_seed = 0.01 * torch.randn(
slots,
num_heads,
D,
D,
dtype=torch.float32,
device=DEVICE,
)
conv_ref = conv_seed.clone()
state_ref = state_seed.clone()
mixed_qkv = causal_conv1d_update(
packed_x,
conv_ref,
weight,
activation="silu",
conv_state_indices=state_indices,
validate_data=True,
out=torch.empty_like(packed_x),
)
expected, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=mixed_qkv,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
initial_state=state_ref,
state_indices=state_indices,
)
if fuse_output_norm:
expected_float = expected.float()
expected = (
expected_float
* torch.rsqrt(expected_float.square().mean(dim=-1, keepdim=True) + norm_eps)
* norm_weight
* output_gate.float().sigmoid().unsqueeze(0)
).to(expected.dtype)
conv_slot_elements = 3 * dim * (W - 1)
state_slot_elements = num_heads * D * D
conv_slot_bytes = conv_slot_elements * torch.bfloat16.itemsize
page_bytes = conv_slot_bytes + state_slot_elements * torch.float32.itemsize
cache_storage = torch.empty(slots * page_bytes, dtype=torch.uint8, device=DEVICE)
conv_actual = torch.as_strided(
cache_storage.view(torch.bfloat16),
size=(slots, 3 * dim, W - 1),
stride=(page_bytes // torch.bfloat16.itemsize, 1, 3 * dim),
)
state_actual = torch.as_strided(
cache_storage.view(torch.float32),
size=(slots, num_heads, D, D),
stride=(page_bytes // torch.float32.itemsize, D * D, D, 1),
storage_offset=conv_slot_bytes // torch.float32.itemsize,
)
conv_actual.copy_(conv_seed)
state_actual.copy_(state_seed)
fused_weight = weight.reshape(3, dim, W).transpose(1, 2).contiguous()
actual = ops.fused_kda_decode(
x=packed_x,
weight=fused_weight,
bias=None,
conv_state=conv_actual,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
state_indices=state_indices,
state=state_actual,
lower_bound=lower_bound,
output_gate=output_gate if fuse_output_norm else None,
norm_weight=norm_weight if fuse_output_norm else None,
norm_eps=norm_eps,
)
torch.testing.assert_close(actual, expected, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(conv_actual, conv_ref, atol=0, rtol=0)
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
def test_fused_kda_decode_rejects_speculative_conv_state():
assert not is_fused_kda_decode_supported(
num_heads=12,
head_dim=128,
conv_width=4,
num_spec=2,
input_dtype=torch.bfloat16,
conv_state_dtype=torch.bfloat16,
)
@torch.inference_mode()
def test_flashkda_correctness():
if not is_flashkda_supported(128, torch.bfloat16, -3.0):
pytest.skip("FlashKDA is not supported on this platform")
import vllm._flashkda_C # noqa: F401
B, T, H, D = 1, 48, 2, 128
torch.manual_seed(11)
q, k, v, raw_g = [
torch.randn(B, T, H, D, dtype=torch.bfloat16, device=DEVICE) for _ in range(4)
]
beta_logits = torch.randn(B, T, H, dtype=torch.bfloat16, device=DEVICE)
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
initial_state = torch.randn(2, H, D, D, dtype=torch.float32, device=DEVICE)
cu_seqlens = torch.tensor([0, 17, T], dtype=torch.int32, device=DEVICE)
lower_bound = -3.0
gate = lower_bound * torch.sigmoid(
A_log.exp()[None, None, :, None] * (raw_g.float() + dt_bias[None, None, :, :])
)
beta = beta_logits.float().sigmoid()
q_norm = l2norm_fwd(q.contiguous())
k_norm = l2norm_fwd(k.contiguous())
expected_outputs = []
expected_states = []
for i, (start, end) in enumerate(
zip(cu_seqlens[:-1].tolist(), cu_seqlens[1:].tolist())
):
output, final_state = naive_recurrent_kda(
q_norm[:, start:end],
k_norm[:, start:end],
v[:, start:end],
gate[:, start:end],
beta[:, start:end],
initial_state=initial_state[i].transpose(-1, -2),
output_final_state=True,
)
expected_outputs.append(output)
expected_states.append(final_state)
expected_out = torch.cat(expected_outputs, dim=1)
expected_state = torch.cat(expected_states).transpose(-1, -2).contiguous()
actual_out = torch.empty_like(v)
actual_state = torch.empty_like(initial_state)
workspace = torch.empty(
torch.ops._flashkda_C.get_workspace_size(T, H, cu_seqlens.numel() - 1),
dtype=torch.uint8,
device=DEVICE,
)
torch.ops._flashkda_C.fwd(
q,
k,
v,
raw_g,
beta_logits,
D**-0.5,
actual_out,
workspace,
A_log,
dt_bias,
lower_bound,
initial_state,
actual_state,
cu_seqlens,
)
assert_close("o", expected_out, actual_out, 0.01)
assert_close("ht", expected_state, actual_state, 0.01)
+411
View File
@@ -0,0 +1,411 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import fields
import pytest
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.models.kimi_k3.nvidia.kda_metadata import (
KimiK3KDAAttentionBackend,
KimiK3KDAMetadata,
KimiK3KDAMetadataBuilder,
_mamba_get_block_table_tensor,
stage_spec_decode_metadata,
)
from vllm.v1.attention.backend import AttentionMetadataBuilder
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import (
NULL_BLOCK_ID,
mamba_get_block_table_tensor,
)
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
DEVICE = torch.device("cpu")
PRUNED_METADATA_FIELDS = {
"chunk_indices",
"chunk_offsets",
"prefill_query_start_loc",
"prefill_state_indices",
"prefill_has_initial_state",
"spec_sequence_masks",
}
def _assert_matches_shared_gdn(reference, actual: KimiK3KDAMetadata):
for field in fields(KimiK3KDAMetadata):
actual_value = getattr(actual, field.name)
expected_value = getattr(reference, field.name)
if field.name in PRUNED_METADATA_FIELDS:
assert actual_value is None
continue
if (
field.name in {"spec_token_indx", "non_spec_token_indx"}
and actual.num_spec_decodes > 0
and actual.num_prefills == 0
and actual.num_decodes == 0
):
assert actual_value is None
continue
if isinstance(actual_value, torch.Tensor):
torch.testing.assert_close(actual_value, expected_value)
elif field.name == "nums_dict":
assert (actual_value is None) == (expected_value is None)
if actual_value is not None:
assert actual_value[8]["tot"] == expected_value[8]["tot"]
torch.testing.assert_close(
actual_value[8]["nums"], expected_value[8]["nums"]
)
else:
assert actual_value == expected_value
def _make_builder(
builder_cls: type[AttentionMetadataBuilder],
num_speculative_tokens: int,
full_cuda_graph: bool,
device: torch.device = DEVICE,
mamba_cache_mode: str = "none",
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
if num_speculative_tokens:
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=num_speculative_tokens,
)
vllm_config.compilation_config.cudagraph_mode = (
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
return builder_cls(
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=num_speculative_tokens,
),
layer_names=["layer.0"],
vllm_config=vllm_config,
device=device,
)
@pytest.mark.parametrize(
(
"batch",
"num_decode_draft_tokens",
"num_speculative_tokens",
"full_cuda_graph",
"is_prefilling",
),
[
pytest.param(
BatchSpec(seq_lens=[50, 30], query_lens=[3, 3]),
[2, 2],
2,
False,
[False, False],
id="pure-spec-decode",
),
pytest.param(
BatchSpec(seq_lens=[100, 65, 20], query_lens=[50, 1, 3]),
[-1, -1, 2],
2,
False,
[True, False, False],
id="mixed-prefill-and-spec-decode",
),
pytest.param(
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
None,
0,
False,
[False, False],
id="regular-decode",
),
pytest.param(
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
[0, 0],
2,
False,
[False, False],
id="no-scheduled-draft-tokens",
),
],
)
def test_kimi_k3_kda_metadata_matches_shared_gdn(
batch: BatchSpec,
num_decode_draft_tokens: list[int] | None,
num_speculative_tokens: int,
full_cuda_graph: bool,
is_prefilling: list[bool],
):
kwargs: dict[str, torch.Tensor] = {}
if num_decode_draft_tokens is not None:
kwargs = {
"num_decode_draft_tokens_cpu": torch.tensor(
num_decode_draft_tokens, dtype=torch.int32
),
"num_accepted_tokens": torch.ones(
batch.batch_size, dtype=torch.int32, device=DEVICE
),
}
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor(is_prefilling, dtype=torch.bool))
reference = _make_builder(
GDNAttentionMetadataBuilder,
num_speculative_tokens,
full_cuda_graph,
).build(
0,
common_attn_metadata,
**kwargs,
)
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens,
full_cuda_graph,
).build(0, common_attn_metadata, **kwargs)
assert isinstance(actual, KimiK3KDAMetadata)
_assert_matches_shared_gdn(reference, actual)
def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():
batch = BatchSpec(seq_lens=[100, 65, 20], query_lens=[1, 1, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([False, False, False]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
).build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
)
# The K3 layer dispatches the non-spec subgroup to packed decode whenever
# it contains no prefill request.
assert actual.num_decodes == 2
assert actual.num_decode_tokens == 2
assert actual.num_prefills == 0
assert actual.num_prefill_tokens == 0
assert actual.has_initial_state is None
assert actual.nums_dict is None
assert actual.non_spec_query_start_loc is None
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0, 1]))
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([2, 3, 4]))
torch.testing.assert_close(
actual.spec_query_start_loc,
torch.tensor([0, 3], dtype=torch.int32),
)
def test_mixed_regular_and_spec_decode_excludes_request_padding():
batch = BatchSpec(seq_lens=[16, 65, 20], query_lens=[0, 1, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([False, False, False]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
).build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
)
assert actual.num_decodes == 1
assert actual.non_spec_state_indices_tensor is not None
assert actual.non_spec_state_indices_tensor.shape == (1,)
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0]))
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
@pytest.mark.parametrize(
("seq_len", "expected_has_initial_state"),
[
pytest.param(1, False, id="first-token-prefill"),
pytest.param(65, True, id="final-one-token-prefill-chunk"),
],
)
def test_mixed_one_token_prefill_and_spec_decode_uses_prefill_metadata(
seq_len: int,
expected_has_initial_state: bool,
):
batch = BatchSpec(seq_lens=[seq_len, 20], query_lens=[1, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([True, False]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
).build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.tensor([-1, 2], dtype=torch.int32),
num_accepted_tokens=torch.ones(2, dtype=torch.int32, device=DEVICE),
)
assert actual.num_prefills == 1
assert actual.num_prefill_tokens == 1
assert actual.num_decodes == 0
assert actual.num_decode_tokens == 0
assert actual.has_initial_state is not None
assert actual.has_initial_state.tolist() == [expected_has_initial_state]
assert actual.non_spec_query_start_loc is not None
torch.testing.assert_close(
actual.non_spec_query_start_loc,
torch.tensor([0, 1], dtype=torch.int32),
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_kimi_k3_kda_cudagraph_capture_matches_shared_gdn():
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device
).replace(is_prefilling=torch.tensor([False, False]))
reference = _make_builder(
GDNAttentionMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=True,
device=device,
).build_for_cudagraph_capture(common_attn_metadata)
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=True,
device=device,
).build_for_cudagraph_capture(common_attn_metadata)
assert isinstance(actual, KimiK3KDAMetadata)
_assert_matches_shared_gdn(reference, actual)
def test_kimi_k3_kda_backend_uses_private_metadata_builder():
assert KimiK3KDAAttentionBackend.get_builder_cls() is KimiK3KDAMetadataBuilder
assert KimiK3KDAAttentionBackend.is_ssm()
assert issubclass(KimiK3KDAAttentionBackend, GDNAttentionBackend)
assert issubclass(KimiK3KDAMetadata, GDNAttentionMetadata)
assert issubclass(KimiK3KDAMetadataBuilder, GDNAttentionMetadataBuilder)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_stage_spec_decode_metadata_matches_pytorch():
device = torch.device("cuda")
num_spec_decodes = 33
batch_size = 65
num_state_slots = 3
state_indices = torch.arange(
num_spec_decodes * 32,
dtype=torch.int32,
device=device,
).reshape(num_spec_decodes, 32)[:, :num_state_slots]
query_start_loc = (
torch.arange(num_spec_decodes + 1, dtype=torch.int32, device=device)
* num_state_slots
)
num_accepted_tokens = (
torch.arange(num_spec_decodes, dtype=torch.int32, device=device)
% num_state_slots
+ 1
)
staged_state_indices = torch.empty(
(batch_size, num_state_slots), dtype=torch.int32, device=device
)
staged_query_start_loc = torch.empty(
batch_size + 1, dtype=torch.int32, device=device
)
staged_num_accepted_tokens = torch.empty(
batch_size, dtype=torch.int32, device=device
)
stage_spec_decode_metadata(
state_indices,
query_start_loc,
num_accepted_tokens,
staged_state_indices,
staged_query_start_loc,
staged_num_accepted_tokens,
num_spec_decodes=num_spec_decodes,
)
expected_state_indices = torch.full_like(staged_state_indices, NULL_BLOCK_ID)
expected_state_indices[:num_spec_decodes] = state_indices
expected_query_start_loc = torch.full(
(batch_size + 1,),
query_start_loc[-1],
dtype=torch.int32,
device=device,
)
expected_query_start_loc[: num_spec_decodes + 1] = query_start_loc
expected_num_accepted_tokens = torch.ones(
batch_size, dtype=torch.int32, device=device
)
expected_num_accepted_tokens[:num_spec_decodes] = num_accepted_tokens
torch.testing.assert_close(staged_state_indices, expected_state_indices)
torch.testing.assert_close(staged_query_start_loc, expected_query_start_loc)
torch.testing.assert_close(staged_num_accepted_tokens, expected_num_accepted_tokens)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_aligned_block_table_matches_shared_gdn():
device = torch.device("cuda")
seq_lens = torch.tensor(
[0, 1, 15, 16, 17, 31, 32, 33, 511, 512, 513],
dtype=torch.int32,
device=device,
).repeat(6)[:65]
block_table_storage = torch.arange(
seq_lens.numel() * 128,
dtype=torch.int32,
device=device,
).reshape(seq_lens.numel(), 128)
block_table = block_table_storage[:, ::2]
kv_cache_spec = MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=2,
)
expected = mamba_get_block_table_tensor(
block_table,
seq_lens,
kv_cache_spec,
"align",
)
actual = _mamba_get_block_table_tensor(
block_table,
seq_lens,
kv_cache_spec,
"align",
)
torch.testing.assert_close(actual, expected)
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import ray
import torch
import torch.distributed as dist
import torch.nn.functional as F
from tests.utils import (
init_test_distributed_environment,
multi_gpu_test,
multi_process_parallel,
)
from vllm.distributed import get_tp_group
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import KimiK3LatentMoETailOp
from vllm.platforms import current_platform
HIDDEN_SIZE = 7168
LATENT_SIZE = 3584
EPS = 0.1
@ray.remote(num_gpus=1, max_calls=1)
def _test_latent_moe_tail_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
)
torch.manual_seed(0)
rms_weight = 1 + 0.1 * torch.randn(
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
up_weight = (
torch.randn(
HIDDEN_SIZE,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
/ LATENT_SIZE**0.5
)
group = get_tp_group().device_group
op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
)
cutedsl_warmup()
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
torch.manual_seed(100 * iteration + rank + 1)
routed_output = torch.randn(
num_tokens,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
).mul_(0.01)
shared_output = torch.randn(
num_tokens,
HIDDEN_SIZE,
device=device,
dtype=torch.bfloat16,
)
routed_reference = routed_output.clone()
shared_reference = shared_output.clone()
dist.all_reduce(routed_reference, group=group)
dist.all_reduce(shared_reference, group=group)
expected = F.linear(
F.rms_norm(
routed_reference,
(LATENT_SIZE,),
rms_weight,
EPS,
),
up_weight,
)
expected.add_(shared_reference)
actual = op(
routed_output,
shared_output,
rms_weight,
up_weight,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
assert actual.is_contiguous()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_output = op(
routed_output,
shared_output,
rms_weight,
up_weight,
)
graph.replay()
torch.testing.assert_close(graph_output, expected, atol=8e-2, rtol=3e-2)
def _run_latent_moe_tail_test(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
) -> None:
if not current_platform.is_device_capability_family(100):
pytest.skip("K3 latent-MoE tail fusion requires SM100")
multi_process_parallel(
monkeypatch,
tp_size,
1,
_test_latent_moe_tail_worker,
)
@multi_gpu_test(num_gpus=8)
def test_latent_moe_tail_tp8_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_latent_moe_tail_test(monkeypatch, 8)
@multi_gpu_test(num_gpus=16)
def test_latent_moe_tail_tp16_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_latent_moe_tail_test(monkeypatch, 16)
@@ -0,0 +1,343 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import MethodType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from torch import nn
from vllm.config import ParallelConfig
from vllm.models.kimi_k3.nvidia import model as kimi_model
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
from vllm.models.kimi_k3.nvidia.ops import sequence_parallel as sp_ops
from vllm.platforms import current_platform
class _IdentityNorm(nn.Module):
def __init__(self, hidden_size: int = 2) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
self.variance_epsilon = 1e-5
def forward(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None = None,
):
if residual is None:
return hidden_states
return hidden_states, residual
class _RecordingMoE(nn.Module):
def __init__(self) -> None:
super().__init__()
self.num_tokens = 0
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
self.num_tokens = hidden_states.shape[0]
return hidden_states
class _Projection(nn.Module):
def __init__(self, hidden_size: int = 2) -> None:
super().__init__()
self.weight = nn.Parameter(
torch.ones(1, hidden_size),
requires_grad=False,
)
class _SequenceParallelMTPBlock:
use_sequence_parallel = True
def __call__(
self,
*,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
):
assert residual is None
return hidden_states * 2, None, hidden_states * 3
def _mock_sequence_parallel_collectives(monkeypatch):
monkeypatch.setattr(
kimi_model,
"sp_reduce_scatter",
lambda tensor: tensor.chunk(2, dim=0)[0],
)
monkeypatch.setattr(
kimi_model,
"sp_shard",
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
)
monkeypatch.setattr(
kimi_model,
"sp_all_gather",
lambda tensor: torch.cat([tensor, tensor], dim=0),
)
@pytest.mark.parametrize(
("num_tokens", "is_padding", "tp_rank", "expected"),
[
(1, None, 0, [False]),
(1, None, 1, [True]),
(5, None, 2, [False, True]),
(5, None, 3, [True, True]),
(5, [False, True, False, False, False], 0, [False, True]),
],
)
def test_sp_padding_mask_marks_added_rows(
monkeypatch,
num_tokens: int,
is_padding: list[bool] | None,
tp_rank: int,
expected: list[bool],
):
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_world_size", lambda: 4)
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
hidden_states = torch.empty(num_tokens, 2)
padding = torch.tensor(is_padding) if is_padding is not None else None
actual = sp_ops.sp_padding_mask(padding, hidden_states)
torch.testing.assert_close(actual, torch.tensor(expected))
@pytest.mark.parametrize(
("data_parallel_size", "expected"),
[
(1, False),
(2, True),
],
)
def test_moe_sequence_parallel_requires_data_parallel(
monkeypatch,
data_parallel_size: int,
expected: bool,
):
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
parallel_config = ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=data_parallel_size,
enable_expert_parallel=True,
all2all_backend="allgather_reducescatter",
)
assert parallel_config.use_sequence_parallel_moe is expected
def test_kimi_decoder_layer_keeps_moe_states_sequence_sharded(monkeypatch):
layer = object.__new__(kimi_model.KimiDecoderLayer)
nn.Module.__init__(layer)
layer.use_attn_res = False
layer.use_sequence_parallel = True
layer.input_layernorm = _IdentityNorm()
layer.post_attention_layernorm = _IdentityNorm()
layer.mlp = _RecordingMoE()
layer._run_self_attn = MethodType(
lambda self, positions, hidden_states: hidden_states,
layer,
)
_mock_sequence_parallel_collectives(monkeypatch)
positions = torch.arange(3)
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
hidden_states = kimi_model.sp_shard(full_hidden_states)
hidden_states, prefix_sum, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=None,
)
assert prefix_sum is None
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.mlp.num_tokens == 2
hidden_states, prefix_sum, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
)
assert prefix_sum is None
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.mlp.num_tokens == 2
def test_kimi_attn_residual_states_stay_sequence_sharded(monkeypatch):
layer = object.__new__(kimi_model.KimiDecoderLayer)
nn.Module.__init__(layer)
layer.use_attn_res = True
layer.use_sequence_parallel = True
layer.prev_valid_blocks = 0
layer.block_write_idx = 0
layer.is_block_write_layer = False
layer.input_layernorm = _IdentityNorm()
layer.post_attention_layernorm = _IdentityNorm()
layer.self_attention_res_norm = _IdentityNorm()
layer.mlp_res_norm = _IdentityNorm()
layer.self_attention_res_proj = _Projection()
layer.mlp_res_proj = _Projection()
layer.mlp = _RecordingMoE()
layer._run_self_attn = MethodType(
lambda self, positions, hidden_states: hidden_states,
layer,
)
_mock_sequence_parallel_collectives(monkeypatch)
monkeypatch.setattr(
kimi_model,
"attn_res",
lambda prefix_sum, hidden_states, *args, **kwargs: (
prefix_sum if hidden_states is None else prefix_sum + hidden_states
),
)
prefix_sum = kimi_model.sp_shard(torch.arange(6, dtype=torch.float32).view(3, 2))
block_residual = torch.zeros(2, 1, 2)
hidden_states, prefix_sum, block_residual = layer(
positions=torch.arange(3),
hidden_states=None,
prefix_sum=prefix_sum,
residual=block_residual,
)
assert hidden_states.shape == prefix_sum.shape == (2, 2)
assert block_residual.shape == (2, 1, 2)
assert layer.mlp.num_tokens == 2
def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
layer = object.__new__(kimi_mtp.KimiK3MultiTokenPredictorLayer)
nn.Module.__init__(layer)
layer.enorm = _IdentityNorm()
layer.hnorm = _IdentityNorm()
layer.eh_proj = nn.Identity()
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
final_norm = Mock(side_effect=lambda hidden_states: hidden_states + 1)
object.__setattr__(
layer,
"shared_head",
SimpleNamespace(norm=final_norm),
)
monkeypatch.setattr(
kimi_mtp,
"fused_mtp_input",
lambda positions, inputs_embeds, *args: inputs_embeds,
)
monkeypatch.setattr(
kimi_mtp,
"sp_shard",
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
)
monkeypatch.setattr(
kimi_mtp,
"sp_all_gather",
lambda tensor: torch.cat([tensor, tensor], dim=0),
)
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
logits_hidden_states, hidden_states = layer(
input_ids=torch.zeros(3, dtype=torch.long),
positions=torch.arange(3),
previous_hidden_states=torch.zeros_like(inputs_embeds),
inputs_embeds=inputs_embeds,
)
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
expected_hidden_states = torch.cat(
[sharded_states * 5, sharded_states * 5],
dim=0,
)[:3]
torch.testing.assert_close(hidden_states, expected_hidden_states)
torch.testing.assert_close(logits_hidden_states, expected_hidden_states + 1)
final_norm.assert_called_once()
torch.testing.assert_close(final_norm.call_args.args[0], expected_hidden_states)
def test_sp_all_gather_uses_custom_kernel(monkeypatch):
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
expected = torch.cat([hidden_states, hidden_states])
custom_all_gather = Mock(return_value=expected)
device_communicator = SimpleNamespace(
custom_all_gather=custom_all_gather,
)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_communicator),
)
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", fallback)
output = sp_ops.sp_all_gather(hidden_states)
torch.testing.assert_close(output, expected)
custom_all_gather.assert_called_once_with(hidden_states)
fallback.assert_not_called()
def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
expected = torch.arange(4, dtype=torch.float32).view(2, 2)
custom_reduce_scatter = Mock(return_value=expected)
device_communicator = SimpleNamespace(
custom_reduce_scatter=custom_reduce_scatter,
)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_communicator),
)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
monkeypatch.setattr(sp_ops, "tensor_model_parallel_reduce_scatter", fallback)
output = sp_ops.sp_reduce_scatter(hidden_states)
torch.testing.assert_close(output, expected)
padded = custom_reduce_scatter.call_args.args[0]
assert padded.shape == (4, 2)
torch.testing.assert_close(padded[:3], hidden_states)
torch.testing.assert_close(padded[3], torch.zeros(2))
fallback.assert_not_called()
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=None),
)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
all_gather = Mock(return_value=hidden_states)
reduce_scatter = Mock(return_value=hidden_states)
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", all_gather)
monkeypatch.setattr(
sp_ops,
"tensor_model_parallel_reduce_scatter",
reduce_scatter,
)
torch.testing.assert_close(sp_ops.sp_all_gather(hidden_states), hidden_states)
torch.testing.assert_close(sp_ops.sp_reduce_scatter(hidden_states), hidden_states)
all_gather.assert_called_once_with(hidden_states, 0)
reduce_scatter.assert_called_once_with(hidden_states, 0)
@@ -1242,7 +1242,7 @@ def test_custom_inputs_models(
create_new_process_for_each_test=True,
),
)
@create_new_process_for_each_test()
@create_new_process_for_each_test("spawn")
def test_single_image_models_heavy(
tmp_path: PosixPath,
model_type: str,