[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: gnovack <novackgm@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-07-29 14:10:58 +08:00
committed by GitHub
co-authored by Isotr0py Thien Tran Bugen Zhao Yongye Zhu Ziming Huang Roger Wang Isotr0py aoshen02 Woosuk Kwon Jee Jee Li aoshen02 Summer Yang Kevin H. Luu Bowen Wang gnovack Nick Hill Yifan Qiao xiaozhoupy Roy Wang Jeff Ma Codex
parent 6f00a1ae3b
commit 7c6729b769
109 changed files with 28028 additions and 792 deletions
+33
View File
@@ -5,6 +5,7 @@ import pytest
import torch
import torch.nn.functional as F
from vllm.models.kimi_k3.common.mtp import fused_mtp_input
from vllm.models.kimi_k3.nvidia.ops import attn_res
from vllm.platforms import current_platform
@@ -191,3 +192,35 @@ def test_attn_res_without_output_norm():
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
@pytest.mark.parametrize("num_tokens", [0, 1, 17])
def test_fused_mtp_input(num_tokens: int):
positions = torch.arange(num_tokens, device="cuda")
inputs_embeds = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=7)
previous_hidden_states = _randn_with_row_padding(
num_tokens, HIDDEN_SIZE, padding=11
)
enorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
hnorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
masked_inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
expected = torch.cat(
(
F.rms_norm(masked_inputs_embeds, (HIDDEN_SIZE,), enorm_weight, EPS),
F.rms_norm(previous_hidden_states, (HIDDEN_SIZE,), hnorm_weight, EPS),
),
dim=-1,
)
actual = fused_mtp_input(
positions,
inputs_embeds,
previous_hidden_states,
enorm_weight,
hnorm_weight,
EPS,
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
assert actual.shape == (num_tokens, 2 * HIDDEN_SIZE)
assert actual.is_contiguous()
+130
View File
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import torch
from vllm.model_executor.models.interfaces import supports_eagle3
from vllm.models.kimi_k3.nvidia import model as kimi_model
from vllm.models.kimi_k3.nvidia.model import (
KimiK3ForConditionalGeneration,
KimiLinearModel,
)
def _make_kimi_linear_model() -> KimiLinearModel:
model = object.__new__(KimiLinearModel)
object.__setattr__(model, "aux_hidden_state_layers", (2,))
object.__setattr__(model, "use_sequence_parallel", False)
return model
def test_kimi_k3_advertises_eagle3_support():
assert supports_eagle3(KimiK3ForConditionalGeneration)
def test_kimi_k3_uses_shared_eagle3_layer_configuration():
target = object.__new__(KimiK3ForConditionalGeneration)
torch.nn.Module.__init__(target)
model = _make_kimi_linear_model()
object.__setattr__(model, "layers", [None] * 93)
language_model = SimpleNamespace(
embed_input_ids=lambda _: None,
model=model,
)
object.__setattr__(target, "language_model", language_model)
object.__setattr__(target, "_language_model_names", ["language_model"])
target.set_aux_hidden_state_layers((2, 46, 90))
assert model.aux_hidden_state_layers == (2, 46, 90)
assert target.get_eagle3_default_aux_hidden_state_layers() == (
2,
46,
90,
)
def test_kimi_linear_forward_extracts_standard_aux_hidden_states(monkeypatch):
model = _make_kimi_linear_model()
initial_hidden_states = torch.tensor([[1.0, 2.0]])
layer_hidden_states = torch.tensor([[3.0, 4.0]])
layer_residual = torch.tensor([[5.0, 6.0]])
object.__setattr__(model, "start_layer", 0)
object.__setattr__(model, "end_layer", 1)
object.__setattr__(
model,
"layers",
[Mock(return_value=(layer_hidden_states, None, layer_residual))],
)
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
object.__setattr__(model, "use_attn_res", False)
monkeypatch.setattr(
kimi_model,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
intermediate_tensors=None,
inputs_embeds=initial_hidden_states,
)
expected_layer_output = layer_hidden_states + layer_residual
torch.testing.assert_close(output, expected_layer_output)
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
torch.testing.assert_close(aux_hidden_states[1], expected_layer_output)
def test_kimi_linear_forward_extracts_attn_res_aux_hidden_states(monkeypatch):
model = _make_kimi_linear_model()
initial_hidden_states = torch.tensor([[1.0, 2.0]])
layer_hidden_states = torch.tensor([[3.0, 4.0]])
prefix_sum = torch.tensor([[5.0, 6.0]])
block_residual = torch.tensor([[[7.0, 8.0]]])
final_hidden_states = torch.tensor([[9.0, 10.0]])
object.__setattr__(model, "start_layer", 0)
object.__setattr__(model, "end_layer", 1)
object.__setattr__(
model,
"layers",
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
)
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
object.__setattr__(model, "use_attn_res", True)
object.__setattr__(model, "num_attn_res_blocks", 1)
object.__setattr__(
model,
"output_attn_res_norm",
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
)
object.__setattr__(
model,
"output_attn_res_proj",
SimpleNamespace(weight=torch.ones(1, 2)),
)
monkeypatch.setattr(
kimi_model,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
final_attn_res = Mock(return_value=final_hidden_states)
monkeypatch.setattr(kimi_model, "attn_res", final_attn_res)
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
intermediate_tensors=None,
inputs_embeds=initial_hidden_states,
)
torch.testing.assert_close(output, final_hidden_states)
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
torch.testing.assert_close(aux_hidden_states[1], prefix_sum + layer_hidden_states)
assert final_attn_res.call_args.args[2] is block_residual
+757
View File
@@ -0,0 +1,757 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Precision tests for vllm's chunk_kda Triton operator.
Compares chunk_kda against a naive recurrent reference (float32).
Uses torch.rand for q/k/v to match FLA's test pattern.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm import _custom_ops as ops
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_update
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
)
from vllm.models.kimi_k3.nvidia.kda import (
is_flashkda_supported,
is_fused_kda_decode_supported,
)
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
chunk_kda,
chunk_kda_with_fused_gate,
fused_kda_gate,
fused_recurrent_kda,
fused_recurrent_kda_fwd,
fused_recurrent_kda_packed_decode,
)
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
DEVICE = "cuda"
@torch.inference_mode()
def test_gather_initial_states_correctness():
row_size = 8 * 128 * 128
storage = torch.randn(5, row_size + 256, dtype=torch.float32, device=DEVICE)
state = storage[:, :row_size].view(5, 8, 128, 128)
assert not state.is_contiguous()
assert state[0].is_contiguous()
indices = torch.tensor([4, 1, 3], dtype=torch.int32, device=DEVICE)
has_initial_state = torch.tensor([True, False, True], device=DEVICE)
expected = state[indices].clone()
expected[~has_initial_state] = 0
torch.testing.assert_close(
gather_initial_states(state, indices, has_initial_state),
expected,
)
def naive_recurrent_kda(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
scale: float | None = None,
initial_state: torch.Tensor | None = None,
output_final_state: bool = False,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
dtype = v.dtype
B, T, H, K = q.shape
V = v.shape[-1]
if scale is None:
scale = K**-0.5
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
q = q * scale
S = k.new_zeros(B, H, K, V).to(q)
if initial_state is not None:
S += initial_state
o = torch.zeros_like(v)
for i in range(T):
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
S = S * g_i[..., None].exp()
S = S + torch.einsum(
"bhk,bhv->bhkv",
b_i[..., None] * k_i,
v_i - (k_i[..., None] * S).sum(-2),
)
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
if not output_final_state:
S = None
return o.to(dtype), S
def assert_close(
name: str,
ref: torch.Tensor,
tri: torch.Tensor,
ratio: float,
err_atol: float = 1e-6,
):
"""RMSE-based relative error comparison."""
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
rel_err = rmse_diff / (rmse_base + 1e-8)
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
if abs_err <= err_atol:
return
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
assert rel_err < ratio, (
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
)
@pytest.mark.parametrize(
("H", "D", "cu_seqlens", "dtype"),
[
pytest.param(
*test,
id="H{}-D{}-cu{}-{}".format(*test),
)
for test in [
(32, 128, [0, 64], torch.float16),
(32, 128, [0, 1024], torch.float16),
(32, 128, [0, 15], torch.float16),
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
(64, 128, [0, 256, 500, 1000], torch.float16),
(32, 128, [0, 8192], torch.float16),
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
]
],
)
@torch.inference_mode()
def test_chunk_kda(
H: int,
D: int,
cu_seqlens: list[int],
dtype: torch.dtype,
):
T = cu_seqlens[-1]
torch.manual_seed(42)
B = 1
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
N = len(cu_seqlens) - 1
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
dtype
)
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
ref_outputs = []
ref_states = []
for i in range(N):
s, e = cu_seqlens[i], cu_seqlens[i + 1]
q_i = l2norm_fwd(q[:, s:e].contiguous())
k_i = l2norm_fwd(k[:, s:e].contiguous())
o_i, ht_i = naive_recurrent_kda(
q_i,
k_i,
v[:, s:e],
g[:, s:e],
beta[:, s:e],
initial_state=h0[i],
output_final_state=True,
)
ref_outputs.append(o_i)
ref_states.append(ht_i)
ref_o = torch.cat(ref_outputs, dim=1)
ref_ht = torch.cat(ref_states, dim=0)
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
tri_o, tri_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=g.clone(),
beta=beta.clone(),
initial_state=h0.transpose(-1, -2).contiguous().clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
assert_close("o", ref_o, tri_o, 0.005)
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
@pytest.mark.parametrize(
("cu_seqlens", "dtype", "lower_bound"),
[
([0, 64], torch.float16, None),
([0, 15, 100, 300], torch.bfloat16, None),
([0, 15, 100, 300], torch.bfloat16, -3.0),
],
)
@torch.inference_mode()
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
cu_seqlens: list[int],
dtype: torch.dtype,
lower_bound: float | None,
):
H, D = 8, 64
T = cu_seqlens[-1]
N = len(cu_seqlens) - 1
torch.manual_seed(123)
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
beta_storage = torch.randn(1, T, 2 * H + 3, dtype=dtype, device=DEVICE)
raw_beta = beta_storage[..., 1 : 2 * H + 1 : 2]
beta = raw_beta.float().sigmoid()
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
dt_bias = (
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
).contiguous()
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
initial_state = h0.transpose(-1, -2).contiguous()
gate = fused_kda_gate(
raw_g.reshape(T, H * D),
A_log,
D,
g_bias=dt_bias,
lower_bound=lower_bound,
)
if lower_bound is not None:
expected_gate = lower_bound * torch.sigmoid(
A_log.exp()[None, :, None]
* (raw_g.float().view(T, H, D) + dt_bias.view(H, D))
)
torch.testing.assert_close(gate, expected_gate)
gate = gate.unsqueeze(0)
old_o, old_ht = chunk_kda(
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=gate,
beta=beta,
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
new_o, new_ht = chunk_kda_with_fused_gate(
q=q.clone(),
k=k.clone(),
v=v.clone(),
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
g_bias=dt_bias,
lower_bound=lower_bound,
initial_state=initial_state.clone(),
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
@pytest.mark.parametrize("num_seqs", [1, 8, 32])
@pytest.mark.parametrize("lower_bound", [-5.0, None])
@pytest.mark.parametrize("state_indices_stride", [1, 8])
@torch.inference_mode()
def test_packed_kda_decode_correctness(
num_seqs: int,
lower_bound: float | None,
state_indices_stride: int,
):
H, D = 8, 128
torch.manual_seed(321)
packed_storage = torch.randn(
num_seqs,
3 * H * D + 1,
dtype=torch.bfloat16,
device=DEVICE,
)
mixed_qkv = packed_storage[:, : 3 * H * D]
assert mixed_qkv.stride(0) == 3 * H * D + 1
q, k, v = (
x.contiguous().view(1, num_seqs, H, D) for x in mixed_qkv.split(H * D, dim=-1)
)
raw_g = torch.randn(
1,
num_seqs,
H,
D,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta = torch.randn(
1,
num_seqs,
H,
dtype=torch.bfloat16,
device=DEVICE,
)
beta = raw_beta.float().sigmoid()
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
state_storage = torch.randn(
num_seqs + 1,
H * D * D + 17,
dtype=torch.float32,
device=DEVICE,
)
state = state_storage[:, : H * D * D].view(num_seqs + 1, H, D, D)
assert not state.is_contiguous()
assert state.stride()[1:] == (D * D, D, 1)
state_indices_storage = torch.zeros(
num_seqs,
state_indices_stride,
dtype=torch.int32,
device=DEVICE,
)
state_indices = state_indices_storage[:, 0]
state_indices.copy_(
torch.arange(
1,
num_seqs + 1,
dtype=torch.int32,
device=DEVICE,
)
)
gate = fused_kda_gate(
raw_g.reshape(num_seqs, H * D),
A_log,
D,
g_bias=dt_bias,
lower_bound=lower_bound,
).unsqueeze(0)
dense_state = state.clone()
dense_out, _ = fused_recurrent_kda_fwd(
q=q,
k=k,
v=v,
g=gate,
beta=beta,
scale=D**-0.5,
initial_state=dense_state,
inplace_final_state=True,
cu_seqlens=torch.arange(
num_seqs + 1,
dtype=torch.int32,
device=DEVICE,
),
ssm_state_indices=state_indices,
use_qk_l2norm_in_kernel=True,
)
packed_state = state
packed_out, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=mixed_qkv,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
initial_state=packed_state,
state_indices=state_indices,
)
assert_close("o", dense_out, packed_out, 1e-3, err_atol=1e-3)
assert_close("ht", dense_state, packed_state, 1e-3, err_atol=1e-3)
@pytest.mark.parametrize(
("H", "fuse_gate"),
[(12, True), (12, False), (12, None), (96, None)],
)
@pytest.mark.parametrize("lower_bound", [-5.0, None])
@torch.inference_mode()
def test_kda_spec_decode_correctness(
H: int,
fuse_gate: bool | None,
lower_bound: float | None,
):
num_seqs, query_len, D = 3, 3, 128
T = num_seqs * query_len
torch.manual_seed(1234)
qkv_storage = torch.randn(
1,
T,
3 * H * D + 7,
dtype=torch.bfloat16,
device=DEVICE,
)
packed_qkv = qkv_storage[..., : 3 * H * D]
q, k, v = (x.view(1, T, H, D) for x in packed_qkv.split(H * D, dim=-1))
gate_storage = torch.randn(
1,
T,
H * D + 5,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_g = gate_storage[..., : H * D].view(1, T, H, D)
beta_storage = torch.randn(
1,
T,
H + 1,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta = beta_storage[..., :H]
A_log = 0.5 * torch.randn(H, dtype=torch.float32, device=DEVICE)
dt_bias = 0.1 * torch.randn(H, D, dtype=torch.float32, device=DEVICE)
cu_seqlens = torch.arange(
0,
T + 1,
query_len,
dtype=torch.int32,
device=DEVICE,
)
state_indices = torch.arange(
1,
T + 1,
dtype=torch.int32,
device=DEVICE,
).view(num_seqs, query_len)
num_accepted_tokens = torch.tensor(
[1, 2, 3],
dtype=torch.int32,
device=DEVICE,
)
state_storage = 0.01 * torch.randn(
T + 1,
H * D * D + 17,
dtype=torch.float32,
device=DEVICE,
)
state = state_storage[:, : H * D * D].view(T + 1, H, D, D)
output_storage = torch.full(
(1, T, H * D + 11),
torch.nan,
dtype=torch.bfloat16,
device=DEVICE,
)
output = output_storage[..., : H * D].view(1, T, H, D)
gate = fused_kda_gate(
raw_g.contiguous().view(T, H * D),
A_log,
D,
g_bias=dt_bias,
lower_bound=lower_bound,
).unsqueeze(0)
beta = raw_beta.float().sigmoid()
q_norm = l2norm_fwd(q.contiguous())
k_norm = l2norm_fwd(k.contiguous())
expected_state = state.clone()
expected_outputs = []
for seq, accepted in enumerate(num_accepted_tokens.tolist()):
recurrent_state = expected_state[state_indices[seq, accepted - 1]].transpose(
-1, -2
)
start = seq * query_len
for token in range(query_len):
token_slice = slice(start + token, start + token + 1)
token_output, recurrent_state = naive_recurrent_kda(
q_norm[:, token_slice],
k_norm[:, token_slice],
v[:, token_slice],
gate[:, token_slice],
beta[:, token_slice],
initial_state=recurrent_state,
output_final_state=True,
)
assert recurrent_state is not None
expected_outputs.append(token_output)
expected_state[state_indices[seq, token]] = recurrent_state.transpose(
-1, -2
)
expected = torch.cat(expected_outputs, dim=1)
actual_state = state.clone()
actual, _ = fused_recurrent_kda(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
initial_state=actual_state,
cu_seqlens=cu_seqlens,
ssm_state_indices=state_indices,
num_accepted_tokens=num_accepted_tokens,
out=output,
fuse_gate=fuse_gate,
)
assert actual.data_ptr() == output.data_ptr()
assert_close("o", expected, actual, 1e-3, err_atol=1e-3)
used_states = state_indices.flatten().long()
assert_close(
"ht",
expected_state[used_states],
actual_state[used_states],
3e-3,
err_atol=3e-3,
)
assert torch.isnan(output_storage[..., H * D :]).all()
@pytest.mark.parametrize(
("num_heads", "num_seqs", "lower_bound", "fuse_output_norm"),
[
(12, 1, -5.0, True),
(12, 4, None, False),
(24, 4, None, False),
(48, 1, -5.0, True),
(96, 1, -5.0, True),
],
)
@torch.inference_mode()
def test_fused_kda_decode_correctness(
num_heads: int,
num_seqs: int,
lower_bound: float | None,
fuse_output_norm: bool,
):
D, W = 128, 4
if not is_fused_kda_decode_supported(
num_heads,
D,
W,
num_spec=0,
input_dtype=torch.bfloat16,
conv_state_dtype=torch.bfloat16,
):
pytest.skip("Fused KDA decode is not supported on this platform")
torch.manual_seed(967 + num_heads + num_seqs)
dim = num_heads * D
slots = num_seqs + 2
packed_x_storage = torch.randn(
num_seqs, 3 * dim + 17, dtype=torch.bfloat16, device=DEVICE
)
packed_x = packed_x_storage[:, : 3 * dim]
weight = 0.1 * torch.randn(3 * dim, W, dtype=torch.float32, device=DEVICE)
conv_seed = 0.1 * torch.randn(
slots,
W - 1,
3 * dim,
dtype=torch.bfloat16,
device=DEVICE,
).transpose(1, 2)
raw_g = torch.randn(
1,
num_seqs,
num_heads,
D,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta_storage = torch.randn(
1,
num_seqs,
num_heads + 1,
dtype=torch.bfloat16,
device=DEVICE,
)
raw_beta = raw_beta_storage[:, :, :num_heads]
output_gate_storage = torch.randn(
num_seqs,
dim + 7,
dtype=torch.bfloat16,
device=DEVICE,
)
output_gate = output_gate_storage[:, :dim].view(num_seqs, num_heads, D)
norm_weight = torch.randn(D, dtype=torch.float32, device=DEVICE)
norm_eps = 1e-5
A_log = 0.5 * torch.randn(num_heads, dtype=torch.float32, device=DEVICE)
dt_bias = 0.1 * torch.randn(dim, dtype=torch.float32, device=DEVICE)
state_indices = torch.arange(
num_seqs,
0,
-1,
dtype=torch.int32,
device=DEVICE,
)
state_seed = 0.01 * torch.randn(
slots,
num_heads,
D,
D,
dtype=torch.float32,
device=DEVICE,
)
conv_ref = conv_seed.clone()
state_ref = state_seed.clone()
mixed_qkv = causal_conv1d_update(
packed_x,
conv_ref,
weight,
activation="silu",
conv_state_indices=state_indices,
validate_data=True,
out=torch.empty_like(packed_x),
)
expected, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=mixed_qkv,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=lower_bound,
initial_state=state_ref,
state_indices=state_indices,
)
if fuse_output_norm:
expected_float = expected.float()
expected = (
expected_float
* torch.rsqrt(expected_float.square().mean(dim=-1, keepdim=True) + norm_eps)
* norm_weight
* output_gate.float().sigmoid().unsqueeze(0)
).to(expected.dtype)
conv_slot_elements = 3 * dim * (W - 1)
state_slot_elements = num_heads * D * D
conv_slot_bytes = conv_slot_elements * torch.bfloat16.itemsize
page_bytes = conv_slot_bytes + state_slot_elements * torch.float32.itemsize
cache_storage = torch.empty(slots * page_bytes, dtype=torch.uint8, device=DEVICE)
conv_actual = torch.as_strided(
cache_storage.view(torch.bfloat16),
size=(slots, 3 * dim, W - 1),
stride=(page_bytes // torch.bfloat16.itemsize, 1, 3 * dim),
)
state_actual = torch.as_strided(
cache_storage.view(torch.float32),
size=(slots, num_heads, D, D),
stride=(page_bytes // torch.float32.itemsize, D * D, D, 1),
storage_offset=conv_slot_bytes // torch.float32.itemsize,
)
conv_actual.copy_(conv_seed)
state_actual.copy_(state_seed)
fused_weight = weight.reshape(3, dim, W).transpose(1, 2).contiguous()
actual = ops.fused_kda_decode(
x=packed_x,
weight=fused_weight,
bias=None,
conv_state=conv_actual,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
state_indices=state_indices,
state=state_actual,
lower_bound=lower_bound,
output_gate=output_gate if fuse_output_norm else None,
norm_weight=norm_weight if fuse_output_norm else None,
norm_eps=norm_eps,
)
torch.testing.assert_close(actual, expected, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(conv_actual, conv_ref, atol=0, rtol=0)
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
def test_fused_kda_decode_rejects_speculative_conv_state():
assert not is_fused_kda_decode_supported(
num_heads=12,
head_dim=128,
conv_width=4,
num_spec=2,
input_dtype=torch.bfloat16,
conv_state_dtype=torch.bfloat16,
)
@torch.inference_mode()
def test_flashkda_correctness():
if not is_flashkda_supported(128, torch.bfloat16, -3.0):
pytest.skip("FlashKDA is not supported on this platform")
import vllm._flashkda_C # noqa: F401
B, T, H, D = 1, 48, 2, 128
torch.manual_seed(11)
q, k, v, raw_g = [
torch.randn(B, T, H, D, dtype=torch.bfloat16, device=DEVICE) for _ in range(4)
]
beta_logits = torch.randn(B, T, H, dtype=torch.bfloat16, device=DEVICE)
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
initial_state = torch.randn(2, H, D, D, dtype=torch.float32, device=DEVICE)
cu_seqlens = torch.tensor([0, 17, T], dtype=torch.int32, device=DEVICE)
lower_bound = -3.0
gate = lower_bound * torch.sigmoid(
A_log.exp()[None, None, :, None] * (raw_g.float() + dt_bias[None, None, :, :])
)
beta = beta_logits.float().sigmoid()
q_norm = l2norm_fwd(q.contiguous())
k_norm = l2norm_fwd(k.contiguous())
expected_outputs = []
expected_states = []
for i, (start, end) in enumerate(
zip(cu_seqlens[:-1].tolist(), cu_seqlens[1:].tolist())
):
output, final_state = naive_recurrent_kda(
q_norm[:, start:end],
k_norm[:, start:end],
v[:, start:end],
gate[:, start:end],
beta[:, start:end],
initial_state=initial_state[i].transpose(-1, -2),
output_final_state=True,
)
expected_outputs.append(output)
expected_states.append(final_state)
expected_out = torch.cat(expected_outputs, dim=1)
expected_state = torch.cat(expected_states).transpose(-1, -2).contiguous()
actual_out = torch.empty_like(v)
actual_state = torch.empty_like(initial_state)
workspace = torch.empty(
torch.ops._flashkda_C.get_workspace_size(T, H, cu_seqlens.numel() - 1),
dtype=torch.uint8,
device=DEVICE,
)
torch.ops._flashkda_C.fwd(
q,
k,
v,
raw_g,
beta_logits,
D**-0.5,
actual_out,
workspace,
A_log,
dt_bias,
lower_bound,
initial_state,
actual_state,
cu_seqlens,
)
assert_close("o", expected_out, actual_out, 0.01)
assert_close("ht", expected_state, actual_state, 0.01)
+411
View File
@@ -0,0 +1,411 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import fields
import pytest
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.models.kimi_k3.nvidia.kda_metadata import (
KimiK3KDAAttentionBackend,
KimiK3KDAMetadata,
KimiK3KDAMetadataBuilder,
_mamba_get_block_table_tensor,
stage_spec_decode_metadata,
)
from vllm.v1.attention.backend import AttentionMetadataBuilder
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import (
NULL_BLOCK_ID,
mamba_get_block_table_tensor,
)
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
DEVICE = torch.device("cpu")
PRUNED_METADATA_FIELDS = {
"chunk_indices",
"chunk_offsets",
"prefill_query_start_loc",
"prefill_state_indices",
"prefill_has_initial_state",
"spec_sequence_masks",
}
def _assert_matches_shared_gdn(reference, actual: KimiK3KDAMetadata):
for field in fields(KimiK3KDAMetadata):
actual_value = getattr(actual, field.name)
expected_value = getattr(reference, field.name)
if field.name in PRUNED_METADATA_FIELDS:
assert actual_value is None
continue
if (
field.name in {"spec_token_indx", "non_spec_token_indx"}
and actual.num_spec_decodes > 0
and actual.num_prefills == 0
and actual.num_decodes == 0
):
assert actual_value is None
continue
if isinstance(actual_value, torch.Tensor):
torch.testing.assert_close(actual_value, expected_value)
elif field.name == "nums_dict":
assert (actual_value is None) == (expected_value is None)
if actual_value is not None:
assert actual_value[8]["tot"] == expected_value[8]["tot"]
torch.testing.assert_close(
actual_value[8]["nums"], expected_value[8]["nums"]
)
else:
assert actual_value == expected_value
def _make_builder(
builder_cls: type[AttentionMetadataBuilder],
num_speculative_tokens: int,
full_cuda_graph: bool,
device: torch.device = DEVICE,
mamba_cache_mode: str = "none",
) -> AttentionMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
if num_speculative_tokens:
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=num_speculative_tokens,
)
vllm_config.compilation_config.cudagraph_mode = (
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
return builder_cls(
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=num_speculative_tokens,
),
layer_names=["layer.0"],
vllm_config=vllm_config,
device=device,
)
@pytest.mark.parametrize(
(
"batch",
"num_decode_draft_tokens",
"num_speculative_tokens",
"full_cuda_graph",
"is_prefilling",
),
[
pytest.param(
BatchSpec(seq_lens=[50, 30], query_lens=[3, 3]),
[2, 2],
2,
False,
[False, False],
id="pure-spec-decode",
),
pytest.param(
BatchSpec(seq_lens=[100, 65, 20], query_lens=[50, 1, 3]),
[-1, -1, 2],
2,
False,
[True, False, False],
id="mixed-prefill-and-spec-decode",
),
pytest.param(
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
None,
0,
False,
[False, False],
id="regular-decode",
),
pytest.param(
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
[0, 0],
2,
False,
[False, False],
id="no-scheduled-draft-tokens",
),
],
)
def test_kimi_k3_kda_metadata_matches_shared_gdn(
batch: BatchSpec,
num_decode_draft_tokens: list[int] | None,
num_speculative_tokens: int,
full_cuda_graph: bool,
is_prefilling: list[bool],
):
kwargs: dict[str, torch.Tensor] = {}
if num_decode_draft_tokens is not None:
kwargs = {
"num_decode_draft_tokens_cpu": torch.tensor(
num_decode_draft_tokens, dtype=torch.int32
),
"num_accepted_tokens": torch.ones(
batch.batch_size, dtype=torch.int32, device=DEVICE
),
}
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor(is_prefilling, dtype=torch.bool))
reference = _make_builder(
GDNAttentionMetadataBuilder,
num_speculative_tokens,
full_cuda_graph,
).build(
0,
common_attn_metadata,
**kwargs,
)
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens,
full_cuda_graph,
).build(0, common_attn_metadata, **kwargs)
assert isinstance(actual, KimiK3KDAMetadata)
_assert_matches_shared_gdn(reference, actual)
def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():
batch = BatchSpec(seq_lens=[100, 65, 20], query_lens=[1, 1, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([False, False, False]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
).build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
)
# The K3 layer dispatches the non-spec subgroup to packed decode whenever
# it contains no prefill request.
assert actual.num_decodes == 2
assert actual.num_decode_tokens == 2
assert actual.num_prefills == 0
assert actual.num_prefill_tokens == 0
assert actual.has_initial_state is None
assert actual.nums_dict is None
assert actual.non_spec_query_start_loc is None
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0, 1]))
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([2, 3, 4]))
torch.testing.assert_close(
actual.spec_query_start_loc,
torch.tensor([0, 3], dtype=torch.int32),
)
def test_mixed_regular_and_spec_decode_excludes_request_padding():
batch = BatchSpec(seq_lens=[16, 65, 20], query_lens=[0, 1, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([False, False, False]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
).build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
)
assert actual.num_decodes == 1
assert actual.non_spec_state_indices_tensor is not None
assert actual.non_spec_state_indices_tensor.shape == (1,)
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0]))
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
@pytest.mark.parametrize(
("seq_len", "expected_has_initial_state"),
[
pytest.param(1, False, id="first-token-prefill"),
pytest.param(65, True, id="final-one-token-prefill-chunk"),
],
)
def test_mixed_one_token_prefill_and_spec_decode_uses_prefill_metadata(
seq_len: int,
expected_has_initial_state: bool,
):
batch = BatchSpec(seq_lens=[seq_len, 20], query_lens=[1, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([True, False]))
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
).build(
0,
common_attn_metadata,
num_decode_draft_tokens_cpu=torch.tensor([-1, 2], dtype=torch.int32),
num_accepted_tokens=torch.ones(2, dtype=torch.int32, device=DEVICE),
)
assert actual.num_prefills == 1
assert actual.num_prefill_tokens == 1
assert actual.num_decodes == 0
assert actual.num_decode_tokens == 0
assert actual.has_initial_state is not None
assert actual.has_initial_state.tolist() == [expected_has_initial_state]
assert actual.non_spec_query_start_loc is not None
torch.testing.assert_close(
actual.non_spec_query_start_loc,
torch.tensor([0, 1], dtype=torch.int32),
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_kimi_k3_kda_cudagraph_capture_matches_shared_gdn():
device = torch.device("cuda")
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, device
).replace(is_prefilling=torch.tensor([False, False]))
reference = _make_builder(
GDNAttentionMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=True,
device=device,
).build_for_cudagraph_capture(common_attn_metadata)
actual = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=True,
device=device,
).build_for_cudagraph_capture(common_attn_metadata)
assert isinstance(actual, KimiK3KDAMetadata)
_assert_matches_shared_gdn(reference, actual)
def test_kimi_k3_kda_backend_uses_private_metadata_builder():
assert KimiK3KDAAttentionBackend.get_builder_cls() is KimiK3KDAMetadataBuilder
assert KimiK3KDAAttentionBackend.is_ssm()
assert issubclass(KimiK3KDAAttentionBackend, GDNAttentionBackend)
assert issubclass(KimiK3KDAMetadata, GDNAttentionMetadata)
assert issubclass(KimiK3KDAMetadataBuilder, GDNAttentionMetadataBuilder)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_stage_spec_decode_metadata_matches_pytorch():
device = torch.device("cuda")
num_spec_decodes = 33
batch_size = 65
num_state_slots = 3
state_indices = torch.arange(
num_spec_decodes * 32,
dtype=torch.int32,
device=device,
).reshape(num_spec_decodes, 32)[:, :num_state_slots]
query_start_loc = (
torch.arange(num_spec_decodes + 1, dtype=torch.int32, device=device)
* num_state_slots
)
num_accepted_tokens = (
torch.arange(num_spec_decodes, dtype=torch.int32, device=device)
% num_state_slots
+ 1
)
staged_state_indices = torch.empty(
(batch_size, num_state_slots), dtype=torch.int32, device=device
)
staged_query_start_loc = torch.empty(
batch_size + 1, dtype=torch.int32, device=device
)
staged_num_accepted_tokens = torch.empty(
batch_size, dtype=torch.int32, device=device
)
stage_spec_decode_metadata(
state_indices,
query_start_loc,
num_accepted_tokens,
staged_state_indices,
staged_query_start_loc,
staged_num_accepted_tokens,
num_spec_decodes=num_spec_decodes,
)
expected_state_indices = torch.full_like(staged_state_indices, NULL_BLOCK_ID)
expected_state_indices[:num_spec_decodes] = state_indices
expected_query_start_loc = torch.full(
(batch_size + 1,),
query_start_loc[-1],
dtype=torch.int32,
device=device,
)
expected_query_start_loc[: num_spec_decodes + 1] = query_start_loc
expected_num_accepted_tokens = torch.ones(
batch_size, dtype=torch.int32, device=device
)
expected_num_accepted_tokens[:num_spec_decodes] = num_accepted_tokens
torch.testing.assert_close(staged_state_indices, expected_state_indices)
torch.testing.assert_close(staged_query_start_loc, expected_query_start_loc)
torch.testing.assert_close(staged_num_accepted_tokens, expected_num_accepted_tokens)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_aligned_block_table_matches_shared_gdn():
device = torch.device("cuda")
seq_lens = torch.tensor(
[0, 1, 15, 16, 17, 31, 32, 33, 511, 512, 513],
dtype=torch.int32,
device=device,
).repeat(6)[:65]
block_table_storage = torch.arange(
seq_lens.numel() * 128,
dtype=torch.int32,
device=device,
).reshape(seq_lens.numel(), 128)
block_table = block_table_storage[:, ::2]
kv_cache_spec = MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=2,
)
expected = mamba_get_block_table_tensor(
block_table,
seq_lens,
kv_cache_spec,
"align",
)
actual = _mamba_get_block_table_tensor(
block_table,
seq_lens,
kv_cache_spec,
"align",
)
torch.testing.assert_close(actual, expected)
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import ray
import torch
import torch.distributed as dist
import torch.nn.functional as F
from tests.utils import (
init_test_distributed_environment,
multi_gpu_test,
multi_process_parallel,
)
from vllm.distributed import get_tp_group
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import KimiK3LatentMoETailOp
from vllm.platforms import current_platform
HIDDEN_SIZE = 7168
LATENT_SIZE = 3584
EPS = 0.1
@ray.remote(num_gpus=1, max_calls=1)
def _test_latent_moe_tail_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
)
torch.manual_seed(0)
rms_weight = 1 + 0.1 * torch.randn(
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
up_weight = (
torch.randn(
HIDDEN_SIZE,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
/ LATENT_SIZE**0.5
)
group = get_tp_group().device_group
op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
)
cutedsl_warmup()
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
torch.manual_seed(100 * iteration + rank + 1)
routed_output = torch.randn(
num_tokens,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
).mul_(0.01)
shared_output = torch.randn(
num_tokens,
HIDDEN_SIZE,
device=device,
dtype=torch.bfloat16,
)
routed_reference = routed_output.clone()
shared_reference = shared_output.clone()
dist.all_reduce(routed_reference, group=group)
dist.all_reduce(shared_reference, group=group)
expected = F.linear(
F.rms_norm(
routed_reference,
(LATENT_SIZE,),
rms_weight,
EPS,
),
up_weight,
)
expected.add_(shared_reference)
actual = op(
routed_output,
shared_output,
rms_weight,
up_weight,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
assert actual.is_contiguous()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_output = op(
routed_output,
shared_output,
rms_weight,
up_weight,
)
graph.replay()
torch.testing.assert_close(graph_output, expected, atol=8e-2, rtol=3e-2)
def _run_latent_moe_tail_test(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
) -> None:
if not current_platform.is_device_capability_family(100):
pytest.skip("K3 latent-MoE tail fusion requires SM100")
multi_process_parallel(
monkeypatch,
tp_size,
1,
_test_latent_moe_tail_worker,
)
@multi_gpu_test(num_gpus=8)
def test_latent_moe_tail_tp8_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_latent_moe_tail_test(monkeypatch, 8)
@multi_gpu_test(num_gpus=16)
def test_latent_moe_tail_tp16_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_latent_moe_tail_test(monkeypatch, 16)
@@ -0,0 +1,343 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import MethodType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from torch import nn
from vllm.config import ParallelConfig
from vllm.models.kimi_k3.nvidia import model as kimi_model
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
from vllm.models.kimi_k3.nvidia.ops import sequence_parallel as sp_ops
from vllm.platforms import current_platform
class _IdentityNorm(nn.Module):
def __init__(self, hidden_size: int = 2) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
self.variance_epsilon = 1e-5
def forward(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None = None,
):
if residual is None:
return hidden_states
return hidden_states, residual
class _RecordingMoE(nn.Module):
def __init__(self) -> None:
super().__init__()
self.num_tokens = 0
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
self.num_tokens = hidden_states.shape[0]
return hidden_states
class _Projection(nn.Module):
def __init__(self, hidden_size: int = 2) -> None:
super().__init__()
self.weight = nn.Parameter(
torch.ones(1, hidden_size),
requires_grad=False,
)
class _SequenceParallelMTPBlock:
use_sequence_parallel = True
def __call__(
self,
*,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
):
assert residual is None
return hidden_states * 2, None, hidden_states * 3
def _mock_sequence_parallel_collectives(monkeypatch):
monkeypatch.setattr(
kimi_model,
"sp_reduce_scatter",
lambda tensor: tensor.chunk(2, dim=0)[0],
)
monkeypatch.setattr(
kimi_model,
"sp_shard",
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
)
monkeypatch.setattr(
kimi_model,
"sp_all_gather",
lambda tensor: torch.cat([tensor, tensor], dim=0),
)
@pytest.mark.parametrize(
("num_tokens", "is_padding", "tp_rank", "expected"),
[
(1, None, 0, [False]),
(1, None, 1, [True]),
(5, None, 2, [False, True]),
(5, None, 3, [True, True]),
(5, [False, True, False, False, False], 0, [False, True]),
],
)
def test_sp_padding_mask_marks_added_rows(
monkeypatch,
num_tokens: int,
is_padding: list[bool] | None,
tp_rank: int,
expected: list[bool],
):
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_world_size", lambda: 4)
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
hidden_states = torch.empty(num_tokens, 2)
padding = torch.tensor(is_padding) if is_padding is not None else None
actual = sp_ops.sp_padding_mask(padding, hidden_states)
torch.testing.assert_close(actual, torch.tensor(expected))
@pytest.mark.parametrize(
("data_parallel_size", "expected"),
[
(1, False),
(2, True),
],
)
def test_moe_sequence_parallel_requires_data_parallel(
monkeypatch,
data_parallel_size: int,
expected: bool,
):
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
parallel_config = ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=data_parallel_size,
enable_expert_parallel=True,
all2all_backend="allgather_reducescatter",
)
assert parallel_config.use_sequence_parallel_moe is expected
def test_kimi_decoder_layer_keeps_moe_states_sequence_sharded(monkeypatch):
layer = object.__new__(kimi_model.KimiDecoderLayer)
nn.Module.__init__(layer)
layer.use_attn_res = False
layer.use_sequence_parallel = True
layer.input_layernorm = _IdentityNorm()
layer.post_attention_layernorm = _IdentityNorm()
layer.mlp = _RecordingMoE()
layer._run_self_attn = MethodType(
lambda self, positions, hidden_states: hidden_states,
layer,
)
_mock_sequence_parallel_collectives(monkeypatch)
positions = torch.arange(3)
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
hidden_states = kimi_model.sp_shard(full_hidden_states)
hidden_states, prefix_sum, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=None,
)
assert prefix_sum is None
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.mlp.num_tokens == 2
hidden_states, prefix_sum, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
)
assert prefix_sum is None
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.mlp.num_tokens == 2
def test_kimi_attn_residual_states_stay_sequence_sharded(monkeypatch):
layer = object.__new__(kimi_model.KimiDecoderLayer)
nn.Module.__init__(layer)
layer.use_attn_res = True
layer.use_sequence_parallel = True
layer.prev_valid_blocks = 0
layer.block_write_idx = 0
layer.is_block_write_layer = False
layer.input_layernorm = _IdentityNorm()
layer.post_attention_layernorm = _IdentityNorm()
layer.self_attention_res_norm = _IdentityNorm()
layer.mlp_res_norm = _IdentityNorm()
layer.self_attention_res_proj = _Projection()
layer.mlp_res_proj = _Projection()
layer.mlp = _RecordingMoE()
layer._run_self_attn = MethodType(
lambda self, positions, hidden_states: hidden_states,
layer,
)
_mock_sequence_parallel_collectives(monkeypatch)
monkeypatch.setattr(
kimi_model,
"attn_res",
lambda prefix_sum, hidden_states, *args, **kwargs: (
prefix_sum if hidden_states is None else prefix_sum + hidden_states
),
)
prefix_sum = kimi_model.sp_shard(torch.arange(6, dtype=torch.float32).view(3, 2))
block_residual = torch.zeros(2, 1, 2)
hidden_states, prefix_sum, block_residual = layer(
positions=torch.arange(3),
hidden_states=None,
prefix_sum=prefix_sum,
residual=block_residual,
)
assert hidden_states.shape == prefix_sum.shape == (2, 2)
assert block_residual.shape == (2, 1, 2)
assert layer.mlp.num_tokens == 2
def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
layer = object.__new__(kimi_mtp.KimiK3MultiTokenPredictorLayer)
nn.Module.__init__(layer)
layer.enorm = _IdentityNorm()
layer.hnorm = _IdentityNorm()
layer.eh_proj = nn.Identity()
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
final_norm = Mock(side_effect=lambda hidden_states: hidden_states + 1)
object.__setattr__(
layer,
"shared_head",
SimpleNamespace(norm=final_norm),
)
monkeypatch.setattr(
kimi_mtp,
"fused_mtp_input",
lambda positions, inputs_embeds, *args: inputs_embeds,
)
monkeypatch.setattr(
kimi_mtp,
"sp_shard",
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
)
monkeypatch.setattr(
kimi_mtp,
"sp_all_gather",
lambda tensor: torch.cat([tensor, tensor], dim=0),
)
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
logits_hidden_states, hidden_states = layer(
input_ids=torch.zeros(3, dtype=torch.long),
positions=torch.arange(3),
previous_hidden_states=torch.zeros_like(inputs_embeds),
inputs_embeds=inputs_embeds,
)
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
expected_hidden_states = torch.cat(
[sharded_states * 5, sharded_states * 5],
dim=0,
)[:3]
torch.testing.assert_close(hidden_states, expected_hidden_states)
torch.testing.assert_close(logits_hidden_states, expected_hidden_states + 1)
final_norm.assert_called_once()
torch.testing.assert_close(final_norm.call_args.args[0], expected_hidden_states)
def test_sp_all_gather_uses_custom_kernel(monkeypatch):
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
expected = torch.cat([hidden_states, hidden_states])
custom_all_gather = Mock(return_value=expected)
device_communicator = SimpleNamespace(
custom_all_gather=custom_all_gather,
)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_communicator),
)
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", fallback)
output = sp_ops.sp_all_gather(hidden_states)
torch.testing.assert_close(output, expected)
custom_all_gather.assert_called_once_with(hidden_states)
fallback.assert_not_called()
def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
expected = torch.arange(4, dtype=torch.float32).view(2, 2)
custom_reduce_scatter = Mock(return_value=expected)
device_communicator = SimpleNamespace(
custom_reduce_scatter=custom_reduce_scatter,
)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_communicator),
)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
monkeypatch.setattr(sp_ops, "tensor_model_parallel_reduce_scatter", fallback)
output = sp_ops.sp_reduce_scatter(hidden_states)
torch.testing.assert_close(output, expected)
padded = custom_reduce_scatter.call_args.args[0]
assert padded.shape == (4, 2)
torch.testing.assert_close(padded[:3], hidden_states)
torch.testing.assert_close(padded[3], torch.zeros(2))
fallback.assert_not_called()
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=None),
)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
all_gather = Mock(return_value=hidden_states)
reduce_scatter = Mock(return_value=hidden_states)
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", all_gather)
monkeypatch.setattr(
sp_ops,
"tensor_model_parallel_reduce_scatter",
reduce_scatter,
)
torch.testing.assert_close(sp_ops.sp_all_gather(hidden_states), hidden_states)
torch.testing.assert_close(sp_ops.sp_reduce_scatter(hidden_states), hidden_states)
all_gather.assert_called_once_with(hidden_states, 0)
reduce_scatter.assert_called_once_with(hidden_states, 0)
@@ -1242,7 +1242,7 @@ def test_custom_inputs_models(
create_new_process_for_each_test=True,
),
)
@create_new_process_for_each_test()
@create_new_process_for_each_test("spawn")
def test_single_image_models_heavy(
tmp_path: PosixPath,
model_type: str,