[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -5,6 +5,7 @@ import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.common.mtp import fused_mtp_input
|
||||
from vllm.models.kimi_k3.nvidia.ops import attn_res
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -191,3 +192,35 @@ def test_attn_res_without_output_norm():
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 17])
|
||||
def test_fused_mtp_input(num_tokens: int):
|
||||
positions = torch.arange(num_tokens, device="cuda")
|
||||
inputs_embeds = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=7)
|
||||
previous_hidden_states = _randn_with_row_padding(
|
||||
num_tokens, HIDDEN_SIZE, padding=11
|
||||
)
|
||||
enorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
hnorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
masked_inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
expected = torch.cat(
|
||||
(
|
||||
F.rms_norm(masked_inputs_embeds, (HIDDEN_SIZE,), enorm_weight, EPS),
|
||||
F.rms_norm(previous_hidden_states, (HIDDEN_SIZE,), hnorm_weight, EPS),
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
actual = fused_mtp_input(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
enorm_weight,
|
||||
hnorm_weight,
|
||||
EPS,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
assert actual.shape == (num_tokens, 2 * HIDDEN_SIZE)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.interfaces import supports_eagle3
|
||||
from vllm.models.kimi_k3.nvidia import model as kimi_model
|
||||
from vllm.models.kimi_k3.nvidia.model import (
|
||||
KimiK3ForConditionalGeneration,
|
||||
KimiLinearModel,
|
||||
)
|
||||
|
||||
|
||||
def _make_kimi_linear_model() -> KimiLinearModel:
|
||||
model = object.__new__(KimiLinearModel)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (2,))
|
||||
object.__setattr__(model, "use_sequence_parallel", False)
|
||||
return model
|
||||
|
||||
|
||||
def test_kimi_k3_advertises_eagle3_support():
|
||||
assert supports_eagle3(KimiK3ForConditionalGeneration)
|
||||
|
||||
|
||||
def test_kimi_k3_uses_shared_eagle3_layer_configuration():
|
||||
target = object.__new__(KimiK3ForConditionalGeneration)
|
||||
torch.nn.Module.__init__(target)
|
||||
model = _make_kimi_linear_model()
|
||||
object.__setattr__(model, "layers", [None] * 93)
|
||||
language_model = SimpleNamespace(
|
||||
embed_input_ids=lambda _: None,
|
||||
model=model,
|
||||
)
|
||||
object.__setattr__(target, "language_model", language_model)
|
||||
object.__setattr__(target, "_language_model_names", ["language_model"])
|
||||
|
||||
target.set_aux_hidden_state_layers((2, 46, 90))
|
||||
|
||||
assert model.aux_hidden_state_layers == (2, 46, 90)
|
||||
assert target.get_eagle3_default_aux_hidden_state_layers() == (
|
||||
2,
|
||||
46,
|
||||
90,
|
||||
)
|
||||
|
||||
|
||||
def test_kimi_linear_forward_extracts_standard_aux_hidden_states(monkeypatch):
|
||||
model = _make_kimi_linear_model()
|
||||
initial_hidden_states = torch.tensor([[1.0, 2.0]])
|
||||
layer_hidden_states = torch.tensor([[3.0, 4.0]])
|
||||
layer_residual = torch.tensor([[5.0, 6.0]])
|
||||
|
||||
object.__setattr__(model, "start_layer", 0)
|
||||
object.__setattr__(model, "end_layer", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"layers",
|
||||
[Mock(return_value=(layer_hidden_states, None, layer_residual))],
|
||||
)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
|
||||
object.__setattr__(model, "use_attn_res", False)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=initial_hidden_states,
|
||||
)
|
||||
|
||||
expected_layer_output = layer_hidden_states + layer_residual
|
||||
torch.testing.assert_close(output, expected_layer_output)
|
||||
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[1], expected_layer_output)
|
||||
|
||||
|
||||
def test_kimi_linear_forward_extracts_attn_res_aux_hidden_states(monkeypatch):
|
||||
model = _make_kimi_linear_model()
|
||||
initial_hidden_states = torch.tensor([[1.0, 2.0]])
|
||||
layer_hidden_states = torch.tensor([[3.0, 4.0]])
|
||||
prefix_sum = torch.tensor([[5.0, 6.0]])
|
||||
block_residual = torch.tensor([[[7.0, 8.0]]])
|
||||
final_hidden_states = torch.tensor([[9.0, 10.0]])
|
||||
|
||||
object.__setattr__(model, "start_layer", 0)
|
||||
object.__setattr__(model, "end_layer", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"layers",
|
||||
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
|
||||
)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
|
||||
object.__setattr__(model, "use_attn_res", True)
|
||||
object.__setattr__(model, "num_attn_res_blocks", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_norm",
|
||||
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
|
||||
)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_proj",
|
||||
SimpleNamespace(weight=torch.ones(1, 2)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
final_attn_res = Mock(return_value=final_hidden_states)
|
||||
monkeypatch.setattr(kimi_model, "attn_res", final_attn_res)
|
||||
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=initial_hidden_states,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, final_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[1], prefix_sum + layer_hidden_states)
|
||||
assert final_attn_res.call_args.args[2] is block_residual
|
||||
@@ -0,0 +1,757 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Precision tests for vllm's chunk_kda Triton operator.
|
||||
|
||||
Compares chunk_kda against a naive recurrent reference (float32).
|
||||
Uses torch.rand for q/k/v to match FLA's test pattern.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_update
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.kda import (
|
||||
is_flashkda_supported,
|
||||
is_fused_kda_decode_supported,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops.third_party.kda import (
|
||||
chunk_kda,
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_kda_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_fwd,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gather_initial_states_correctness():
|
||||
row_size = 8 * 128 * 128
|
||||
storage = torch.randn(5, row_size + 256, dtype=torch.float32, device=DEVICE)
|
||||
state = storage[:, :row_size].view(5, 8, 128, 128)
|
||||
assert not state.is_contiguous()
|
||||
assert state[0].is_contiguous()
|
||||
indices = torch.tensor([4, 1, 3], dtype=torch.int32, device=DEVICE)
|
||||
has_initial_state = torch.tensor([True, False, True], device=DEVICE)
|
||||
|
||||
expected = state[indices].clone()
|
||||
expected[~has_initial_state] = 0
|
||||
|
||||
torch.testing.assert_close(
|
||||
gather_initial_states(state, indices, has_initial_state),
|
||||
expected,
|
||||
)
|
||||
|
||||
|
||||
def naive_recurrent_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Naive recurrent KDA reference, ported from FLA's naive.py."""
|
||||
dtype = v.dtype
|
||||
B, T, H, K = q.shape
|
||||
V = v.shape[-1]
|
||||
if scale is None:
|
||||
scale = K**-0.5
|
||||
|
||||
q, k, v, g, beta = (x.to(torch.float) for x in [q, k, v, g, beta])
|
||||
q = q * scale
|
||||
|
||||
S = k.new_zeros(B, H, K, V).to(q)
|
||||
if initial_state is not None:
|
||||
S += initial_state
|
||||
o = torch.zeros_like(v)
|
||||
for i in range(T):
|
||||
q_i, k_i, v_i, g_i, b_i = q[:, i], k[:, i], v[:, i], g[:, i], beta[:, i]
|
||||
S = S * g_i[..., None].exp()
|
||||
S = S + torch.einsum(
|
||||
"bhk,bhv->bhkv",
|
||||
b_i[..., None] * k_i,
|
||||
v_i - (k_i[..., None] * S).sum(-2),
|
||||
)
|
||||
o[:, i] = torch.einsum("bhk,bhkv->bhv", q_i, S)
|
||||
if not output_final_state:
|
||||
S = None
|
||||
return o.to(dtype), S
|
||||
|
||||
|
||||
def assert_close(
|
||||
name: str,
|
||||
ref: torch.Tensor,
|
||||
tri: torch.Tensor,
|
||||
ratio: float,
|
||||
err_atol: float = 1e-6,
|
||||
):
|
||||
"""RMSE-based relative error comparison."""
|
||||
abs_err = (ref.detach() - tri.detach()).flatten().abs().max().item()
|
||||
rmse_diff = (ref.detach() - tri.detach()).flatten().square().mean().sqrt().item()
|
||||
rmse_base = ref.detach().flatten().square().mean().sqrt().item()
|
||||
rel_err = rmse_diff / (rmse_base + 1e-8)
|
||||
print(f"{name:>4} | abs={abs_err:.6f} | rmse={rel_err:.6f} | thr={ratio}")
|
||||
if abs_err <= err_atol:
|
||||
return
|
||||
assert not torch.isnan(ref).any(), f"{name}: NaN detected in ref"
|
||||
assert not torch.isnan(tri).any(), f"{name}: NaN detected in tri"
|
||||
assert rel_err < ratio, (
|
||||
f"{name}: max abs err {abs_err:.6f}, rmse ratio {rel_err:.6f} >= {ratio}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("H", "D", "cu_seqlens", "dtype"),
|
||||
[
|
||||
pytest.param(
|
||||
*test,
|
||||
id="H{}-D{}-cu{}-{}".format(*test),
|
||||
)
|
||||
for test in [
|
||||
(32, 128, [0, 64], torch.float16),
|
||||
(32, 128, [0, 1024], torch.float16),
|
||||
(32, 128, [0, 15], torch.float16),
|
||||
(32, 128, [0, 256, 512, 768, 1024], torch.float16),
|
||||
(32, 128, [0, 15, 100, 300, 1200], torch.float16),
|
||||
(64, 128, [0, 256, 500, 1000], torch.float16),
|
||||
(32, 128, [0, 8192], torch.float16),
|
||||
(32, 128, [0, 256, 500, 1000], torch.bfloat16),
|
||||
]
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda(
|
||||
H: int,
|
||||
D: int,
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
T = cu_seqlens[-1]
|
||||
torch.manual_seed(42)
|
||||
B = 1
|
||||
cu_seqlens_t = torch.LongTensor(cu_seqlens).to(DEVICE)
|
||||
N = len(cu_seqlens) - 1
|
||||
|
||||
q = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.rand(B, T, H, D, dtype=dtype, device=DEVICE)
|
||||
g = F.logsigmoid(torch.randn(B, T, H, D, dtype=torch.float32, device=DEVICE)).to(
|
||||
dtype
|
||||
)
|
||||
beta = torch.rand(B, T, H, dtype=dtype, device=DEVICE).sigmoid()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
|
||||
# Naive reference with l2norm_fwd (same kernel as chunk_kda)
|
||||
ref_outputs = []
|
||||
ref_states = []
|
||||
for i in range(N):
|
||||
s, e = cu_seqlens[i], cu_seqlens[i + 1]
|
||||
q_i = l2norm_fwd(q[:, s:e].contiguous())
|
||||
k_i = l2norm_fwd(k[:, s:e].contiguous())
|
||||
o_i, ht_i = naive_recurrent_kda(
|
||||
q_i,
|
||||
k_i,
|
||||
v[:, s:e],
|
||||
g[:, s:e],
|
||||
beta[:, s:e],
|
||||
initial_state=h0[i],
|
||||
output_final_state=True,
|
||||
)
|
||||
ref_outputs.append(o_i)
|
||||
ref_states.append(ht_i)
|
||||
ref_o = torch.cat(ref_outputs, dim=1)
|
||||
ref_ht = torch.cat(ref_states, dim=0)
|
||||
|
||||
# h0 transposed to (V, K) layout for the kernel; naive uses (K, V)
|
||||
tri_o, tri_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=g.clone(),
|
||||
beta=beta.clone(),
|
||||
initial_state=h0.transpose(-1, -2).contiguous().clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert not torch.isnan(tri_o).any(), "Triton output o contains NaN"
|
||||
assert not torch.isnan(tri_ht).any(), "Triton output ht contains NaN"
|
||||
assert_close("o", ref_o, tri_o, 0.005)
|
||||
assert_close("ht", ref_ht, tri_ht.transpose(-1, -2).contiguous(), 0.005)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cu_seqlens", "dtype", "lower_bound"),
|
||||
[
|
||||
([0, 64], torch.float16, None),
|
||||
([0, 15, 100, 300], torch.bfloat16, None),
|
||||
([0, 15, 100, 300], torch.bfloat16, -3.0),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_chunk_kda_fused_gate_cumsum_matches_unfused(
|
||||
cu_seqlens: list[int],
|
||||
dtype: torch.dtype,
|
||||
lower_bound: float | None,
|
||||
):
|
||||
H, D = 8, 64
|
||||
T = cu_seqlens[-1]
|
||||
N = len(cu_seqlens) - 1
|
||||
torch.manual_seed(123)
|
||||
|
||||
cu_seqlens_t = torch.tensor(cu_seqlens, dtype=torch.int32, device=DEVICE)
|
||||
q = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
k = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
v = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
raw_g = torch.randn(1, T, H, D, dtype=dtype, device=DEVICE)
|
||||
beta_storage = torch.randn(1, T, 2 * H + 3, dtype=dtype, device=DEVICE)
|
||||
raw_beta = beta_storage[..., 1 : 2 * H + 1 : 2]
|
||||
beta = raw_beta.float().sigmoid()
|
||||
A_log = (torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5).contiguous()
|
||||
dt_bias = (
|
||||
torch.randn(H * D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
).contiguous()
|
||||
h0 = torch.randn(N, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
initial_state = h0.transpose(-1, -2).contiguous()
|
||||
|
||||
gate = fused_kda_gate(
|
||||
raw_g.reshape(T, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
)
|
||||
if lower_bound is not None:
|
||||
expected_gate = lower_bound * torch.sigmoid(
|
||||
A_log.exp()[None, :, None]
|
||||
* (raw_g.float().view(T, H, D) + dt_bias.view(H, D))
|
||||
)
|
||||
torch.testing.assert_close(gate, expected_gate)
|
||||
gate = gate.unsqueeze(0)
|
||||
old_o, old_ht = chunk_kda(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
g=gate,
|
||||
beta=beta,
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
new_o, new_ht = chunk_kda_with_fused_gate(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
v=v.clone(),
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=initial_state.clone(),
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_seqs", [1, 8, 32])
|
||||
@pytest.mark.parametrize("lower_bound", [-5.0, None])
|
||||
@pytest.mark.parametrize("state_indices_stride", [1, 8])
|
||||
@torch.inference_mode()
|
||||
def test_packed_kda_decode_correctness(
|
||||
num_seqs: int,
|
||||
lower_bound: float | None,
|
||||
state_indices_stride: int,
|
||||
):
|
||||
H, D = 8, 128
|
||||
torch.manual_seed(321)
|
||||
|
||||
packed_storage = torch.randn(
|
||||
num_seqs,
|
||||
3 * H * D + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
mixed_qkv = packed_storage[:, : 3 * H * D]
|
||||
assert mixed_qkv.stride(0) == 3 * H * D + 1
|
||||
q, k, v = (
|
||||
x.contiguous().view(1, num_seqs, H, D) for x in mixed_qkv.split(H * D, dim=-1)
|
||||
)
|
||||
raw_g = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
H,
|
||||
D,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
H,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
beta = raw_beta.float().sigmoid()
|
||||
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
|
||||
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
state_storage = torch.randn(
|
||||
num_seqs + 1,
|
||||
H * D * D + 17,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state = state_storage[:, : H * D * D].view(num_seqs + 1, H, D, D)
|
||||
assert not state.is_contiguous()
|
||||
assert state.stride()[1:] == (D * D, D, 1)
|
||||
state_indices_storage = torch.zeros(
|
||||
num_seqs,
|
||||
state_indices_stride,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_indices = state_indices_storage[:, 0]
|
||||
state_indices.copy_(
|
||||
torch.arange(
|
||||
1,
|
||||
num_seqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
)
|
||||
gate = fused_kda_gate(
|
||||
raw_g.reshape(num_seqs, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
).unsqueeze(0)
|
||||
dense_state = state.clone()
|
||||
dense_out, _ = fused_recurrent_kda_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=gate,
|
||||
beta=beta,
|
||||
scale=D**-0.5,
|
||||
initial_state=dense_state,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=torch.arange(
|
||||
num_seqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
),
|
||||
ssm_state_indices=state_indices,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
packed_state = state
|
||||
packed_out, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=packed_state,
|
||||
state_indices=state_indices,
|
||||
)
|
||||
|
||||
assert_close("o", dense_out, packed_out, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", dense_state, packed_state, 1e-3, err_atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("H", "fuse_gate"),
|
||||
[(12, True), (12, False), (12, None), (96, None)],
|
||||
)
|
||||
@pytest.mark.parametrize("lower_bound", [-5.0, None])
|
||||
@torch.inference_mode()
|
||||
def test_kda_spec_decode_correctness(
|
||||
H: int,
|
||||
fuse_gate: bool | None,
|
||||
lower_bound: float | None,
|
||||
):
|
||||
num_seqs, query_len, D = 3, 3, 128
|
||||
T = num_seqs * query_len
|
||||
torch.manual_seed(1234)
|
||||
|
||||
qkv_storage = torch.randn(
|
||||
1,
|
||||
T,
|
||||
3 * H * D + 7,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
packed_qkv = qkv_storage[..., : 3 * H * D]
|
||||
q, k, v = (x.view(1, T, H, D) for x in packed_qkv.split(H * D, dim=-1))
|
||||
gate_storage = torch.randn(
|
||||
1,
|
||||
T,
|
||||
H * D + 5,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_g = gate_storage[..., : H * D].view(1, T, H, D)
|
||||
beta_storage = torch.randn(
|
||||
1,
|
||||
T,
|
||||
H + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta = beta_storage[..., :H]
|
||||
A_log = 0.5 * torch.randn(H, dtype=torch.float32, device=DEVICE)
|
||||
dt_bias = 0.1 * torch.randn(H, D, dtype=torch.float32, device=DEVICE)
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
T + 1,
|
||||
query_len,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_indices = torch.arange(
|
||||
1,
|
||||
T + 1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
).view(num_seqs, query_len)
|
||||
num_accepted_tokens = torch.tensor(
|
||||
[1, 2, 3],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_storage = 0.01 * torch.randn(
|
||||
T + 1,
|
||||
H * D * D + 17,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state = state_storage[:, : H * D * D].view(T + 1, H, D, D)
|
||||
output_storage = torch.full(
|
||||
(1, T, H * D + 11),
|
||||
torch.nan,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
output = output_storage[..., : H * D].view(1, T, H, D)
|
||||
|
||||
gate = fused_kda_gate(
|
||||
raw_g.contiguous().view(T, H * D),
|
||||
A_log,
|
||||
D,
|
||||
g_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
).unsqueeze(0)
|
||||
beta = raw_beta.float().sigmoid()
|
||||
q_norm = l2norm_fwd(q.contiguous())
|
||||
k_norm = l2norm_fwd(k.contiguous())
|
||||
expected_state = state.clone()
|
||||
expected_outputs = []
|
||||
for seq, accepted in enumerate(num_accepted_tokens.tolist()):
|
||||
recurrent_state = expected_state[state_indices[seq, accepted - 1]].transpose(
|
||||
-1, -2
|
||||
)
|
||||
start = seq * query_len
|
||||
for token in range(query_len):
|
||||
token_slice = slice(start + token, start + token + 1)
|
||||
token_output, recurrent_state = naive_recurrent_kda(
|
||||
q_norm[:, token_slice],
|
||||
k_norm[:, token_slice],
|
||||
v[:, token_slice],
|
||||
gate[:, token_slice],
|
||||
beta[:, token_slice],
|
||||
initial_state=recurrent_state,
|
||||
output_final_state=True,
|
||||
)
|
||||
assert recurrent_state is not None
|
||||
expected_outputs.append(token_output)
|
||||
expected_state[state_indices[seq, token]] = recurrent_state.transpose(
|
||||
-1, -2
|
||||
)
|
||||
expected = torch.cat(expected_outputs, dim=1)
|
||||
|
||||
actual_state = state.clone()
|
||||
actual, _ = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=actual_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
out=output,
|
||||
fuse_gate=fuse_gate,
|
||||
)
|
||||
|
||||
assert actual.data_ptr() == output.data_ptr()
|
||||
assert_close("o", expected, actual, 1e-3, err_atol=1e-3)
|
||||
used_states = state_indices.flatten().long()
|
||||
assert_close(
|
||||
"ht",
|
||||
expected_state[used_states],
|
||||
actual_state[used_states],
|
||||
3e-3,
|
||||
err_atol=3e-3,
|
||||
)
|
||||
assert torch.isnan(output_storage[..., H * D :]).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_heads", "num_seqs", "lower_bound", "fuse_output_norm"),
|
||||
[
|
||||
(12, 1, -5.0, True),
|
||||
(12, 4, None, False),
|
||||
(24, 4, None, False),
|
||||
(48, 1, -5.0, True),
|
||||
(96, 1, -5.0, True),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_correctness(
|
||||
num_heads: int,
|
||||
num_seqs: int,
|
||||
lower_bound: float | None,
|
||||
fuse_output_norm: bool,
|
||||
):
|
||||
D, W = 128, 4
|
||||
if not is_fused_kda_decode_supported(
|
||||
num_heads,
|
||||
D,
|
||||
W,
|
||||
num_spec=0,
|
||||
input_dtype=torch.bfloat16,
|
||||
conv_state_dtype=torch.bfloat16,
|
||||
):
|
||||
pytest.skip("Fused KDA decode is not supported on this platform")
|
||||
torch.manual_seed(967 + num_heads + num_seqs)
|
||||
dim = num_heads * D
|
||||
slots = num_seqs + 2
|
||||
packed_x_storage = torch.randn(
|
||||
num_seqs, 3 * dim + 17, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
packed_x = packed_x_storage[:, : 3 * dim]
|
||||
weight = 0.1 * torch.randn(3 * dim, W, dtype=torch.float32, device=DEVICE)
|
||||
conv_seed = 0.1 * torch.randn(
|
||||
slots,
|
||||
W - 1,
|
||||
3 * dim,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
).transpose(1, 2)
|
||||
raw_g = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
num_heads,
|
||||
D,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta_storage = torch.randn(
|
||||
1,
|
||||
num_seqs,
|
||||
num_heads + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
raw_beta = raw_beta_storage[:, :, :num_heads]
|
||||
output_gate_storage = torch.randn(
|
||||
num_seqs,
|
||||
dim + 7,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
output_gate = output_gate_storage[:, :dim].view(num_seqs, num_heads, D)
|
||||
norm_weight = torch.randn(D, dtype=torch.float32, device=DEVICE)
|
||||
norm_eps = 1e-5
|
||||
A_log = 0.5 * torch.randn(num_heads, dtype=torch.float32, device=DEVICE)
|
||||
dt_bias = 0.1 * torch.randn(dim, dtype=torch.float32, device=DEVICE)
|
||||
state_indices = torch.arange(
|
||||
num_seqs,
|
||||
0,
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
state_seed = 0.01 * torch.randn(
|
||||
slots,
|
||||
num_heads,
|
||||
D,
|
||||
D,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
conv_ref = conv_seed.clone()
|
||||
state_ref = state_seed.clone()
|
||||
mixed_qkv = causal_conv1d_update(
|
||||
packed_x,
|
||||
conv_ref,
|
||||
weight,
|
||||
activation="silu",
|
||||
conv_state_indices=state_indices,
|
||||
validate_data=True,
|
||||
out=torch.empty_like(packed_x),
|
||||
)
|
||||
expected, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=lower_bound,
|
||||
initial_state=state_ref,
|
||||
state_indices=state_indices,
|
||||
)
|
||||
if fuse_output_norm:
|
||||
expected_float = expected.float()
|
||||
expected = (
|
||||
expected_float
|
||||
* torch.rsqrt(expected_float.square().mean(dim=-1, keepdim=True) + norm_eps)
|
||||
* norm_weight
|
||||
* output_gate.float().sigmoid().unsqueeze(0)
|
||||
).to(expected.dtype)
|
||||
|
||||
conv_slot_elements = 3 * dim * (W - 1)
|
||||
state_slot_elements = num_heads * D * D
|
||||
conv_slot_bytes = conv_slot_elements * torch.bfloat16.itemsize
|
||||
page_bytes = conv_slot_bytes + state_slot_elements * torch.float32.itemsize
|
||||
cache_storage = torch.empty(slots * page_bytes, dtype=torch.uint8, device=DEVICE)
|
||||
conv_actual = torch.as_strided(
|
||||
cache_storage.view(torch.bfloat16),
|
||||
size=(slots, 3 * dim, W - 1),
|
||||
stride=(page_bytes // torch.bfloat16.itemsize, 1, 3 * dim),
|
||||
)
|
||||
state_actual = torch.as_strided(
|
||||
cache_storage.view(torch.float32),
|
||||
size=(slots, num_heads, D, D),
|
||||
stride=(page_bytes // torch.float32.itemsize, D * D, D, 1),
|
||||
storage_offset=conv_slot_bytes // torch.float32.itemsize,
|
||||
)
|
||||
conv_actual.copy_(conv_seed)
|
||||
state_actual.copy_(state_seed)
|
||||
fused_weight = weight.reshape(3, dim, W).transpose(1, 2).contiguous()
|
||||
actual = ops.fused_kda_decode(
|
||||
x=packed_x,
|
||||
weight=fused_weight,
|
||||
bias=None,
|
||||
conv_state=conv_actual,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
state_indices=state_indices,
|
||||
state=state_actual,
|
||||
lower_bound=lower_bound,
|
||||
output_gate=output_gate if fuse_output_norm else None,
|
||||
norm_weight=norm_weight if fuse_output_norm else None,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(conv_actual, conv_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def test_fused_kda_decode_rejects_speculative_conv_state():
|
||||
assert not is_fused_kda_decode_supported(
|
||||
num_heads=12,
|
||||
head_dim=128,
|
||||
conv_width=4,
|
||||
num_spec=2,
|
||||
input_dtype=torch.bfloat16,
|
||||
conv_state_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_flashkda_correctness():
|
||||
if not is_flashkda_supported(128, torch.bfloat16, -3.0):
|
||||
pytest.skip("FlashKDA is not supported on this platform")
|
||||
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
B, T, H, D = 1, 48, 2, 128
|
||||
torch.manual_seed(11)
|
||||
q, k, v, raw_g = [
|
||||
torch.randn(B, T, H, D, dtype=torch.bfloat16, device=DEVICE) for _ in range(4)
|
||||
]
|
||||
beta_logits = torch.randn(B, T, H, dtype=torch.bfloat16, device=DEVICE)
|
||||
A_log = torch.randn(H, dtype=torch.float32, device=DEVICE) * 0.5
|
||||
dt_bias = torch.randn(H, D, dtype=torch.float32, device=DEVICE) * 0.1
|
||||
initial_state = torch.randn(2, H, D, D, dtype=torch.float32, device=DEVICE)
|
||||
cu_seqlens = torch.tensor([0, 17, T], dtype=torch.int32, device=DEVICE)
|
||||
lower_bound = -3.0
|
||||
|
||||
gate = lower_bound * torch.sigmoid(
|
||||
A_log.exp()[None, None, :, None] * (raw_g.float() + dt_bias[None, None, :, :])
|
||||
)
|
||||
beta = beta_logits.float().sigmoid()
|
||||
q_norm = l2norm_fwd(q.contiguous())
|
||||
k_norm = l2norm_fwd(k.contiguous())
|
||||
|
||||
expected_outputs = []
|
||||
expected_states = []
|
||||
for i, (start, end) in enumerate(
|
||||
zip(cu_seqlens[:-1].tolist(), cu_seqlens[1:].tolist())
|
||||
):
|
||||
output, final_state = naive_recurrent_kda(
|
||||
q_norm[:, start:end],
|
||||
k_norm[:, start:end],
|
||||
v[:, start:end],
|
||||
gate[:, start:end],
|
||||
beta[:, start:end],
|
||||
initial_state=initial_state[i].transpose(-1, -2),
|
||||
output_final_state=True,
|
||||
)
|
||||
expected_outputs.append(output)
|
||||
expected_states.append(final_state)
|
||||
expected_out = torch.cat(expected_outputs, dim=1)
|
||||
expected_state = torch.cat(expected_states).transpose(-1, -2).contiguous()
|
||||
|
||||
actual_out = torch.empty_like(v)
|
||||
actual_state = torch.empty_like(initial_state)
|
||||
workspace = torch.empty(
|
||||
torch.ops._flashkda_C.get_workspace_size(T, H, cu_seqlens.numel() - 1),
|
||||
dtype=torch.uint8,
|
||||
device=DEVICE,
|
||||
)
|
||||
torch.ops._flashkda_C.fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
raw_g,
|
||||
beta_logits,
|
||||
D**-0.5,
|
||||
actual_out,
|
||||
workspace,
|
||||
A_log,
|
||||
dt_bias,
|
||||
lower_bound,
|
||||
initial_state,
|
||||
actual_state,
|
||||
cu_seqlens,
|
||||
)
|
||||
|
||||
assert_close("o", expected_out, actual_out, 0.01)
|
||||
assert_close("ht", expected_state, actual_state, 0.01)
|
||||
@@ -0,0 +1,411 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import fields
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.models.kimi_k3.nvidia.kda_metadata import (
|
||||
KimiK3KDAAttentionBackend,
|
||||
KimiK3KDAMetadata,
|
||||
KimiK3KDAMetadataBuilder,
|
||||
_mamba_get_block_table_tensor,
|
||||
stage_spec_decode_metadata,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
NULL_BLOCK_ID,
|
||||
mamba_get_block_table_tensor,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
PRUNED_METADATA_FIELDS = {
|
||||
"chunk_indices",
|
||||
"chunk_offsets",
|
||||
"prefill_query_start_loc",
|
||||
"prefill_state_indices",
|
||||
"prefill_has_initial_state",
|
||||
"spec_sequence_masks",
|
||||
}
|
||||
|
||||
|
||||
def _assert_matches_shared_gdn(reference, actual: KimiK3KDAMetadata):
|
||||
for field in fields(KimiK3KDAMetadata):
|
||||
actual_value = getattr(actual, field.name)
|
||||
expected_value = getattr(reference, field.name)
|
||||
if field.name in PRUNED_METADATA_FIELDS:
|
||||
assert actual_value is None
|
||||
continue
|
||||
if (
|
||||
field.name in {"spec_token_indx", "non_spec_token_indx"}
|
||||
and actual.num_spec_decodes > 0
|
||||
and actual.num_prefills == 0
|
||||
and actual.num_decodes == 0
|
||||
):
|
||||
assert actual_value is None
|
||||
continue
|
||||
if isinstance(actual_value, torch.Tensor):
|
||||
torch.testing.assert_close(actual_value, expected_value)
|
||||
elif field.name == "nums_dict":
|
||||
assert (actual_value is None) == (expected_value is None)
|
||||
if actual_value is not None:
|
||||
assert actual_value[8]["tot"] == expected_value[8]["tot"]
|
||||
torch.testing.assert_close(
|
||||
actual_value[8]["nums"], expected_value[8]["nums"]
|
||||
)
|
||||
else:
|
||||
assert actual_value == expected_value
|
||||
|
||||
|
||||
def _make_builder(
|
||||
builder_cls: type[AttentionMetadataBuilder],
|
||||
num_speculative_tokens: int,
|
||||
full_cuda_graph: bool,
|
||||
device: torch.device = DEVICE,
|
||||
mamba_cache_mode: str = "none",
|
||||
) -> AttentionMetadataBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
if num_speculative_tokens:
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = (
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
return builder_cls(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=num_speculative_tokens,
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"batch",
|
||||
"num_decode_draft_tokens",
|
||||
"num_speculative_tokens",
|
||||
"full_cuda_graph",
|
||||
"is_prefilling",
|
||||
),
|
||||
[
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[50, 30], query_lens=[3, 3]),
|
||||
[2, 2],
|
||||
2,
|
||||
False,
|
||||
[False, False],
|
||||
id="pure-spec-decode",
|
||||
),
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[100, 65, 20], query_lens=[50, 1, 3]),
|
||||
[-1, -1, 2],
|
||||
2,
|
||||
False,
|
||||
[True, False, False],
|
||||
id="mixed-prefill-and-spec-decode",
|
||||
),
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
|
||||
None,
|
||||
0,
|
||||
False,
|
||||
[False, False],
|
||||
id="regular-decode",
|
||||
),
|
||||
pytest.param(
|
||||
BatchSpec(seq_lens=[40, 30], query_lens=[1, 1]),
|
||||
[0, 0],
|
||||
2,
|
||||
False,
|
||||
[False, False],
|
||||
id="no-scheduled-draft-tokens",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_kimi_k3_kda_metadata_matches_shared_gdn(
|
||||
batch: BatchSpec,
|
||||
num_decode_draft_tokens: list[int] | None,
|
||||
num_speculative_tokens: int,
|
||||
full_cuda_graph: bool,
|
||||
is_prefilling: list[bool],
|
||||
):
|
||||
kwargs: dict[str, torch.Tensor] = {}
|
||||
if num_decode_draft_tokens is not None:
|
||||
kwargs = {
|
||||
"num_decode_draft_tokens_cpu": torch.tensor(
|
||||
num_decode_draft_tokens, dtype=torch.int32
|
||||
),
|
||||
"num_accepted_tokens": torch.ones(
|
||||
batch.batch_size, dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
}
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor(is_prefilling, dtype=torch.bool))
|
||||
reference = _make_builder(
|
||||
GDNAttentionMetadataBuilder,
|
||||
num_speculative_tokens,
|
||||
full_cuda_graph,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
**kwargs,
|
||||
)
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens,
|
||||
full_cuda_graph,
|
||||
).build(0, common_attn_metadata, **kwargs)
|
||||
|
||||
assert isinstance(actual, KimiK3KDAMetadata)
|
||||
_assert_matches_shared_gdn(reference, actual)
|
||||
|
||||
|
||||
def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():
|
||||
batch = BatchSpec(seq_lens=[100, 65, 20], query_lens=[1, 1, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([False, False, False]))
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
# The K3 layer dispatches the non-spec subgroup to packed decode whenever
|
||||
# it contains no prefill request.
|
||||
assert actual.num_decodes == 2
|
||||
assert actual.num_decode_tokens == 2
|
||||
assert actual.num_prefills == 0
|
||||
assert actual.num_prefill_tokens == 0
|
||||
assert actual.has_initial_state is None
|
||||
assert actual.nums_dict is None
|
||||
assert actual.non_spec_query_start_loc is None
|
||||
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0, 1]))
|
||||
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([2, 3, 4]))
|
||||
torch.testing.assert_close(
|
||||
actual.spec_query_start_loc,
|
||||
torch.tensor([0, 3], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_regular_and_spec_decode_excludes_request_padding():
|
||||
batch = BatchSpec(seq_lens=[16, 65, 20], query_lens=[0, 1, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([False, False, False]))
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.tensor([-1, -1, 2], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(3, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
assert actual.num_decodes == 1
|
||||
assert actual.non_spec_state_indices_tensor is not None
|
||||
assert actual.non_spec_state_indices_tensor.shape == (1,)
|
||||
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0]))
|
||||
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seq_len", "expected_has_initial_state"),
|
||||
[
|
||||
pytest.param(1, False, id="first-token-prefill"),
|
||||
pytest.param(65, True, id="final-one-token-prefill-chunk"),
|
||||
],
|
||||
)
|
||||
def test_mixed_one_token_prefill_and_spec_decode_uses_prefill_metadata(
|
||||
seq_len: int,
|
||||
expected_has_initial_state: bool,
|
||||
):
|
||||
batch = BatchSpec(seq_lens=[seq_len, 20], query_lens=[1, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([True, False]))
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
).build(
|
||||
0,
|
||||
common_attn_metadata,
|
||||
num_decode_draft_tokens_cpu=torch.tensor([-1, 2], dtype=torch.int32),
|
||||
num_accepted_tokens=torch.ones(2, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
assert actual.num_prefills == 1
|
||||
assert actual.num_prefill_tokens == 1
|
||||
assert actual.num_decodes == 0
|
||||
assert actual.num_decode_tokens == 0
|
||||
assert actual.has_initial_state is not None
|
||||
assert actual.has_initial_state.tolist() == [expected_has_initial_state]
|
||||
assert actual.non_spec_query_start_loc is not None
|
||||
torch.testing.assert_close(
|
||||
actual.non_spec_query_start_loc,
|
||||
torch.tensor([0, 1], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_kimi_k3_kda_cudagraph_capture_matches_shared_gdn():
|
||||
device = torch.device("cuda")
|
||||
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, device
|
||||
).replace(is_prefilling=torch.tensor([False, False]))
|
||||
reference = _make_builder(
|
||||
GDNAttentionMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=True,
|
||||
device=device,
|
||||
).build_for_cudagraph_capture(common_attn_metadata)
|
||||
actual = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=True,
|
||||
device=device,
|
||||
).build_for_cudagraph_capture(common_attn_metadata)
|
||||
|
||||
assert isinstance(actual, KimiK3KDAMetadata)
|
||||
_assert_matches_shared_gdn(reference, actual)
|
||||
|
||||
|
||||
def test_kimi_k3_kda_backend_uses_private_metadata_builder():
|
||||
assert KimiK3KDAAttentionBackend.get_builder_cls() is KimiK3KDAMetadataBuilder
|
||||
assert KimiK3KDAAttentionBackend.is_ssm()
|
||||
assert issubclass(KimiK3KDAAttentionBackend, GDNAttentionBackend)
|
||||
assert issubclass(KimiK3KDAMetadata, GDNAttentionMetadata)
|
||||
assert issubclass(KimiK3KDAMetadataBuilder, GDNAttentionMetadataBuilder)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_stage_spec_decode_metadata_matches_pytorch():
|
||||
device = torch.device("cuda")
|
||||
num_spec_decodes = 33
|
||||
batch_size = 65
|
||||
num_state_slots = 3
|
||||
state_indices = torch.arange(
|
||||
num_spec_decodes * 32,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).reshape(num_spec_decodes, 32)[:, :num_state_slots]
|
||||
query_start_loc = (
|
||||
torch.arange(num_spec_decodes + 1, dtype=torch.int32, device=device)
|
||||
* num_state_slots
|
||||
)
|
||||
num_accepted_tokens = (
|
||||
torch.arange(num_spec_decodes, dtype=torch.int32, device=device)
|
||||
% num_state_slots
|
||||
+ 1
|
||||
)
|
||||
|
||||
staged_state_indices = torch.empty(
|
||||
(batch_size, num_state_slots), dtype=torch.int32, device=device
|
||||
)
|
||||
staged_query_start_loc = torch.empty(
|
||||
batch_size + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
staged_num_accepted_tokens = torch.empty(
|
||||
batch_size, dtype=torch.int32, device=device
|
||||
)
|
||||
stage_spec_decode_metadata(
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
num_accepted_tokens,
|
||||
staged_state_indices,
|
||||
staged_query_start_loc,
|
||||
staged_num_accepted_tokens,
|
||||
num_spec_decodes=num_spec_decodes,
|
||||
)
|
||||
|
||||
expected_state_indices = torch.full_like(staged_state_indices, NULL_BLOCK_ID)
|
||||
expected_state_indices[:num_spec_decodes] = state_indices
|
||||
expected_query_start_loc = torch.full(
|
||||
(batch_size + 1,),
|
||||
query_start_loc[-1],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
expected_query_start_loc[: num_spec_decodes + 1] = query_start_loc
|
||||
expected_num_accepted_tokens = torch.ones(
|
||||
batch_size, dtype=torch.int32, device=device
|
||||
)
|
||||
expected_num_accepted_tokens[:num_spec_decodes] = num_accepted_tokens
|
||||
|
||||
torch.testing.assert_close(staged_state_indices, expected_state_indices)
|
||||
torch.testing.assert_close(staged_query_start_loc, expected_query_start_loc)
|
||||
torch.testing.assert_close(staged_num_accepted_tokens, expected_num_accepted_tokens)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_aligned_block_table_matches_shared_gdn():
|
||||
device = torch.device("cuda")
|
||||
seq_lens = torch.tensor(
|
||||
[0, 1, 15, 16, 17, 31, 32, 33, 511, 512, 513],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).repeat(6)[:65]
|
||||
block_table_storage = torch.arange(
|
||||
seq_lens.numel() * 128,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).reshape(seq_lens.numel(), 128)
|
||||
block_table = block_table_storage[:, ::2]
|
||||
kv_cache_spec = MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
|
||||
expected = mamba_get_block_table_tensor(
|
||||
block_table,
|
||||
seq_lens,
|
||||
kv_cache_spec,
|
||||
"align",
|
||||
)
|
||||
actual = _mamba_get_block_table_tensor(
|
||||
block_table,
|
||||
seq_lens,
|
||||
kv_cache_spec,
|
||||
"align",
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import (
|
||||
init_test_distributed_environment,
|
||||
multi_gpu_test,
|
||||
multi_process_parallel,
|
||||
)
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import KimiK3LatentMoETailOp
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
LATENT_SIZE = 3584
|
||||
EPS = 0.1
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _test_latent_moe_tail_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
) -> None:
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rms_weight = 1 + 0.1 * torch.randn(
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
up_weight = (
|
||||
torch.randn(
|
||||
HIDDEN_SIZE,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
/ LATENT_SIZE**0.5
|
||||
)
|
||||
|
||||
group = get_tp_group().device_group
|
||||
op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
latent_size=LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
rms_eps=EPS,
|
||||
)
|
||||
cutedsl_warmup()
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
|
||||
torch.manual_seed(100 * iteration + rank + 1)
|
||||
routed_output = torch.randn(
|
||||
num_tokens,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).mul_(0.01)
|
||||
shared_output = torch.randn(
|
||||
num_tokens,
|
||||
HIDDEN_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
routed_reference = routed_output.clone()
|
||||
shared_reference = shared_output.clone()
|
||||
dist.all_reduce(routed_reference, group=group)
|
||||
dist.all_reduce(shared_reference, group=group)
|
||||
expected = F.linear(
|
||||
F.rms_norm(
|
||||
routed_reference,
|
||||
(LATENT_SIZE,),
|
||||
rms_weight,
|
||||
EPS,
|
||||
),
|
||||
up_weight,
|
||||
)
|
||||
expected.add_(shared_reference)
|
||||
|
||||
actual = op(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_output = op(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(graph_output, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def _run_latent_moe_tail_test(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
) -> None:
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("K3 latent-MoE tail fusion requires SM100")
|
||||
multi_process_parallel(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
1,
|
||||
_test_latent_moe_tail_worker,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_latent_moe_tail_tp8_matches_native_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_run_latent_moe_tail_test(monkeypatch, 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=16)
|
||||
def test_latent_moe_tail_tp16_matches_native_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_run_latent_moe_tail_test(monkeypatch, 16)
|
||||
@@ -0,0 +1,343 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.models.kimi_k3.nvidia import model as kimi_model
|
||||
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
|
||||
from vllm.models.kimi_k3.nvidia.ops import sequence_parallel as sp_ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class _IdentityNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
|
||||
self.variance_epsilon = 1e-5
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
):
|
||||
if residual is None:
|
||||
return hidden_states
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class _RecordingMoE(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.num_tokens = 0
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
self.num_tokens = hidden_states.shape[0]
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _Projection(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.ones(1, hidden_size),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
|
||||
class _SequenceParallelMTPBlock:
|
||||
use_sequence_parallel = True
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
):
|
||||
assert residual is None
|
||||
return hidden_states * 2, None, hidden_states * 3
|
||||
|
||||
|
||||
def _mock_sequence_parallel_collectives(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_reduce_scatter",
|
||||
lambda tensor: tensor.chunk(2, dim=0)[0],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "is_padding", "tp_rank", "expected"),
|
||||
[
|
||||
(1, None, 0, [False]),
|
||||
(1, None, 1, [True]),
|
||||
(5, None, 2, [False, True]),
|
||||
(5, None, 3, [True, True]),
|
||||
(5, [False, True, False, False, False], 0, [False, True]),
|
||||
],
|
||||
)
|
||||
def test_sp_padding_mask_marks_added_rows(
|
||||
monkeypatch,
|
||||
num_tokens: int,
|
||||
is_padding: list[bool] | None,
|
||||
tp_rank: int,
|
||||
expected: list[bool],
|
||||
):
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_world_size", lambda: 4)
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
|
||||
hidden_states = torch.empty(num_tokens, 2)
|
||||
padding = torch.tensor(is_padding) if is_padding is not None else None
|
||||
actual = sp_ops.sp_padding_mask(padding, hidden_states)
|
||||
|
||||
torch.testing.assert_close(actual, torch.tensor(expected))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data_parallel_size", "expected"),
|
||||
[
|
||||
(1, False),
|
||||
(2, True),
|
||||
],
|
||||
)
|
||||
def test_moe_sequence_parallel_requires_data_parallel(
|
||||
monkeypatch,
|
||||
data_parallel_size: int,
|
||||
expected: bool,
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=data_parallel_size,
|
||||
enable_expert_parallel=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
)
|
||||
|
||||
assert parallel_config.use_sequence_parallel_moe is expected
|
||||
|
||||
|
||||
def test_kimi_decoder_layer_keeps_moe_states_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(kimi_model.KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_attn_res = False
|
||||
layer.use_sequence_parallel = True
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.mlp = _RecordingMoE()
|
||||
layer._run_self_attn = MethodType(
|
||||
lambda self, positions, hidden_states: hidden_states,
|
||||
layer,
|
||||
)
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch)
|
||||
|
||||
positions = torch.arange(3)
|
||||
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
hidden_states = kimi_model.sp_shard(full_hidden_states)
|
||||
hidden_states, prefix_sum, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
|
||||
assert prefix_sum is None
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
hidden_states, prefix_sum, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
assert prefix_sum is None
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_kimi_attn_residual_states_stay_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(kimi_model.KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_attn_res = True
|
||||
layer.use_sequence_parallel = True
|
||||
layer.prev_valid_blocks = 0
|
||||
layer.block_write_idx = 0
|
||||
layer.is_block_write_layer = False
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.self_attention_res_norm = _IdentityNorm()
|
||||
layer.mlp_res_norm = _IdentityNorm()
|
||||
layer.self_attention_res_proj = _Projection()
|
||||
layer.mlp_res_proj = _Projection()
|
||||
layer.mlp = _RecordingMoE()
|
||||
layer._run_self_attn = MethodType(
|
||||
lambda self, positions, hidden_states: hidden_states,
|
||||
layer,
|
||||
)
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"attn_res",
|
||||
lambda prefix_sum, hidden_states, *args, **kwargs: (
|
||||
prefix_sum if hidden_states is None else prefix_sum + hidden_states
|
||||
),
|
||||
)
|
||||
|
||||
prefix_sum = kimi_model.sp_shard(torch.arange(6, dtype=torch.float32).view(3, 2))
|
||||
block_residual = torch.zeros(2, 1, 2)
|
||||
hidden_states, prefix_sum, block_residual = layer(
|
||||
positions=torch.arange(3),
|
||||
hidden_states=None,
|
||||
prefix_sum=prefix_sum,
|
||||
residual=block_residual,
|
||||
)
|
||||
|
||||
assert hidden_states.shape == prefix_sum.shape == (2, 2)
|
||||
assert block_residual.shape == (2, 1, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
|
||||
layer = object.__new__(kimi_mtp.KimiK3MultiTokenPredictorLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.enorm = _IdentityNorm()
|
||||
layer.hnorm = _IdentityNorm()
|
||||
layer.eh_proj = nn.Identity()
|
||||
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
|
||||
|
||||
final_norm = Mock(side_effect=lambda hidden_states: hidden_states + 1)
|
||||
object.__setattr__(
|
||||
layer,
|
||||
"shared_head",
|
||||
SimpleNamespace(norm=final_norm),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"fused_mtp_input",
|
||||
lambda positions, inputs_embeds, *args: inputs_embeds,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
logits_hidden_states, hidden_states = layer(
|
||||
input_ids=torch.zeros(3, dtype=torch.long),
|
||||
positions=torch.arange(3),
|
||||
previous_hidden_states=torch.zeros_like(inputs_embeds),
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
|
||||
expected_hidden_states = torch.cat(
|
||||
[sharded_states * 5, sharded_states * 5],
|
||||
dim=0,
|
||||
)[:3]
|
||||
torch.testing.assert_close(hidden_states, expected_hidden_states)
|
||||
torch.testing.assert_close(logits_hidden_states, expected_hidden_states + 1)
|
||||
final_norm.assert_called_once()
|
||||
torch.testing.assert_close(final_norm.call_args.args[0], expected_hidden_states)
|
||||
|
||||
|
||||
def test_sp_all_gather_uses_custom_kernel(monkeypatch):
|
||||
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
expected = torch.cat([hidden_states, hidden_states])
|
||||
custom_all_gather = Mock(return_value=expected)
|
||||
device_communicator = SimpleNamespace(
|
||||
custom_all_gather=custom_all_gather,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_communicator),
|
||||
)
|
||||
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", fallback)
|
||||
|
||||
output = sp_ops.sp_all_gather(hidden_states)
|
||||
|
||||
torch.testing.assert_close(output, expected)
|
||||
custom_all_gather.assert_called_once_with(hidden_states)
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
|
||||
hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
expected = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
custom_reduce_scatter = Mock(return_value=expected)
|
||||
device_communicator = SimpleNamespace(
|
||||
custom_reduce_scatter=custom_reduce_scatter,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_communicator),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_reduce_scatter", fallback)
|
||||
|
||||
output = sp_ops.sp_reduce_scatter(hidden_states)
|
||||
|
||||
torch.testing.assert_close(output, expected)
|
||||
padded = custom_reduce_scatter.call_args.args[0]
|
||||
assert padded.shape == (4, 2)
|
||||
torch.testing.assert_close(padded[:3], hidden_states)
|
||||
torch.testing.assert_close(padded[3], torch.zeros(2))
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
|
||||
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
all_gather = Mock(return_value=hidden_states)
|
||||
reduce_scatter = Mock(return_value=hidden_states)
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", all_gather)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"tensor_model_parallel_reduce_scatter",
|
||||
reduce_scatter,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(sp_ops.sp_all_gather(hidden_states), hidden_states)
|
||||
torch.testing.assert_close(sp_ops.sp_reduce_scatter(hidden_states), hidden_states)
|
||||
all_gather.assert_called_once_with(hidden_states, 0)
|
||||
reduce_scatter.assert_called_once_with(hidden_states, 0)
|
||||
@@ -1242,7 +1242,7 @@ def test_custom_inputs_models(
|
||||
create_new_process_for_each_test=True,
|
||||
),
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_single_image_models_heavy(
|
||||
tmp_path: PosixPath,
|
||||
model_type: str,
|
||||
|
||||
Reference in New Issue
Block a user