[Kimi-K3] Overlap low-M TP8 KDA projections (#54697)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Yongye Zhu
2026-09-02 03:35:28 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 12b9573c98
commit 3ba9907a1d
7 changed files with 1092 additions and 12 deletions
+118
View File
@@ -253,6 +253,65 @@ def test_residual_cute_configs_match_measured_table() -> None:
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
def test_kda_overlap_configs_match_measured_table() -> None:
assert k3_gemm.KDA_PROJECTION_OVERLAP_MAX_TOKENS == 14
assert k3_gemm.KDA_SKINNY_N_MAX_TOKENS == 14
assert k3_gemm.KDA_SKINNY_K_MAX_TOKENS == 14
assert _config_tuple(k3_gemm.KDA_M1_QKVG_CONFIG) == (64, 4, 2, 8)
assert _config_tuple(k3_gemm.KDA_M1_FAB_CONFIG) == (224, 1, 2, 8)
assert {
num_tokens: _config_tuple(config)
for num_tokens, config in k3_gemm.KDA_QKVG_CONFIGS.items()
} == {
1: (64, 4, 2, 8),
2: (64, 3, 2, 8),
}
@pytest.mark.parametrize("tp_size", [1, 2, 4, 16])
def test_kda_projection_overlap_is_tp8_only(tp_size: int) -> None:
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
kda = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
nn.Module.__init__(kda)
kda.tp_size = tp_size
kda._projection_overlap_max_tokens = 0
assert not k3_gemm._enable_kda_projection_overlap(kda)
assert kda._projection_overlap_max_tokens == 0
def test_kda_qkvg_autotune_enables_full_overlap(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import flashinfer.gemm
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
kda = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
nn.Module.__init__(kda)
kda._projection_overlap_max_tokens = max(k3_gemm.KDA_QKVG_CONFIGS)
kda.in_proj_qkvgfab = nn.Module()
kda.in_proj_qkvgfab.weight = nn.Parameter(
torch.empty(6144, 8, dtype=torch.bfloat16),
requires_grad=False,
)
calls = []
def fake_mm_bf16(a, b, *, pdl, backend):
calls.append((a.shape, b.shape, pdl, backend))
return torch.empty(a.shape[0], b.shape[1], dtype=a.dtype)
monkeypatch.setattr(flashinfer.gemm, "mm_bf16", fake_mm_bf16)
k3_gemm.autotune_kda_qkvg(kda)
assert calls == [(torch.Size([14, 8]), torch.Size([8, 6144]), True, "cute-dsl")]
assert (
kda._projection_overlap_max_tokens == k3_gemm.KDA_PROJECTION_OVERLAP_MAX_TOKENS
)
def test_glm52_projection_plans_are_separate() -> None:
qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION
q_b = glm52_gemm.GLM52_Q_B_PROJECTION
@@ -812,6 +871,65 @@ def test_cute_selected_shapes(
assert cosine > 0.999
@pytest.mark.parametrize("num_tokens", [1, 2, 3, 4, 8, 9, 10, 11, 12, 13, 14])
def test_kda_projection_overlap_cuda_graph(num_tokens: int) -> None:
"""The TP8 split must preserve projection results across both streams."""
_require_sm103_and_cute()
_require_sm103_and_dsv3()
torch.manual_seed(43)
hidden_states = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
packed_weight = torch.randn(6288, 7168, dtype=torch.bfloat16, device="cuda")
f_b_weight = torch.randn(1536, 128, dtype=torch.bfloat16, device="cuda")
aux_stream = torch.cuda.Stream()
events = (torch.cuda.Event(), torch.cuda.Event())
k3_gemm.run_kda_projection_overlap(
hidden_states,
packed_weight,
f_b_weight,
aux_stream,
events,
)
torch.accelerator.synchronize()
capture_stream = torch.cuda.Stream()
capture_stream.wait_stream(torch.cuda.current_stream())
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=capture_stream):
projected_qkvg, g1, beta = k3_gemm.run_kda_projection_overlap(
hidden_states,
packed_weight,
f_b_weight,
aux_stream,
events,
)
torch.cuda.current_stream().wait_stream(capture_stream)
graph.replay()
torch.accelerator.synchronize()
assert beta.stride() == ((140 if num_tokens == 1 else 144), 1)
expected_qkvg = torch.nn.functional.linear(
hidden_states.float(), packed_weight[:6144].float()
)
expected_fab = torch.nn.functional.linear(
hidden_states.float(), packed_weight[6144:6284].float()
)
expected_g1 = torch.nn.functional.linear(
expected_fab[:, :128].to(torch.bfloat16).float(),
f_b_weight.float(),
)
for actual, expected in (
(projected_qkvg, expected_qkvg),
(g1, expected_g1),
(beta, expected_fab[:, 128:]),
):
cosine = torch.nn.functional.cosine_similarity(
actual.float().flatten(), expected.flatten(), dim=0
).item()
assert cosine > 0.999
def _dsv3_probe_tokens(tokens: frozenset[int]) -> set[int]:
"""Extremes, plus both sides of the kernel's num_tokens<=8 tile_n branch."""
if not tokens: