[Bugfix][Kimi-K3] Fix KDA projection overlap on Hopper (#55426)

Signed-off-by: Chengcheng Pei <5881383+chengchengpei@users.noreply.github.com>
Co-authored-by: Chengcheng Pei <5881383+chengchengpei@users.noreply.github.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
chengchengpei
2026-09-11 19:07:30 +00:00
committed by GitHub
co-authored by Tyler Michael Smith
parent 00e5cda926
commit 0c1e89ceb9
3 changed files with 238 additions and 42 deletions
+101 -1
View File
@@ -269,6 +269,70 @@ def test_kda_overlap_configs_match_measured_table() -> None:
}
@pytest.mark.parametrize(
"compute_capability,supported",
[((9, 0), False), ((10, 0), True), ((10, 3), True), ((12, 0), True)],
)
def test_kda_mixed_precision_bf16_fma_capability(
monkeypatch: pytest.MonkeyPatch,
compute_capability: tuple[int, int],
supported: bool,
) -> None:
from vllm.models.kimi_k3.nvidia.ops.cute_dsl import kda_skinny_gemm
capability = compute_capability[0] * 10 + compute_capability[1]
monkeypatch.setattr(
kda_skinny_gemm.current_platform,
"has_device_capability",
lambda target, device_id=0: capability >= target,
)
assert kda_skinny_gemm._has_mixed_precision_bf16_fma() is supported
@pytest.mark.parametrize("num_tokens", [2, 8, 14])
def test_kda_skinny_gemms_sm90_cuda_graph(num_tokens: int) -> None:
"""Hopper must compile and capture the pre-SM100 FMA specialization."""
_require_capability_and_cute((9, 0))
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.kda_skinny_gemm import (
KdaSkinnyGemm,
)
torch.manual_seed(43 + num_tokens)
hidden_states = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
f_ab_weight = torch.randn(144, 7168, dtype=torch.bfloat16, device="cuda")
f_b_weight = torch.randn(1536, 128, dtype=torch.bfloat16, device="cuda")
gemm = KdaSkinnyGemm()
gemm.run_k(gemm.run_n(hidden_states, f_ab_weight), f_b_weight)
torch.accelerator.synchronize()
capture_stream = torch.cuda.Stream()
capture_stream.wait_stream(torch.cuda.current_stream())
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=capture_stream):
projected_fab = gemm.run_n(hidden_states, f_ab_weight)
projected_fb = gemm.run_k(projected_fab, f_b_weight)
torch.cuda.current_stream().wait_stream(capture_stream)
graph.replay()
torch.accelerator.synchronize()
expected_fab = torch.nn.functional.linear(
hidden_states.float(), f_ab_weight.float()
)
expected_fb = torch.nn.functional.linear(
expected_fab[:, :128].to(torch.bfloat16).float(), f_b_weight.float()
)
for actual, expected in (
(projected_fab, expected_fab),
(projected_fb, expected_fb),
):
cosine = torch.nn.functional.cosine_similarity(
actual.float().flatten(), expected.flatten(), dim=0
).item()
assert cosine > 0.999
@pytest.mark.parametrize("tp_size", [1, 2, 4, 16])
def test_kda_projection_overlap_is_tp8_only(tp_size: int) -> None:
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
@@ -282,8 +346,34 @@ def test_kda_projection_overlap_is_tp8_only(tp_size: int) -> None:
assert kda._projection_overlap_max_tokens == 0
@pytest.mark.parametrize(
("capability", "expected"),
[
((9, 0), None),
((10, 0), "cute-dsl"),
((10, 3), "cute-dsl"),
((10, 7), None),
((12, 0), None),
],
)
def test_kda_qkvg_flashinfer_backend(
monkeypatch: pytest.MonkeyPatch,
capability: tuple[int, int],
expected: str | None,
) -> None:
monkeypatch.setattr(
current_platform,
"is_device_capability",
lambda candidate: candidate == capability,
)
assert k3_gemm._kda_qkvg_flashinfer_backend() == expected
@pytest.mark.parametrize("backend", ["cute-dsl", None])
def test_kda_qkvg_autotune_enables_full_overlap(
monkeypatch: pytest.MonkeyPatch,
backend: str | None,
) -> None:
flashinfer_gemm = pytest.importorskip("flashinfer.gemm")
@@ -304,10 +394,20 @@ def test_kda_qkvg_autotune_enables_full_overlap(
return torch.empty(a.shape[0], b.shape[1], dtype=a.dtype)
monkeypatch.setattr(flashinfer_gemm, "mm_bf16", fake_mm_bf16)
monkeypatch.setattr(
k3_gemm,
"_kda_qkvg_flashinfer_backend",
lambda: backend,
)
k3_gemm.autotune_kda_qkvg(kda)
assert calls == [(torch.Size([14, 8]), torch.Size([8, 6144]), True, "cute-dsl")]
expected_calls = []
if backend is not None:
expected_calls.append(
(torch.Size([14, 8]), torch.Size([8, 6144]), True, "cute-dsl")
)
assert calls == expected_calls
assert (
kda._projection_overlap_max_tokens == k3_gemm.KDA_PROJECTION_OVERLAP_MAX_TOKENS
)