[Bugfix][Kimi-K3] Fix KDA projection overlap on Hopper (#55426)
Signed-off-by: Chengcheng Pei <5881383+chengchengpei@users.noreply.github.com> Co-authored-by: Chengcheng Pei <5881383+chengchengpei@users.noreply.github.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
00e5cda926
commit
0c1e89ceb9
@@ -269,6 +269,70 @@ def test_kda_overlap_configs_match_measured_table() -> None:
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"compute_capability,supported",
|
||||
[((9, 0), False), ((10, 0), True), ((10, 3), True), ((12, 0), True)],
|
||||
)
|
||||
def test_kda_mixed_precision_bf16_fma_capability(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
compute_capability: tuple[int, int],
|
||||
supported: bool,
|
||||
) -> None:
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl import kda_skinny_gemm
|
||||
|
||||
capability = compute_capability[0] * 10 + compute_capability[1]
|
||||
monkeypatch.setattr(
|
||||
kda_skinny_gemm.current_platform,
|
||||
"has_device_capability",
|
||||
lambda target, device_id=0: capability >= target,
|
||||
)
|
||||
|
||||
assert kda_skinny_gemm._has_mixed_precision_bf16_fma() is supported
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [2, 8, 14])
|
||||
def test_kda_skinny_gemms_sm90_cuda_graph(num_tokens: int) -> None:
|
||||
"""Hopper must compile and capture the pre-SM100 FMA specialization."""
|
||||
_require_capability_and_cute((9, 0))
|
||||
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.kda_skinny_gemm import (
|
||||
KdaSkinnyGemm,
|
||||
)
|
||||
|
||||
torch.manual_seed(43 + num_tokens)
|
||||
hidden_states = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
|
||||
f_ab_weight = torch.randn(144, 7168, dtype=torch.bfloat16, device="cuda")
|
||||
f_b_weight = torch.randn(1536, 128, dtype=torch.bfloat16, device="cuda")
|
||||
gemm = KdaSkinnyGemm()
|
||||
|
||||
gemm.run_k(gemm.run_n(hidden_states, f_ab_weight), f_b_weight)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
capture_stream = torch.cuda.Stream()
|
||||
capture_stream.wait_stream(torch.cuda.current_stream())
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=capture_stream):
|
||||
projected_fab = gemm.run_n(hidden_states, f_ab_weight)
|
||||
projected_fb = gemm.run_k(projected_fab, f_b_weight)
|
||||
torch.cuda.current_stream().wait_stream(capture_stream)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected_fab = torch.nn.functional.linear(
|
||||
hidden_states.float(), f_ab_weight.float()
|
||||
)
|
||||
expected_fb = torch.nn.functional.linear(
|
||||
expected_fab[:, :128].to(torch.bfloat16).float(), f_b_weight.float()
|
||||
)
|
||||
for actual, expected in (
|
||||
(projected_fab, expected_fab),
|
||||
(projected_fb, expected_fb),
|
||||
):
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
actual.float().flatten(), expected.flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4, 16])
|
||||
def test_kda_projection_overlap_is_tp8_only(tp_size: int) -> None:
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
@@ -282,8 +346,34 @@ def test_kda_projection_overlap_is_tp8_only(tp_size: int) -> None:
|
||||
assert kda._projection_overlap_max_tokens == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("capability", "expected"),
|
||||
[
|
||||
((9, 0), None),
|
||||
((10, 0), "cute-dsl"),
|
||||
((10, 3), "cute-dsl"),
|
||||
((10, 7), None),
|
||||
((12, 0), None),
|
||||
],
|
||||
)
|
||||
def test_kda_qkvg_flashinfer_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capability: tuple[int, int],
|
||||
expected: str | None,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability",
|
||||
lambda candidate: candidate == capability,
|
||||
)
|
||||
|
||||
assert k3_gemm._kda_qkvg_flashinfer_backend() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["cute-dsl", None])
|
||||
def test_kda_qkvg_autotune_enables_full_overlap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
backend: str | None,
|
||||
) -> None:
|
||||
flashinfer_gemm = pytest.importorskip("flashinfer.gemm")
|
||||
|
||||
@@ -304,10 +394,20 @@ def test_kda_qkvg_autotune_enables_full_overlap(
|
||||
return torch.empty(a.shape[0], b.shape[1], dtype=a.dtype)
|
||||
|
||||
monkeypatch.setattr(flashinfer_gemm, "mm_bf16", fake_mm_bf16)
|
||||
monkeypatch.setattr(
|
||||
k3_gemm,
|
||||
"_kda_qkvg_flashinfer_backend",
|
||||
lambda: backend,
|
||||
)
|
||||
|
||||
k3_gemm.autotune_kda_qkvg(kda)
|
||||
|
||||
assert calls == [(torch.Size([14, 8]), torch.Size([8, 6144]), True, "cute-dsl")]
|
||||
expected_calls = []
|
||||
if backend is not None:
|
||||
expected_calls.append(
|
||||
(torch.Size([14, 8]), torch.Size([8, 6144]), True, "cute-dsl")
|
||||
)
|
||||
assert calls == expected_calls
|
||||
assert (
|
||||
kda._projection_overlap_max_tokens == k3_gemm.KDA_PROJECTION_OVERLAP_MAX_TOKENS
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user