[Kimi K3][Kernel] Enable low-latency decode GEMM dispatch on SM100 (#53534)
Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for BF16 skinny GEMMs and the Kimi-K3 SM103 selector."""
|
||||
"""Tests for BF16 skinny GEMMs and the Kimi-K3 SM103/SM100 selectors."""
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -182,6 +182,7 @@ def test_every_dsv3_routed_shape_is_instantiated() -> None:
|
||||
|
||||
specs = [
|
||||
*KIMI_K3_PROJECTIONS.values(),
|
||||
*k3_gemm.KIMI_K3_PROJECTIONS_SM100.values(),
|
||||
glm52_gemm.GLM52_QKV_A_PROJECTION,
|
||||
glm52_gemm.GLM52_Q_B_PROJECTION,
|
||||
]
|
||||
@@ -404,6 +405,92 @@ def test_build_plan_matches_selector() -> None:
|
||||
assert plan[num_tokens][0] == backend
|
||||
|
||||
|
||||
# Keyed by local (N, K): (cute token counts, dsv3 token counts). Mirrors
|
||||
# KIMI_K3_PROJECTIONS_SM100, measured on B200; intentionally different from
|
||||
# EXPECTED_SELECTIONS (e.g. 3584x7168 routes dsv3 at M2..8 on SM103 but only
|
||||
# wins with CuTe at M1..3 on SM100).
|
||||
EXPECTED_SELECTIONS_SM100 = {
|
||||
(1536, 128): (set(), {1, 16}),
|
||||
(3072, 128): (set(), {8}),
|
||||
(1536, 7168): ({1, 2}, {4, 8}),
|
||||
(3072, 7168): ({1, 2}, set()),
|
||||
(2112, 7168): ({1, 2}, {4, 16}),
|
||||
(2304, 1536): (set(), set(range(1, 17))),
|
||||
(4608, 1536): (set(), {1, 2, 4}),
|
||||
(6288, 7168): ({1}, set()),
|
||||
(12448, 7168): ({1, 2, 3, 4}, set()),
|
||||
(7168, 768): (set(), {1}),
|
||||
# SM103-tuned config wins on B200 where the heuristic config tied.
|
||||
(7168, 1536): ({1}, set()),
|
||||
# M2 config lifted from the SM103 table (wins on B200).
|
||||
(7168, 3072): ({1, 2}, set()),
|
||||
(7168, 3584): ({1, 2}, set()),
|
||||
(7168, 8448): ({1, 2, 3}, set()),
|
||||
(8448, 7168): ({1, 2}, set()),
|
||||
(16896, 7168): ({1}, set()),
|
||||
(20480, 7168): ({1, 2, 3, 4}, set()),
|
||||
(40960, 7168): ({1, 2, 3, 4}, set()),
|
||||
(10240, 7168): ({1, 2, 3, 4}, set()),
|
||||
(3584, 7168): ({1, 2, 3}, set()),
|
||||
# TP16 shapes measured on B200.
|
||||
(768, 7168): ({1, 2, 3, 4}, set()),
|
||||
(1152, 1536): ({1}, set()),
|
||||
(3216, 7168): ({1, 2}, set()),
|
||||
(4224, 7168): ({1, 2, 3}, set()),
|
||||
}
|
||||
|
||||
|
||||
def test_sm100_table_is_keyed_by_shape() -> None:
|
||||
for (n, k), spec in k3_gemm.KIMI_K3_PROJECTIONS_SM100.items():
|
||||
assert (spec.n, spec.k) == (n, k)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS_SM100)
|
||||
def test_sm100_selector_table(key: tuple[int, int]) -> None:
|
||||
n, k = key
|
||||
spec = k3_gemm.KIMI_K3_PROJECTIONS_SM100[key]
|
||||
cute_tokens, dsv3_tokens = EXPECTED_SELECTIONS_SM100[key]
|
||||
for num_tokens in range(1, 17):
|
||||
backend = k3_gemm._backend_for(spec, num_tokens, has_residual=False)
|
||||
if num_tokens in cute_tokens:
|
||||
assert backend == "cute"
|
||||
elif num_tokens in dsv3_tokens:
|
||||
assert backend == "dsv3_fused_a"
|
||||
else:
|
||||
assert backend is None
|
||||
|
||||
|
||||
def test_sm100_build_plan_matches_table() -> None:
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS_SM100.values():
|
||||
plan = k3_gemm._build_plan(spec)
|
||||
for num_tokens in range(1, 17):
|
||||
backend = k3_gemm._backend_for(spec, num_tokens, has_residual=False)
|
||||
if backend is None:
|
||||
assert num_tokens not in plan
|
||||
else:
|
||||
assert plan[num_tokens][0] == backend
|
||||
|
||||
|
||||
def test_low_latency_table_capability_routing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: True)
|
||||
assert k3_gemm._low_latency_table() is k3_gemm.KIMI_K3_PROJECTIONS
|
||||
|
||||
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
k3_gemm.current_platform,
|
||||
"is_device_capability",
|
||||
lambda cc: cc == (10, 0),
|
||||
)
|
||||
assert k3_gemm._low_latency_table() is k3_gemm.KIMI_K3_PROJECTIONS_SM100
|
||||
|
||||
monkeypatch.setattr(
|
||||
k3_gemm.current_platform, "is_device_capability", lambda cc: False
|
||||
)
|
||||
assert k3_gemm._low_latency_table() is None
|
||||
|
||||
|
||||
def test_installation_is_shape_specific_and_unquantized(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -491,6 +578,13 @@ def test_installation_requires_bf16_sm103(
|
||||
root.projection = FakeLinear()
|
||||
monkeypatch.setattr(k3_gemm, "LinearBase", FakeLinear)
|
||||
monkeypatch.setattr(k3_gemm, "_is_sm103", lambda: platform_enabled)
|
||||
# Pin the rest of the capability probe so platform_enabled=False stays a
|
||||
# no-op even when the test itself runs on SM100 hardware.
|
||||
monkeypatch.setattr(
|
||||
k3_gemm.current_platform,
|
||||
"is_device_capability",
|
||||
lambda cc: platform_enabled and cc == (10, 3),
|
||||
)
|
||||
|
||||
k3_gemm.enable_kimi_k3_low_latency_gemm(root, dtype)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user