[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
xiaozhoupy
2026-08-03 16:49:41 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 6a9109d865
commit c810937573
7 changed files with 619 additions and 51 deletions
+255 -6
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch)."""
"""Tests for BF16 skinny GEMMs and the Kimi-K3 SM103 selector."""
from pathlib import Path
from types import SimpleNamespace
@@ -11,7 +11,10 @@ import regex as re
import torch
from torch import nn
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
)
from vllm.models.deepseek_v32.nvidia import glm52_low_latency_gemm as glm52_gemm
from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
@@ -60,6 +63,12 @@ RESIDUAL_CUTE_CASES = [
for num_tokens, _ in spec.residual_configs
]
GLM_CUTE_CASES = [
(spec, config)
for spec in glm52_gemm.GLM52_PROJECTIONS.values()
for _, config in spec.cute_configs
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
@@ -164,13 +173,21 @@ def test_every_dsv3_routed_shape_is_instantiated() -> None:
)
} | {
(int(hd_in), int(hd_out))
for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit)
for hd_in, hd_out in re.findall(
r"hd_in == (\d+) && hd_out == (\d+)",
production_macros + explicit,
)
}
assert compiled, "failed to parse the dispatch list"
specs = [
*KIMI_K3_PROJECTIONS.values(),
glm52_gemm.GLM52_QKV_A_PROJECTION,
glm52_gemm.GLM52_Q_B_PROJECTION,
]
missing = sorted(
(spec.n, spec.k)
for spec in KIMI_K3_PROJECTIONS.values()
for spec in specs
if spec.dsv3_tokens and (spec.k, spec.n) not in compiled
)
assert not missing, (
@@ -190,12 +207,17 @@ def test_packed_row_major_rejects_single_row_slice() -> None:
def test_cute_configs_match_measured_table() -> None:
actual = {
(spec.n, spec.k, num_tokens): _config_tuple(config)
configs = [
(spec.n, spec.k, num_tokens, config)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, config in spec.cute_configs
]
actual = {
(n, k, num_tokens): _config_tuple(config)
for n, k, num_tokens, config in configs
}
assert actual == EXPECTED_CUTE_CONFIGS
assert all(config.static_k is None for *_, config in configs)
def test_residual_cute_configs_match_measured_table() -> None:
@@ -207,6 +229,134 @@ def test_residual_cute_configs_match_measured_table() -> None:
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
def test_glm52_projection_plans_are_separate() -> None:
qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION
q_b = glm52_gemm.GLM52_Q_B_PROJECTION
qkv_a_plan = qkv_a.build_plan()
q_b_plan = q_b.build_plan()
assert (qkv_a.n, qkv_a.k, set(qkv_a_plan)) == (
2624,
6144,
set(range(1, 17)),
)
assert (q_b.n, q_b.k, set(q_b_plan)) == (
2048,
2048,
set(range(1, 17)),
)
assert {
num_tokens
for num_tokens, (backend, _) in qkv_a_plan.items()
if backend == "cute"
} == {1, 2}
assert {
num_tokens
for num_tokens, (backend, _) in qkv_a_plan.items()
if backend == "dsv3_fused_a"
} == set(range(3, 17))
assert {
num_tokens for num_tokens, (backend, _) in q_b_plan.items() if backend == "cute"
} == {1, 2}
assert {
num_tokens
for num_tokens, (backend, _) in q_b_plan.items()
if backend == "dsv3_fused_a"
} == set(range(3, 17))
eh = glm52_gemm.GLM52_EH_PROJECTION
eh_plan = eh.build_plan()
assert (eh.n, eh.k) == (6144, 12288)
# The MTP eh_proj has no dsv3 winners; M >= 4 falls back to cuBLAS.
assert set(eh_plan) == {1, 2, 3}
assert all(backend == "cute" for backend, _ in eh_plan.values())
def test_glm52_layout_rejects_nonpacked_single_row_view() -> None:
single_row = torch.empty(1, 144)[:, :128]
multiple_rows = torch.empty(2, 144)[:, :128]
assert single_row.stride() == multiple_rows.stride() == (144, 1)
assert not glm52_gemm._is_supported_row_major(single_row)
assert not glm52_gemm._is_supported_row_major(multiple_rows)
def test_glm52_installer_maps_only_selected_unquantized_shapes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class FakeLinearBase(nn.Module):
def __init__(
self,
n: int,
k: int,
quant_method: object,
) -> None:
super().__init__()
self.weight = nn.Parameter(
torch.empty(
n,
k,
dtype=torch.bfloat16,
device="meta",
)
)
self.quant_method = quant_method
qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION
q_b = glm52_gemm.GLM52_Q_B_PROJECTION
root = nn.Module()
root.attn = nn.Module()
root.attn.fused_qkv_a_proj = FakeLinearBase(
qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
)
root.attn.q_b_proj = FakeLinearBase(
q_b.n, q_b.k, glm52_gemm.UnquantizedLinearMethod()
)
root.same_shape_other_name = FakeLinearBase(
qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
)
root.quantized = nn.Module()
quantized_method = object()
root.quantized.q_b_proj = FakeLinearBase(q_b.n, q_b.k, quantized_method)
root.wrong_shape = nn.Module()
root.wrong_shape.fused_qkv_a_proj = FakeLinearBase(
qkv_a.n + 1, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
)
monkeypatch.setattr(glm52_gemm, "LinearBase", FakeLinearBase)
monkeypatch.setattr(glm52_gemm, "_is_sm103", lambda: True)
monkeypatch.setattr(
glm52_gemm.shape_dynamic_skinny_gemm,
"is_available",
lambda: False,
)
glm52_gemm.enable_glm52_low_latency_gemm(
root,
torch.bfloat16,
)
assert isinstance(
root.attn.fused_qkv_a_proj.quant_method,
glm52_gemm.GLM52LowLatencyLinearMethod,
)
assert isinstance(
root.attn.q_b_proj.quant_method,
glm52_gemm.GLM52LowLatencyLinearMethod,
)
assert root.attn.fused_qkv_a_proj.quant_method._plan == qkv_a.build_plan()
assert root.attn.q_b_proj.quant_method._plan == q_b.build_plan()
assert isinstance(
root.same_shape_other_name.quant_method,
glm52_gemm.GLM52LowLatencyLinearMethod,
)
assert root.quantized.q_b_proj.quant_method is quantized_method
assert (
type(root.wrong_shape.fused_qkv_a_proj.quant_method)
is glm52_gemm.UnquantizedLinearMethod
)
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
def test_sm103_selector_table(key: tuple[int, int]) -> None:
n, k = key
@@ -361,6 +511,76 @@ def _require_sm103_and_cute() -> None:
pytest.skip("CuTe DSL is not available")
@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES)
def test_glm_cute_selected_shapes(
spec: glm52_gemm.GLM52ProjectionSpec,
config: SkinnyGemmConfig,
) -> None:
_require_sm103_and_cute()
torch.manual_seed(42)
x = torch.randn(
config.num_rows,
spec.k,
dtype=torch.bfloat16,
device="cuda",
)
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
plan = spec.build_plan()
output = glm52_gemm.run_glm52_plan(plan, x, weight)
assert output is not None
reference = x.float() @ weight.float().t()
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
_require_sm103_and_cute()
spec = glm52_gemm.GLM52_Q_B_PROJECTION
storage = torch.randn(
1,
glm52_gemm.GLM52_QKV_A_PROJECTION.n,
dtype=torch.bfloat16,
device="cuda",
)
x = storage[:, : spec.k]
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
assert x.stride() == (glm52_gemm.GLM52_QKV_A_PROJECTION.n, 1)
assert not glm52_gemm._runtime_ok(x, weight)
output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight)
assert output is None
@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES)
def test_glm_cute_selected_shapes_cuda_graph_capture(
spec: glm52_gemm.GLM52ProjectionSpec,
config: SkinnyGemmConfig,
) -> None:
_require_sm103_and_cute()
x = torch.randn(
config.num_rows,
spec.k,
dtype=torch.bfloat16,
device="cuda",
)
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
plan = spec.build_plan()
glm52_gemm.run_glm52_plan(plan, x, weight)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = glm52_gemm.run_glm52_plan(plan, x, weight)
graph.replay()
torch.accelerator.synchronize()
assert output is not None
reference = x.float() @ weight.float().t()
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
@pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES)
def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None:
_require_sm103_and_cute()
@@ -393,6 +613,15 @@ DSV3_CASES = sorted(
for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens)
)
GLM_DSV3_CASES = [
(num_tokens, spec)
for spec in (
glm52_gemm.GLM52_QKV_A_PROJECTION,
glm52_gemm.GLM52_Q_B_PROJECTION,
)
for num_tokens in sorted(_dsv3_probe_tokens(spec.dsv3_tokens))
]
@pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES)
def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
@@ -413,6 +642,26 @@ def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
assert cosine > 0.999
@pytest.mark.parametrize("num_tokens,spec", GLM_DSV3_CASES)
def test_glm_dsv3_selected_shapes(
num_tokens: int,
spec: glm52_gemm.GLM52ProjectionSpec,
) -> None:
_require_sm103_and_dsv3()
torch.manual_seed(42)
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight)
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_nonpacked_single_token_dsv3_falls_back() -> None:
_require_sm103_and_dsv3()
n, k = 1536, 128