[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
6a9109d865
commit
c810937573
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch)."""
|
||||
"""Tests for BF16 skinny GEMMs and the Kimi-K3 SM103 selector."""
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -11,7 +11,10 @@ import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
SkinnyGemmConfig,
|
||||
)
|
||||
from vllm.models.deepseek_v32.nvidia import glm52_low_latency_gemm as glm52_gemm
|
||||
from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm
|
||||
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
|
||||
|
||||
@@ -60,6 +63,12 @@ RESIDUAL_CUTE_CASES = [
|
||||
for num_tokens, _ in spec.residual_configs
|
||||
]
|
||||
|
||||
GLM_CUTE_CASES = [
|
||||
(spec, config)
|
||||
for spec in glm52_gemm.GLM52_PROJECTIONS.values()
|
||||
for _, config in spec.cute_configs
|
||||
]
|
||||
|
||||
EXPECTED_CUTE_CONFIGS = {
|
||||
(3072, 7168, 1): (224, 3, 4, 8),
|
||||
(3072, 7168, 2): (128, 3, 2, 8),
|
||||
@@ -164,13 +173,21 @@ def test_every_dsv3_routed_shape_is_instantiated() -> None:
|
||||
)
|
||||
} | {
|
||||
(int(hd_in), int(hd_out))
|
||||
for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit)
|
||||
for hd_in, hd_out in re.findall(
|
||||
r"hd_in == (\d+) && hd_out == (\d+)",
|
||||
production_macros + explicit,
|
||||
)
|
||||
}
|
||||
assert compiled, "failed to parse the dispatch list"
|
||||
|
||||
specs = [
|
||||
*KIMI_K3_PROJECTIONS.values(),
|
||||
glm52_gemm.GLM52_QKV_A_PROJECTION,
|
||||
glm52_gemm.GLM52_Q_B_PROJECTION,
|
||||
]
|
||||
missing = sorted(
|
||||
(spec.n, spec.k)
|
||||
for spec in KIMI_K3_PROJECTIONS.values()
|
||||
for spec in specs
|
||||
if spec.dsv3_tokens and (spec.k, spec.n) not in compiled
|
||||
)
|
||||
assert not missing, (
|
||||
@@ -190,12 +207,17 @@ def test_packed_row_major_rejects_single_row_slice() -> None:
|
||||
|
||||
|
||||
def test_cute_configs_match_measured_table() -> None:
|
||||
actual = {
|
||||
(spec.n, spec.k, num_tokens): _config_tuple(config)
|
||||
configs = [
|
||||
(spec.n, spec.k, num_tokens, config)
|
||||
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
|
||||
for num_tokens, config in spec.cute_configs
|
||||
]
|
||||
actual = {
|
||||
(n, k, num_tokens): _config_tuple(config)
|
||||
for n, k, num_tokens, config in configs
|
||||
}
|
||||
assert actual == EXPECTED_CUTE_CONFIGS
|
||||
assert all(config.static_k is None for *_, config in configs)
|
||||
|
||||
|
||||
def test_residual_cute_configs_match_measured_table() -> None:
|
||||
@@ -207,6 +229,134 @@ def test_residual_cute_configs_match_measured_table() -> None:
|
||||
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
|
||||
|
||||
|
||||
def test_glm52_projection_plans_are_separate() -> None:
|
||||
qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION
|
||||
q_b = glm52_gemm.GLM52_Q_B_PROJECTION
|
||||
|
||||
qkv_a_plan = qkv_a.build_plan()
|
||||
q_b_plan = q_b.build_plan()
|
||||
|
||||
assert (qkv_a.n, qkv_a.k, set(qkv_a_plan)) == (
|
||||
2624,
|
||||
6144,
|
||||
set(range(1, 17)),
|
||||
)
|
||||
assert (q_b.n, q_b.k, set(q_b_plan)) == (
|
||||
2048,
|
||||
2048,
|
||||
set(range(1, 17)),
|
||||
)
|
||||
assert {
|
||||
num_tokens
|
||||
for num_tokens, (backend, _) in qkv_a_plan.items()
|
||||
if backend == "cute"
|
||||
} == {1, 2}
|
||||
assert {
|
||||
num_tokens
|
||||
for num_tokens, (backend, _) in qkv_a_plan.items()
|
||||
if backend == "dsv3_fused_a"
|
||||
} == set(range(3, 17))
|
||||
assert {
|
||||
num_tokens for num_tokens, (backend, _) in q_b_plan.items() if backend == "cute"
|
||||
} == {1, 2}
|
||||
assert {
|
||||
num_tokens
|
||||
for num_tokens, (backend, _) in q_b_plan.items()
|
||||
if backend == "dsv3_fused_a"
|
||||
} == set(range(3, 17))
|
||||
|
||||
eh = glm52_gemm.GLM52_EH_PROJECTION
|
||||
eh_plan = eh.build_plan()
|
||||
assert (eh.n, eh.k) == (6144, 12288)
|
||||
# The MTP eh_proj has no dsv3 winners; M >= 4 falls back to cuBLAS.
|
||||
assert set(eh_plan) == {1, 2, 3}
|
||||
assert all(backend == "cute" for backend, _ in eh_plan.values())
|
||||
|
||||
|
||||
def test_glm52_layout_rejects_nonpacked_single_row_view() -> None:
|
||||
single_row = torch.empty(1, 144)[:, :128]
|
||||
multiple_rows = torch.empty(2, 144)[:, :128]
|
||||
|
||||
assert single_row.stride() == multiple_rows.stride() == (144, 1)
|
||||
assert not glm52_gemm._is_supported_row_major(single_row)
|
||||
assert not glm52_gemm._is_supported_row_major(multiple_rows)
|
||||
|
||||
|
||||
def test_glm52_installer_maps_only_selected_unquantized_shapes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class FakeLinearBase(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
n: int,
|
||||
k: int,
|
||||
quant_method: object,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.empty(
|
||||
n,
|
||||
k,
|
||||
dtype=torch.bfloat16,
|
||||
device="meta",
|
||||
)
|
||||
)
|
||||
self.quant_method = quant_method
|
||||
|
||||
qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION
|
||||
q_b = glm52_gemm.GLM52_Q_B_PROJECTION
|
||||
root = nn.Module()
|
||||
root.attn = nn.Module()
|
||||
root.attn.fused_qkv_a_proj = FakeLinearBase(
|
||||
qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
|
||||
)
|
||||
root.attn.q_b_proj = FakeLinearBase(
|
||||
q_b.n, q_b.k, glm52_gemm.UnquantizedLinearMethod()
|
||||
)
|
||||
root.same_shape_other_name = FakeLinearBase(
|
||||
qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
|
||||
)
|
||||
root.quantized = nn.Module()
|
||||
quantized_method = object()
|
||||
root.quantized.q_b_proj = FakeLinearBase(q_b.n, q_b.k, quantized_method)
|
||||
root.wrong_shape = nn.Module()
|
||||
root.wrong_shape.fused_qkv_a_proj = FakeLinearBase(
|
||||
qkv_a.n + 1, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
|
||||
)
|
||||
monkeypatch.setattr(glm52_gemm, "LinearBase", FakeLinearBase)
|
||||
monkeypatch.setattr(glm52_gemm, "_is_sm103", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
glm52_gemm.shape_dynamic_skinny_gemm,
|
||||
"is_available",
|
||||
lambda: False,
|
||||
)
|
||||
|
||||
glm52_gemm.enable_glm52_low_latency_gemm(
|
||||
root,
|
||||
torch.bfloat16,
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
root.attn.fused_qkv_a_proj.quant_method,
|
||||
glm52_gemm.GLM52LowLatencyLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
root.attn.q_b_proj.quant_method,
|
||||
glm52_gemm.GLM52LowLatencyLinearMethod,
|
||||
)
|
||||
assert root.attn.fused_qkv_a_proj.quant_method._plan == qkv_a.build_plan()
|
||||
assert root.attn.q_b_proj.quant_method._plan == q_b.build_plan()
|
||||
assert isinstance(
|
||||
root.same_shape_other_name.quant_method,
|
||||
glm52_gemm.GLM52LowLatencyLinearMethod,
|
||||
)
|
||||
assert root.quantized.q_b_proj.quant_method is quantized_method
|
||||
assert (
|
||||
type(root.wrong_shape.fused_qkv_a_proj.quant_method)
|
||||
is glm52_gemm.UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
|
||||
def test_sm103_selector_table(key: tuple[int, int]) -> None:
|
||||
n, k = key
|
||||
@@ -361,6 +511,76 @@ def _require_sm103_and_cute() -> None:
|
||||
pytest.skip("CuTe DSL is not available")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES)
|
||||
def test_glm_cute_selected_shapes(
|
||||
spec: glm52_gemm.GLM52ProjectionSpec,
|
||||
config: SkinnyGemmConfig,
|
||||
) -> None:
|
||||
_require_sm103_and_cute()
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(
|
||||
config.num_rows,
|
||||
spec.k,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
plan = spec.build_plan()
|
||||
|
||||
output = glm52_gemm.run_glm52_plan(plan, x, weight)
|
||||
|
||||
assert output is not None
|
||||
reference = x.float() @ weight.float().t()
|
||||
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
|
||||
|
||||
|
||||
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
|
||||
_require_sm103_and_cute()
|
||||
spec = glm52_gemm.GLM52_Q_B_PROJECTION
|
||||
storage = torch.randn(
|
||||
1,
|
||||
glm52_gemm.GLM52_QKV_A_PROJECTION.n,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
x = storage[:, : spec.k]
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
assert x.stride() == (glm52_gemm.GLM52_QKV_A_PROJECTION.n, 1)
|
||||
assert not glm52_gemm._runtime_ok(x, weight)
|
||||
output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight)
|
||||
|
||||
assert output is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES)
|
||||
def test_glm_cute_selected_shapes_cuda_graph_capture(
|
||||
spec: glm52_gemm.GLM52ProjectionSpec,
|
||||
config: SkinnyGemmConfig,
|
||||
) -> None:
|
||||
_require_sm103_and_cute()
|
||||
x = torch.randn(
|
||||
config.num_rows,
|
||||
spec.k,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
plan = spec.build_plan()
|
||||
glm52_gemm.run_glm52_plan(plan, x, weight)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = glm52_gemm.run_glm52_plan(plan, x, weight)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert output is not None
|
||||
reference = x.float() @ weight.float().t()
|
||||
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES)
|
||||
def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None:
|
||||
_require_sm103_and_cute()
|
||||
@@ -393,6 +613,15 @@ DSV3_CASES = sorted(
|
||||
for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens)
|
||||
)
|
||||
|
||||
GLM_DSV3_CASES = [
|
||||
(num_tokens, spec)
|
||||
for spec in (
|
||||
glm52_gemm.GLM52_QKV_A_PROJECTION,
|
||||
glm52_gemm.GLM52_Q_B_PROJECTION,
|
||||
)
|
||||
for num_tokens in sorted(_dsv3_probe_tokens(spec.dsv3_tokens))
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES)
|
||||
def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
|
||||
@@ -413,6 +642,26 @@ def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens,spec", GLM_DSV3_CASES)
|
||||
def test_glm_dsv3_selected_shapes(
|
||||
num_tokens: int,
|
||||
spec: glm52_gemm.GLM52ProjectionSpec,
|
||||
) -> None:
|
||||
_require_sm103_and_dsv3()
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight)
|
||||
|
||||
assert output is not None
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def test_nonpacked_single_token_dsv3_falls_back() -> None:
|
||||
_require_sm103_and_dsv3()
|
||||
n, k = 1536, 128
|
||||
|
||||
Reference in New Issue
Block a user