[Perf][Qwen4Exp] Add SM121 TP=1 skinny-GEMM plans (#59753)

Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Stefano Castagnetta
2026-10-02 17:17:45 +00:00
committed by GitHub
co-authored by Benjamin Chislett
parent 9cf087fd14
commit f590eb2448
2 changed files with 136 additions and 0 deletions
+30
View File
@@ -91,6 +91,12 @@ QWEN4_EXP_SM100_CASES = [
for num_tokens, config in plans.items()
]
QWEN4_EXP_SM121_CASES = [
(n, k, num_tokens, config)
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS.items()
for num_tokens, config in plans.items()
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
@@ -725,6 +731,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
((12, 1), qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS),
((8, 0), {}),
],
)
@@ -939,6 +946,29 @@ def test_qwen4_exp_sm100_selected_shapes(
assert cosine > 0.999
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM121_CASES)
def test_qwen4_exp_sm121_selected_shapes(
n: int,
k: int,
num_tokens: int,
config: SkinnyGemmConfig,
) -> None:
_require_capability_and_cute((12, 1))
torch.manual_seed(42 + num_tokens)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
selected = qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS[(n, k)][num_tokens]
assert selected == config
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
_require_sm103_and_cute()
spec = glm52_gemm.GLM52_Q_B_PROJECTION
@@ -207,6 +207,110 @@ QWEN4_EXP_SM100_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] =
}
# DGX Spark (GB10) plans, TP=1.
QWEN4_EXP_SM121_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
# Shared-expert gate.
(1, 2560): {
1: SkinnyGemmConfig(1, 64, 1, k_unroll=2, static_k=2560),
2: SkinnyGemmConfig(2, 64, 1, k_unroll=2, vector_width=4, static_k=2560),
4: SkinnyGemmConfig(4, 128, 1, vector_width=2, static_k=2560),
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560),
16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560),
},
# GDN fused B/A projection.
(96, 2560): {
1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=2, static_k=2560),
2: SkinnyGemmConfig(2, 64, 4, vector_width=2, static_k=2560),
4: SkinnyGemmConfig(4, 128, 4, vector_width=2, static_k=2560),
8: SkinnyGemmConfig(8, 128, 4, k_unroll=2, vector_width=4, static_k=2560),
16: SkinnyGemmConfig(16, 64, 1),
},
# HC merged down/injection projection.
(336, 10240): {
1: SkinnyGemmConfig(1, 256, 1, vector_width=4, static_k=10240),
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, vector_width=4, static_k=10240),
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, static_k=10240),
8: SkinnyGemmConfig(8, 128, 4, k_unroll=4, static_k=10240),
16: SkinnyGemmConfig(16, 128, 4, k_unroll=4, static_k=10240),
},
# Router.
(512, 2560): {
1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, vector_width=2, static_k=2560),
2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, vector_width=2, static_k=2560),
4: SkinnyGemmConfig(4, 64, 1, k_unroll=2, static_k=2560),
8: SkinnyGemmConfig(8, 64, 2, static_k=2560),
16: SkinnyGemmConfig(16, 64, 2, k_unroll=2, static_k=2560),
},
# QSA indexer Q/K projection.
(640, 2560): {
1: SkinnyGemmConfig(1, 256, 1, vector_width=2, static_k=2560),
2: SkinnyGemmConfig(2, 256, 1, vector_width=2, static_k=2560),
4: SkinnyGemmConfig(4, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
8: SkinnyGemmConfig(8, 128, 1, vector_width=4, static_k=2560),
16: SkinnyGemmConfig(16, 64, 1, static_k=2560),
},
# Shared-expert fused gate/up projection.
(1280, 2560): {
1: SkinnyGemmConfig(1, 32, 1, k_unroll=2, static_k=2560),
2: SkinnyGemmConfig(2, 64, 1, vector_width=4, static_k=2560),
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=2, static_k=2560),
8: SkinnyGemmConfig(8, 128, 1, k_unroll=2, vector_width=2, static_k=2560),
16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, static_k=2560),
},
# Shared-expert down projection.
(2560, 640): {
1: SkinnyGemmConfig(1, 64, 1, k_unroll=4, vector_width=2, static_k=640),
2: SkinnyGemmConfig(2, 64, 1, vector_width=2, static_k=640),
4: SkinnyGemmConfig(4, 64, 1, vector_width=2, static_k=640),
8: SkinnyGemmConfig(8, 32, 1, k_unroll=4, vector_width=4, static_k=640),
16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, vector_width=4, static_k=640),
},
# GDN and QSA output projections.
(2560, 6144): {
1: SkinnyGemmConfig(1, 64, 2, vector_width=4, static_k=6144),
2: SkinnyGemmConfig(2, 64, 1, k_unroll=4, vector_width=4, static_k=6144),
4: SkinnyGemmConfig(4, 64, 1, k_unroll=4, vector_width=4, static_k=6144),
8: SkinnyGemmConfig(8, 128, 1, vector_width=2, static_k=6144),
16: SkinnyGemmConfig(16, 32, 1, static_k=6144),
},
# HC up projection.
(10240, 320): {
1: SkinnyGemmConfig(1, 32, 1, k_unroll=2, vector_width=2, static_k=320),
2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=320),
4: SkinnyGemmConfig(4, 32, 1, k_unroll=2, vector_width=2, static_k=320),
8: SkinnyGemmConfig(8, 32, 1, vector_width=2, static_k=320),
},
# QSA fused QKV/gate projection.
(13312, 2560): {
1: SkinnyGemmConfig(1, 32, 1, vector_width=4, static_k=2560),
2: SkinnyGemmConfig(2, 32, 1, vector_width=4, static_k=2560),
4: SkinnyGemmConfig(4, 32, 1, vector_width=4, static_k=2560),
8: SkinnyGemmConfig(8, 64, 1, vector_width=2, static_k=2560),
16: SkinnyGemmConfig(16, 32, 1, vector_width=4, static_k=2560),
},
# GDN fused QKVZ projection.
(16384, 2560): {
1: SkinnyGemmConfig(1, 32, 1, k_unroll=4, vector_width=2, static_k=2560),
2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=2560),
4: SkinnyGemmConfig(4, 32, 1, k_unroll=2, vector_width=2, static_k=2560),
8: SkinnyGemmConfig(8, 64, 1, vector_width=2, static_k=2560),
16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, vector_width=2, static_k=2560),
},
# LM head.
(248320, 2560): {
1: SkinnyGemmConfig(1, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
2: SkinnyGemmConfig(2, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
4: SkinnyGemmConfig(4, 32, 1, vector_width=4, static_k=2560),
8: SkinnyGemmConfig(8, 64, 1, k_unroll=2, vector_width=4, static_k=2560),
16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560),
},
}
def _is_sm121() -> bool:
return current_platform.is_device_capability((12, 1))
def _is_sm100() -> bool:
return current_platform.is_device_capability((10, 0))
@@ -226,6 +330,8 @@ def _gemm_plans() -> dict[tuple[int, int], dict[int, SkinnyGemmConfig]]:
return QWEN4_EXP_SM100_GEMM_PLANS
if _is_sm90():
return QWEN4_EXP_SM90_GEMM_PLANS
if _is_sm121():
return QWEN4_EXP_SM121_GEMM_PLANS
return {}