From f590eb2448a51028f6044c6f60104ce0ef3189e9 Mon Sep 17 00:00:00 2001 From: Stefano Castagnetta Date: Fri, 2 Oct 2026 19:17:45 +0200 Subject: [PATCH] [Perf][Qwen4Exp] Add SM121 TP=1 skinny-GEMM plans (#59753) Signed-off-by: Stefano Castagnetta Co-authored-by: Benjamin Chislett --- tests/kernels/test_bf16_skinny_gemm.py | 30 +++++ .../qwen4_exp/nvidia/low_latency_gemm.py | 106 ++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py index 412e772a2b..7cdaf2b495 100644 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -91,6 +91,12 @@ QWEN4_EXP_SM100_CASES = [ for num_tokens, config in plans.items() ] +QWEN4_EXP_SM121_CASES = [ + (n, k, num_tokens, config) + for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS.items() + for num_tokens, config in plans.items() +] + EXPECTED_CUTE_CONFIGS = { (3072, 7168, 1): (224, 3, 4, 8), (3072, 7168, 2): (128, 3, 2, 8), @@ -725,6 +731,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None: ((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS), ((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS), ((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS), + ((12, 1), qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS), ((8, 0), {}), ], ) @@ -939,6 +946,29 @@ def test_qwen4_exp_sm100_selected_shapes( assert cosine > 0.999 +@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM121_CASES) +def test_qwen4_exp_sm121_selected_shapes( + n: int, + k: int, + num_tokens: int, + config: SkinnyGemmConfig, +) -> None: + _require_capability_and_cute((12, 1)) + torch.manual_seed(42 + num_tokens) + x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") + + selected = qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS[(n, k)][num_tokens] + assert selected == config + output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight) + + reference = torch.nn.functional.linear(x, weight) + cosine = torch.nn.functional.cosine_similarity( + output.float().flatten(), reference.float().flatten(), dim=0 + ).item() + assert cosine > 0.999 + + def test_glm52_q_b_nonpacked_single_row_falls_back() -> None: _require_sm103_and_cute() spec = glm52_gemm.GLM52_Q_B_PROJECTION diff --git a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py index f34f0b6335..fa01c75698 100644 --- a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py +++ b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py @@ -207,6 +207,110 @@ QWEN4_EXP_SM100_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = } +# DGX Spark (GB10) plans, TP=1. +QWEN4_EXP_SM121_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = { + # Shared-expert gate. + (1, 2560): { + 1: SkinnyGemmConfig(1, 64, 1, k_unroll=2, static_k=2560), + 2: SkinnyGemmConfig(2, 64, 1, k_unroll=2, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 1, vector_width=2, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560), + }, + # GDN fused B/A projection. + (96, 2560): { + 1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=2, static_k=2560), + 2: SkinnyGemmConfig(2, 64, 4, vector_width=2, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 4, vector_width=2, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 4, k_unroll=2, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 64, 1), + }, + # HC merged down/injection projection. + (336, 10240): { + 1: SkinnyGemmConfig(1, 256, 1, vector_width=4, static_k=10240), + 2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, vector_width=4, static_k=10240), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, static_k=10240), + 8: SkinnyGemmConfig(8, 128, 4, k_unroll=4, static_k=10240), + 16: SkinnyGemmConfig(16, 128, 4, k_unroll=4, static_k=10240), + }, + # Router. + (512, 2560): { + 1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, vector_width=2, static_k=2560), + 2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, vector_width=2, static_k=2560), + 4: SkinnyGemmConfig(4, 64, 1, k_unroll=2, static_k=2560), + 8: SkinnyGemmConfig(8, 64, 2, static_k=2560), + 16: SkinnyGemmConfig(16, 64, 2, k_unroll=2, static_k=2560), + }, + # QSA indexer Q/K projection. + (640, 2560): { + 1: SkinnyGemmConfig(1, 256, 1, vector_width=2, static_k=2560), + 2: SkinnyGemmConfig(2, 256, 1, vector_width=2, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=2, vector_width=4, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 64, 1, static_k=2560), + }, + # Shared-expert fused gate/up projection. + (1280, 2560): { + 1: SkinnyGemmConfig(1, 32, 1, k_unroll=2, static_k=2560), + 2: SkinnyGemmConfig(2, 64, 1, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=2, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=2, vector_width=2, static_k=2560), + 16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, static_k=2560), + }, + # Shared-expert down projection. + (2560, 640): { + 1: SkinnyGemmConfig(1, 64, 1, k_unroll=4, vector_width=2, static_k=640), + 2: SkinnyGemmConfig(2, 64, 1, vector_width=2, static_k=640), + 4: SkinnyGemmConfig(4, 64, 1, vector_width=2, static_k=640), + 8: SkinnyGemmConfig(8, 32, 1, k_unroll=4, vector_width=4, static_k=640), + 16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, vector_width=4, static_k=640), + }, + # GDN and QSA output projections. + (2560, 6144): { + 1: SkinnyGemmConfig(1, 64, 2, vector_width=4, static_k=6144), + 2: SkinnyGemmConfig(2, 64, 1, k_unroll=4, vector_width=4, static_k=6144), + 4: SkinnyGemmConfig(4, 64, 1, k_unroll=4, vector_width=4, static_k=6144), + 8: SkinnyGemmConfig(8, 128, 1, vector_width=2, static_k=6144), + 16: SkinnyGemmConfig(16, 32, 1, static_k=6144), + }, + # HC up projection. + (10240, 320): { + 1: SkinnyGemmConfig(1, 32, 1, k_unroll=2, vector_width=2, static_k=320), + 2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=320), + 4: SkinnyGemmConfig(4, 32, 1, k_unroll=2, vector_width=2, static_k=320), + 8: SkinnyGemmConfig(8, 32, 1, vector_width=2, static_k=320), + }, + # QSA fused QKV/gate projection. + (13312, 2560): { + 1: SkinnyGemmConfig(1, 32, 1, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 32, 1, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 32, 1, vector_width=4, static_k=2560), + 8: SkinnyGemmConfig(8, 64, 1, vector_width=2, static_k=2560), + 16: SkinnyGemmConfig(16, 32, 1, vector_width=4, static_k=2560), + }, + # GDN fused QKVZ projection. + (16384, 2560): { + 1: SkinnyGemmConfig(1, 32, 1, k_unroll=4, vector_width=2, static_k=2560), + 2: SkinnyGemmConfig(2, 32, 1, vector_width=2, static_k=2560), + 4: SkinnyGemmConfig(4, 32, 1, k_unroll=2, vector_width=2, static_k=2560), + 8: SkinnyGemmConfig(8, 64, 1, vector_width=2, static_k=2560), + 16: SkinnyGemmConfig(16, 32, 1, k_unroll=2, vector_width=2, static_k=2560), + }, + # LM head. + (248320, 2560): { + 1: SkinnyGemmConfig(1, 128, 1, k_unroll=2, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 1, k_unroll=2, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 32, 1, vector_width=4, static_k=2560), + 8: SkinnyGemmConfig(8, 64, 1, k_unroll=2, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560), + }, +} + + +def _is_sm121() -> bool: + return current_platform.is_device_capability((12, 1)) + + def _is_sm100() -> bool: return current_platform.is_device_capability((10, 0)) @@ -226,6 +330,8 @@ def _gemm_plans() -> dict[tuple[int, int], dict[int, SkinnyGemmConfig]]: return QWEN4_EXP_SM100_GEMM_PLANS if _is_sm90(): return QWEN4_EXP_SM90_GEMM_PLANS + if _is_sm121(): + return QWEN4_EXP_SM121_GEMM_PLANS return {}