From 4ed438b79ceaff5fe5d6d4a98b09bfb8718cceaa Mon Sep 17 00:00:00 2001 From: namgyu-youn Date: Thu, 1 Oct 2026 12:40:55 +0900 Subject: [PATCH] [Perf][Qwen4Exp] Add SM100 low-latency decode GEMM plans (#59214) Signed-off-by: namgyu-youn Co-authored-by: Thien Tran --- tests/kernels/test_bf16_skinny_gemm.py | 30 ++++++++ .../qwen4_exp/nvidia/low_latency_gemm.py | 69 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py index d452354e5f..412e772a2b 100644 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -85,6 +85,12 @@ QWEN4_EXP_SM90_CASES = [ for num_tokens, config in plans.items() ] +QWEN4_EXP_SM100_CASES = [ + (n, k, num_tokens, config) + for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS.items() + for num_tokens, config in plans.items() +] + EXPECTED_CUTE_CONFIGS = { (3072, 7168, 1): (224, 3, 4, 8), (3072, 7168, 2): (128, 3, 2, 8), @@ -717,6 +723,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None: "capability,expected_plans", [ ((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS), + ((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS), ((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS), ((8, 0), {}), ], @@ -909,6 +916,29 @@ def test_qwen4_exp_sm90_selected_shapes( assert cosine > 0.999 +@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM100_CASES) +def test_qwen4_exp_sm100_selected_shapes( + n: int, + k: int, + num_tokens: int, + config: SkinnyGemmConfig, +) -> None: + _require_capability_and_cute((10, 0)) + torch.manual_seed(42 + num_tokens) + x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") + + selected = qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS[(n, k)][num_tokens] + assert selected == config + output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight) + + reference = torch.nn.functional.linear(x, weight) + cosine = torch.nn.functional.cosine_similarity( + output.float().flatten(), reference.float().flatten(), dim=0 + ).item() + assert cosine > 0.999 + + def test_glm52_q_b_nonpacked_single_row_falls_back() -> None: _require_sm103_and_cute() spec = glm52_gemm.GLM52_Q_B_PROJECTION diff --git a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py index c177f07745..f34f0b6335 100644 --- a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py +++ b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py @@ -144,6 +144,73 @@ QWEN4_EXP_SM90_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = } +# B200 plans; the winning configs differ from the B300 table. +QWEN4_EXP_SM100_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = { + # GDN fused B/A projection, TP=4. + (24, 2560): { + 1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560), + }, + # Shared-expert fused gate/up projection, TP=4. + (320, 2560): { + 1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 160, 1, k_unroll=2, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560), + }, + # Final HC down projection, replicated in a TP=4 deployment. + (320, 10240): { + 1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240), + 2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240), + 4: SkinnyGemmConfig(4, 256, 1, k_unroll=2), + 8: SkinnyGemmConfig(8, 256, 1), + }, + # HC merged down/injection projection, replicated in a TP=4 deployment. + (336, 10240): { + 1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240), + 2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240), + 8: SkinnyGemmConfig(8, 256, 3, static_k=10240), + }, + # QSA indexer Q/K projection, replicated in a TP=4 deployment. + (640, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + }, + # GDN and QSA output projections, TP=4. + (2560, 1536): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4, static_k=1536), + 2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=1536), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=2, static_k=1536), + }, + # QSA fused QKV/gate projection, TP=4. + (3584, 2560): { + 1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 4, k_unroll=2, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560), + }, + # GDN fused QKVZ projection, TP=4. + (4096, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=2560), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560), + }, + # LM head, TP=4. + (62080, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4), + }, +} + + +def _is_sm100() -> bool: + return current_platform.is_device_capability((10, 0)) + + def _is_sm103() -> bool: return current_platform.is_device_capability((10, 3)) @@ -155,6 +222,8 @@ def _is_sm90() -> bool: def _gemm_plans() -> dict[tuple[int, int], dict[int, SkinnyGemmConfig]]: if _is_sm103(): return QWEN4_EXP_GEMM_PLANS + if _is_sm100(): + return QWEN4_EXP_SM100_GEMM_PLANS if _is_sm90(): return QWEN4_EXP_SM90_GEMM_PLANS return {}