[Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Thien Tran
2026-09-09 07:33:12 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 3116c5d06b
commit cc4210f671
6 changed files with 314 additions and 88 deletions
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the experimental SM100 BF16x3 router GEMM."""
"""Tests for the SM100 BF16x3 router GEMM."""
import pytest
import torch
@@ -20,7 +20,16 @@ def _requires_sm100_cutedsl():
@pytest.mark.parametrize(
("num_tokens", "hidden_dim", "num_experts"),
[(48, 6144, 128), (96, 3072, 256), (129, 3072, 17)],
[
(48, 6144, 128),
(96, 3072, 256),
(129, 3072, 17),
# long-K cases exercise the multi-accumulation path (the split-K
# heuristic leaves chains of 15 and 32 K-tiles here, above the
# kernel's num_tmem_acc bound)
(1024, 8192, 256),
(2048, 8192, 256),
],
)
def test_bf16x3_router_gemm_matches_reference(
num_tokens: int, hidden_dim: int, num_experts: int
@@ -36,8 +45,9 @@ def test_bf16x3_router_gemm_matches_reference(
# Match the observed router weight scale
w *= 0.053
out = bf16x3_router_gemm(x, w)
ref = torch.nn.functional.linear(x.float(), w)
# FP64 reference: the FP32 reference itself drifts by ~5e-6 at N=2048
ref = torch.nn.functional.linear(x.double(), w.double())
assert out.shape == (num_tokens, num_experts)
assert out.dtype == torch.float32
assert torch.mean(torch.abs(out - ref)).item() < 5e-6
assert torch.mean(torch.abs(out.double() - ref)).item() < 5e-6