[Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
3116c5d06b
commit
cc4210f671
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the experimental SM100 BF16x3 router GEMM."""
|
||||
"""Tests for the SM100 BF16x3 router GEMM."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -20,7 +20,16 @@ def _requires_sm100_cutedsl():
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "hidden_dim", "num_experts"),
|
||||
[(48, 6144, 128), (96, 3072, 256), (129, 3072, 17)],
|
||||
[
|
||||
(48, 6144, 128),
|
||||
(96, 3072, 256),
|
||||
(129, 3072, 17),
|
||||
# long-K cases exercise the multi-accumulation path (the split-K
|
||||
# heuristic leaves chains of 15 and 32 K-tiles here, above the
|
||||
# kernel's num_tmem_acc bound)
|
||||
(1024, 8192, 256),
|
||||
(2048, 8192, 256),
|
||||
],
|
||||
)
|
||||
def test_bf16x3_router_gemm_matches_reference(
|
||||
num_tokens: int, hidden_dim: int, num_experts: int
|
||||
@@ -36,8 +45,9 @@ def test_bf16x3_router_gemm_matches_reference(
|
||||
# Match the observed router weight scale
|
||||
w *= 0.053
|
||||
out = bf16x3_router_gemm(x, w)
|
||||
ref = torch.nn.functional.linear(x.float(), w)
|
||||
# FP64 reference: the FP32 reference itself drifts by ~5e-6 at N=2048
|
||||
ref = torch.nn.functional.linear(x.double(), w.double())
|
||||
|
||||
assert out.shape == (num_tokens, num_experts)
|
||||
assert out.dtype == torch.float32
|
||||
assert torch.mean(torch.abs(out - ref)).item() < 5e-6
|
||||
assert torch.mean(torch.abs(out.double() - ref)).item() < 5e-6
|
||||
|
||||
Reference in New Issue
Block a user