[Kernel] Add router GEMM kernels for (5120, 256) and BF16 output to ll_bf16_gemm (#62042)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Lain <fusiyuan2000@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-10-16 15:39:46 -07:00
committed by GitHub
co-authored by Lain Codex Claude Opus 5.5 vllm-agent
parent 153a17166e
commit 596e530675
12 changed files with 258 additions and 64 deletions
+49 -12
View File
@@ -3,9 +3,9 @@
"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
Supported (hidden_size, num_experts) pairs:
(3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
(3072, 256), (6144, 128), (5120, 256)
Correctness baseline: F.linear in float32. Every M in [1, 32] is covered so
Correctness baseline: F.linear in float32. Every supported M is covered so
all tuned geometries (wide-block, experts-per-block, token-group; boundaries
at M=4/5, odd/even, M=15/16) are exercised on Blackwell, and the legacy
128/1 geometry everywhere else.
@@ -18,8 +18,12 @@ from vllm._custom_ops import fp32_router_gemm
from vllm.platforms import current_platform
# (hidden_size, num_experts)
SHAPES = [(3072, 256), (6144, 128)]
ALL_M = list(range(1, 33))
SHAPES = [(3072, 256), (6144, 128), (5120, 256)]
CASES = [
(m, hidden, experts)
for hidden, experts in SHAPES
for m in range(1, 49 if hidden == 5120 else 33)
]
# Absolute tolerance for fp32 kernel vs float64 reference
ATOL_FP32 = 2e-4
ATOL_BF16 = 2e-2 # bf16 activation has lower precision
@@ -40,8 +44,7 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
return torch.nn.functional.linear(mat_a.float(), mat_b.float())
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", ALL_M)
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""fp32 activation → fp32 output should match reference closely."""
_requires_sm90()
@@ -58,8 +61,7 @@ def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", ALL_M)
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""bf16 activation → fp32 output should match reference within bf16 error."""
_requires_sm90()
@@ -136,17 +138,19 @@ def test_zero_tokens_returns_empty():
assert out.dtype == torch.float32
def test_rejects_invalid_inputs():
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
def test_rejects_invalid_inputs(hidden_dim, num_experts):
"""The entry must fail loudly, never compute silently wrong results."""
_requires_sm90()
device = torch.device("cuda")
hidden_dim, num_experts = SHAPES[0]
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
# num_tokens > 32 (beyond the instantiated range)
# Beyond the instantiated range for this shape.
invalid_tokens = 49 if hidden_dim == 5120 else 33
with pytest.raises(Exception, match="num_tokens"):
fp32_router_gemm(
torch.randn(33, hidden_dim, dtype=torch.float32, device=device), mat_b
torch.randn(invalid_tokens, hidden_dim, dtype=torch.float32, device=device),
mat_b,
)
# unsupported (hidden_dim, num_experts) pair
@@ -173,3 +177,36 @@ def test_rejects_invalid_inputs():
fp32_router_gemm(
torch.randn(4, hidden_dim, dtype=torch.float16, device=device), mat_b
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.parametrize(
"tokens", [*range(1, 17), 17, 24, 32, 33, 40, 48, 56, 64, 65, 128]
)
@torch.inference_mode()
def test_router_retains_fp32_weights_and_logits(default_vllm_config, dist_init, tokens):
"""Low-latency routing must retain FP32 weight precision and logits."""
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("SM100 required")
torch.manual_seed(42)
router = GateLinear(
5120, 256, params_dtype=torch.float32, out_dtype=torch.float32
).cuda()
router.weight.normal_(std=0.01)
x = torch.randn(tokens, 5120, device="cuda", dtype=torch.bfloat16)
expected = (x.double() @ router.weight.double().T).float()
actual, bias = router(x)
assert bias is None and actual.dtype == torch.float32
torch.testing.assert_close(actual, expected, rtol=2e-5, atol=2e-5)
torch.testing.assert_close(actual.topk(8).indices, expected.topk(8).indices)
if tokens in (1, 16, 48, 56):
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured, _ = router(x)
x.normal_()
graph.replay()
expected = (x.double() @ router.weight.double().T).float()
torch.testing.assert_close(captured, expected, rtol=2e-5, atol=2e-5)