FP32 router GEMV optimization (#48335)

Signed-off-by: peiyuanz <peiyuanz@inferact.ai>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: peiyuanz <peiyuanz@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: zhouzhou <zhouzhou@zhouzhoudeMacBook-Pro.local>
This commit is contained in:
Jee Jee Li
2026-07-11 13:07:48 +00:00
committed by GitHub
co-authored by peiyuanz Claude zhouzhou
parent 1bd8f80a64
commit 19069bcbd5
4 changed files with 284 additions and 67 deletions
@@ -15,8 +15,8 @@ class GateLinear(ReplicatedLinear):
"""MoE gate linear layer with multi-tier GEMM dispatch:
1. DSV3 specialized kernel (SM90+, M<=16, H=7168 E=256/384, H=6144 E=256)
2. fp32 specialized kernel (SM90+, bf16/fp32 in, fp32 out,
M<=32, H=3072, E=256)
2. fp32 specialized kernel (SM90+, bf16/fp32 in, fp32 out, M<=32,
(H, E) in {(3072, 256), (6144, 128), (6144, 256)})
3. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
4. F.linear via ReplicatedLinear (ultimate fallback)
@@ -36,7 +36,7 @@ class GateLinear(ReplicatedLinear):
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
# (3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128)}
FP32_SUPPORTED_SHAPES = {(3072, 256), (6144, 128), (6144, 256)}
FP32_MAX_TOKENS = 32
def __init__(