[Hardware][AMD] Enable fused bf16→fp32 router GEMM on ROCm (#50268)

Signed-off-by: Matvei Pashkovskii <mpashkov@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Matvei Pashkovskii
2026-08-12 02:46:13 +00:00
committed by GitHub
co-authored by Claude
parent f97e502969
commit 20727e2841
2 changed files with 112 additions and 3 deletions
@@ -114,9 +114,17 @@ class GateLinear(ReplicatedLinear):
if self.allow_bf16x3_router_gemm:
logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.")
# cuBLAS bf16→fp32 eligibility
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
# epilogue folds the fp32 cast into the GEMM, removing the standalone
# bf16->fp32 copy kernel that otherwise runs before grouped_topk.
# cuBLAS on CUDA (SM90+, via allow_specialized_router_gemm); hipBLASLt on
# ROCm, which supports the same out_dtype epilogue.
self._router_gemm_no_bias = not bias
self.allow_cublas_router_gemm = (
self.allow_specialized_router_gemm
(
self.allow_specialized_router_gemm
or (current_platform.is_rocm() and self._router_gemm_no_bias)
)
and self.weight.dtype == torch.bfloat16
and self.out_dtype == torch.float32
)
@@ -148,7 +156,10 @@ class GateLinear(ReplicatedLinear):
if (
not self.allow_cublas_router_gemm
and self.allow_specialized_router_gemm
and (
self.allow_specialized_router_gemm
or (current_platform.is_rocm() and self._router_gemm_no_bias)
)
and out_dtype == torch.float32
):
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16