[Hardware][AMD] Enable fused bf16→fp32 router GEMM on ROCm (#50268)
Signed-off-by: Matvei Pashkovskii <mpashkov@amd.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
f97e502969
commit
20727e2841
@@ -114,9 +114,17 @@ class GateLinear(ReplicatedLinear):
|
||||
if self.allow_bf16x3_router_gemm:
|
||||
logger.info_once("Enabled experimental SM100 BF16x3 router GEMM.")
|
||||
|
||||
# cuBLAS bf16→fp32 eligibility
|
||||
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
|
||||
# epilogue folds the fp32 cast into the GEMM, removing the standalone
|
||||
# bf16->fp32 copy kernel that otherwise runs before grouped_topk.
|
||||
# cuBLAS on CUDA (SM90+, via allow_specialized_router_gemm); hipBLASLt on
|
||||
# ROCm, which supports the same out_dtype epilogue.
|
||||
self._router_gemm_no_bias = not bias
|
||||
self.allow_cublas_router_gemm = (
|
||||
self.allow_specialized_router_gemm
|
||||
(
|
||||
self.allow_specialized_router_gemm
|
||||
or (current_platform.is_rocm() and self._router_gemm_no_bias)
|
||||
)
|
||||
and self.weight.dtype == torch.bfloat16
|
||||
and self.out_dtype == torch.float32
|
||||
)
|
||||
@@ -148,7 +156,10 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
if (
|
||||
not self.allow_cublas_router_gemm
|
||||
and self.allow_specialized_router_gemm
|
||||
and (
|
||||
self.allow_specialized_router_gemm
|
||||
or (current_platform.is_rocm() and self._router_gemm_no_bias)
|
||||
)
|
||||
and out_dtype == torch.float32
|
||||
):
|
||||
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16
|
||||
|
||||
Reference in New Issue
Block a user