[Bugfix][MoE] Enable cuBLAS out_dtype router GEMM on all CUDA archs (fixes family-120/GB10) (#54048)
Signed-off-by: Chenglun Hu <chenglunhu@gmail.com>
This commit is contained in:
@@ -116,15 +116,17 @@ class GateLinear(ReplicatedLinear):
|
|||||||
|
|
||||||
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
|
# Fused bf16 x bf16 -> fp32 GEMM eligibility. torch.mm's out_dtype
|
||||||
# epilogue folds the fp32 cast into the GEMM, removing the standalone
|
# epilogue folds the fp32 cast into the GEMM, removing the standalone
|
||||||
# bf16->fp32 copy kernel that otherwise runs before grouped_topk.
|
# bf16->fp32 copy kernel that otherwise runs before grouped_topk. This is
|
||||||
# cuBLAS on CUDA (SM90+, via allow_specialized_router_gemm); hipBLASLt on
|
# the plain cuBLAS (CUDA) / hipBLASLt (ROCm) out_dtype epilogue, so it
|
||||||
# ROCm, which supports the same out_dtype epilogue.
|
# applies on any CUDA-alike device (no bias, since torch.mm has no bias
|
||||||
|
# term). The specialized-kernel gate above excludes family-120 Blackwell
|
||||||
|
# (GB10 / DGX Spark), which this tier still covers. See #49921.
|
||||||
self._router_gemm_no_bias = not bias
|
self._router_gemm_no_bias = not bias
|
||||||
|
self._router_gemm_cublas_capable = (
|
||||||
|
current_platform.is_cuda() or current_platform.is_rocm()
|
||||||
|
) and self._router_gemm_no_bias
|
||||||
self.allow_cublas_router_gemm = (
|
self.allow_cublas_router_gemm = (
|
||||||
(
|
self._router_gemm_cublas_capable
|
||||||
self.allow_specialized_router_gemm
|
|
||||||
or (current_platform.is_rocm() and self._router_gemm_no_bias)
|
|
||||||
)
|
|
||||||
and self.weight.dtype == torch.bfloat16
|
and self.weight.dtype == torch.bfloat16
|
||||||
and self.out_dtype == torch.float32
|
and self.out_dtype == torch.float32
|
||||||
)
|
)
|
||||||
@@ -156,10 +158,7 @@ class GateLinear(ReplicatedLinear):
|
|||||||
|
|
||||||
if (
|
if (
|
||||||
not self.allow_cublas_router_gemm
|
not self.allow_cublas_router_gemm
|
||||||
and (
|
and self._router_gemm_cublas_capable
|
||||||
self.allow_specialized_router_gemm
|
|
||||||
or (current_platform.is_rocm() and self._router_gemm_no_bias)
|
|
||||||
)
|
|
||||||
and out_dtype == torch.float32
|
and out_dtype == torch.float32
|
||||||
):
|
):
|
||||||
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16
|
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16
|
||||||
|
|||||||
Reference in New Issue
Block a user