support GLM-5.2 gate use FP32 (#47410)

Signed-off-by: zRzRzRzRzRzRzR <Yuxuan.Zhang2@liverpool.ac.uk>
This commit is contained in:
Yuxuan Zhang
2026-07-02 22:45:39 +08:00
committed by GitHub
parent a47f38f825
commit 178fd56094
2 changed files with 19 additions and 0 deletions
@@ -74,6 +74,7 @@ class GateLinear(ReplicatedLinear):
self.allow_specialized_router_gemm = can_use_specialized_kernels
self.allow_dsv3_router_gemm = (
self.allow_specialized_router_gemm
and self.weight.dtype == torch.bfloat16
and output_size in self.DSV3_SUPPORTED_NUM_EXPERTS
and input_size in self.DSV3_SUPPORTED_HIDDEN_SIZES
and (input_size, output_size) not in self.DSV3_UNSUPPORTED_SHAPES
+18
View File
@@ -117,6 +117,19 @@ from .utils import (
logger = init_logger(__name__)
def _get_moe_router_dtype(
config: DeepseekV2Config | DeepseekV3Config,
) -> torch.dtype | None:
router_dtype = getattr(config, "moe_router_dtype", None)
if getattr(config, "model_type", None) == "glm_moe_dsa":
# Older GLM-5/5.2 configs require fp32 routing but do not expose
# moe_router_dtype yet.
return torch.float32
if router_dtype == "float32":
return torch.float32
return None
class DeepseekAttention(nn.Module):
"""Normal MHA implementation used by Deepseek v1."""
@@ -288,9 +301,13 @@ class DeepseekV2MoE(nn.Module):
"Only silu is supported for now."
)
self.router_dtype = _get_moe_router_dtype(config)
self.gate = GateLinear(
config.hidden_size,
config.n_routed_experts,
params_dtype=self.router_dtype,
out_dtype=self.router_dtype,
force_fp32_compute=self.router_dtype == torch.float32,
prefix=f"{prefix}.gate",
)
if getattr(config, "topk_method", None) == "noaux_tc":
@@ -321,6 +338,7 @@ class DeepseekV2MoE(nn.Module):
if (
self.is_rocm_aiter_moe_enabled
and self.gate.e_score_correction_bias is not None
and self.gate.out_dtype is None
):
# Accumulates in fp32; avoids bf16->fp32 cast.
self.gate.set_out_dtype(self.gate.weight.dtype)