support GLM-5.2 gate use FP32 (#47410)
Signed-off-by: zRzRzRzRzRzRzR <Yuxuan.Zhang2@liverpool.ac.uk>
This commit is contained in:
@@ -74,6 +74,7 @@ class GateLinear(ReplicatedLinear):
|
||||
self.allow_specialized_router_gemm = can_use_specialized_kernels
|
||||
self.allow_dsv3_router_gemm = (
|
||||
self.allow_specialized_router_gemm
|
||||
and self.weight.dtype == torch.bfloat16
|
||||
and output_size in self.DSV3_SUPPORTED_NUM_EXPERTS
|
||||
and input_size in self.DSV3_SUPPORTED_HIDDEN_SIZES
|
||||
and (input_size, output_size) not in self.DSV3_UNSUPPORTED_SHAPES
|
||||
|
||||
@@ -117,6 +117,19 @@ from .utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_moe_router_dtype(
|
||||
config: DeepseekV2Config | DeepseekV3Config,
|
||||
) -> torch.dtype | None:
|
||||
router_dtype = getattr(config, "moe_router_dtype", None)
|
||||
if getattr(config, "model_type", None) == "glm_moe_dsa":
|
||||
# Older GLM-5/5.2 configs require fp32 routing but do not expose
|
||||
# moe_router_dtype yet.
|
||||
return torch.float32
|
||||
if router_dtype == "float32":
|
||||
return torch.float32
|
||||
return None
|
||||
|
||||
|
||||
class DeepseekAttention(nn.Module):
|
||||
"""Normal MHA implementation used by Deepseek v1."""
|
||||
|
||||
@@ -288,9 +301,13 @@ class DeepseekV2MoE(nn.Module):
|
||||
"Only silu is supported for now."
|
||||
)
|
||||
|
||||
self.router_dtype = _get_moe_router_dtype(config)
|
||||
self.gate = GateLinear(
|
||||
config.hidden_size,
|
||||
config.n_routed_experts,
|
||||
params_dtype=self.router_dtype,
|
||||
out_dtype=self.router_dtype,
|
||||
force_fp32_compute=self.router_dtype == torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
if getattr(config, "topk_method", None) == "noaux_tc":
|
||||
@@ -321,6 +338,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
if (
|
||||
self.is_rocm_aiter_moe_enabled
|
||||
and self.gate.e_score_correction_bias is not None
|
||||
and self.gate.out_dtype is None
|
||||
):
|
||||
# Accumulates in fp32; avoids bf16->fp32 cast.
|
||||
self.gate.set_out_dtype(self.gate.weight.dtype)
|
||||
|
||||
Reference in New Issue
Block a user