From 178fd5609427d4da1ca901ec0e78364d4d1c6d15 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhang Date: Thu, 2 Jul 2026 22:45:39 +0800 Subject: [PATCH] support GLM-5.2 gate use FP32 (#47410) Signed-off-by: zRzRzRzRzRzRzR --- .../layers/fused_moe/router/gate_linear.py | 1 + vllm/model_executor/models/deepseek_v2.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/vllm/model_executor/layers/fused_moe/router/gate_linear.py b/vllm/model_executor/layers/fused_moe/router/gate_linear.py index 63e40ab9b5..b11464b19d 100644 --- a/vllm/model_executor/layers/fused_moe/router/gate_linear.py +++ b/vllm/model_executor/layers/fused_moe/router/gate_linear.py @@ -74,6 +74,7 @@ class GateLinear(ReplicatedLinear): self.allow_specialized_router_gemm = can_use_specialized_kernels self.allow_dsv3_router_gemm = ( self.allow_specialized_router_gemm + and self.weight.dtype == torch.bfloat16 and output_size in self.DSV3_SUPPORTED_NUM_EXPERTS and input_size in self.DSV3_SUPPORTED_HIDDEN_SIZES and (input_size, output_size) not in self.DSV3_UNSUPPORTED_SHAPES diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 9530ac58b6..b20db76692 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -117,6 +117,19 @@ from .utils import ( logger = init_logger(__name__) +def _get_moe_router_dtype( + config: DeepseekV2Config | DeepseekV3Config, +) -> torch.dtype | None: + router_dtype = getattr(config, "moe_router_dtype", None) + if getattr(config, "model_type", None) == "glm_moe_dsa": + # Older GLM-5/5.2 configs require fp32 routing but do not expose + # moe_router_dtype yet. + return torch.float32 + if router_dtype == "float32": + return torch.float32 + return None + + class DeepseekAttention(nn.Module): """Normal MHA implementation used by Deepseek v1.""" @@ -288,9 +301,13 @@ class DeepseekV2MoE(nn.Module): "Only silu is supported for now." ) + self.router_dtype = _get_moe_router_dtype(config) self.gate = GateLinear( config.hidden_size, config.n_routed_experts, + params_dtype=self.router_dtype, + out_dtype=self.router_dtype, + force_fp32_compute=self.router_dtype == torch.float32, prefix=f"{prefix}.gate", ) if getattr(config, "topk_method", None) == "noaux_tc": @@ -321,6 +338,7 @@ class DeepseekV2MoE(nn.Module): if ( self.is_rocm_aiter_moe_enabled and self.gate.e_score_correction_bias is not None + and self.gate.out_dtype is None ): # Accumulates in fp32; avoids bf16->fp32 cast. self.gate.set_out_dtype(self.gate.weight.dtype)