From 3895a8babee2aed49c203f4214c119c393a867c0 Mon Sep 17 00:00:00 2001 From: zexplorerhj Date: Thu, 15 Oct 2026 01:09:23 +0800 Subject: [PATCH] [Bugfix] Fix TP loading of small block-FP8 MLA gates in BailingMoeV3 (#61642) Signed-off-by: zexplorerhj Co-authored-by: Codex --- vllm/model_executor/models/bailing_moe_v3.py | 30 ++++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/models/bailing_moe_v3.py b/vllm/model_executor/models/bailing_moe_v3.py index d82e4c1b8b..fdf7856e24 100644 --- a/vllm/model_executor/models/bailing_moe_v3.py +++ b/vllm/model_executor/models/bailing_moe_v3.py @@ -538,6 +538,26 @@ class BailingMoeV3MLAAttention(nn.Module): g_out = self.num_heads * self.v_head_dim else: g_out = 0 + block_size = ( + quant_config.weight_block_size + if _is_block_fp8_config(quant_config) + else None + ) + self.replicate_g_proj = ( + self.gated_attention_proj_granularity_type == "head_wise" + and tp_size > 1 + and block_size is not None + and g_out <= block_size[0] + and not _is_fp8_module_excluded(quant_config, f"{prefix}.g_proj") + and self.num_local_heads % block_size[0] != 0 + ) + # A gate spanning at most one FP8 row block cannot split its scales + # across TP ranks. Replicate it and select each rank's heads instead. + self.g_proj_start = ( + get_tensor_model_parallel_rank() * self.num_local_heads + if self.replicate_g_proj + else 0 + ) self.g_proj = ( ColumnParallelLinear( self.hidden_size, @@ -545,6 +565,7 @@ class BailingMoeV3MLAAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.g_proj", + disable_tp=self.replicate_g_proj, ) if g_out > 0 else None @@ -593,9 +614,12 @@ class BailingMoeV3MLAAttention(nn.Module): def forward(self, hidden_states: torch.Tensor, positions: torch.Tensor): attn_out = self.mla_attn(positions, hidden_states) if self.g_proj is not None: - gate = torch.sigmoid(self.g_proj(hidden_states)[0].float()).to( - hidden_states.dtype - ) + gate = self.g_proj(hidden_states)[0] + if self.replicate_g_proj: + gate = gate[ + ..., self.g_proj_start : self.g_proj_start + self.num_local_heads + ] + gate = torch.sigmoid(gate.float()).to(hidden_states.dtype) if self.gated_attention_proj_granularity_type == "head_wise": attn_out = attn_out.view(-1, self.num_local_heads, self.v_head_dim) attn_out = attn_out * gate.unsqueeze(-1)