From 9b2f34cad446f73b1699e8236ec0b611a65f48af Mon Sep 17 00:00:00 2001 From: Zyann <62597503+Zyann7@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:25:56 +0800 Subject: [PATCH] [Feature] support bf16 MoE router and mxfp4 MoE for MiMo V2 (#57784) Signed-off-by: Zyann7 <62597503+Zyann7@users.noreply.github.com> Co-authored-by: Abatom <182586866+Abatom@users.noreply.github.com> --- vllm/model_executor/models/mimo_v2.py | 13 ++++++++----- vllm/model_executor/models/mimo_v2_omni.py | 5 ++++- vllm/model_executor/models/qwen3_dflash.py | 9 +++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index be665e7e06..dfbed25655 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -27,6 +27,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -156,14 +157,16 @@ class MiMoV2MoE(nn.Module): dtype = getattr(config, "moe_router_dtype", "float32") self.gate_dtype = str_dtype_to_torch_dtype(dtype) - self.gate = nn.Linear( + self.gate = GateLinear( config.hidden_size, config.n_routed_experts, bias=False, - dtype=self.gate_dtype, + params_dtype=self.gate_dtype, + out_dtype=torch.float32, + prefix=f"{prefix}.gate", ) self.gate.e_score_correction_bias = nn.Parameter( - torch.empty(config.n_routed_experts, dtype=self.gate_dtype) + torch.empty(config.n_routed_experts, dtype=self.gate.out_dtype) ) self.experts = FusedMoEFactory( @@ -182,7 +185,7 @@ class MiMoV2MoE(nn.Module): num_expert_group=config.n_group, topk_group=config.topk_group, scoring_func="sigmoid", - router_logits_dtype=self.gate_dtype, + router_logits_dtype=self.gate.out_dtype, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -198,7 +201,7 @@ class MiMoV2MoE(nn.Module): gate_input = hidden_states.to(self.gate_dtype) else: gate_input = hidden_states - router_logits = self.gate(gate_input) + router_logits, _ = self.gate(gate_input) final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=router_logits ) diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index 91a3177b7f..f55cf44f37 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -52,6 +52,7 @@ from vllm.transformers_utils.processors.mimo_v2_omni import ( from .interfaces import ( MultiModalEmbeddings, + SupportsEagle3, SupportsMultiModal, SupportsPP, SupportsQuant, @@ -1209,7 +1210,9 @@ class MiMoV2OmniDummyInputsBuilder(BaseDummyInputsBuilder[MiMoV2OmniProcessingIn info=MiMoV2OmniProcessingInfo, dummy_inputs=MiMoV2OmniDummyInputsBuilder, ) -class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant): +class MiMoV2OmniForCausalLM( + nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant, SupportsEagle3 +): # To ensure correct weight loading and mapping. hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 5504304c73..1636e3852c 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -168,6 +168,7 @@ class DFlashQwen3Attention(nn.Module): rms_norm_eps: float = 1e-06, attention_bias: bool = False, add_swa_attention_sink_bias: bool = False, + v_scale: float | None = None, sliding_window: int | None = None, causal: bool = False, is_neox_style: bool = True, @@ -193,6 +194,7 @@ class DFlashQwen3Attention(nn.Module): self.q_size = self.num_heads * self.head_dim self.kv_size = self.num_kv_heads * self.head_dim self.scaling = self.head_dim**-0.5 + self.v_scale = v_scale self.qkv_proj = QKVParallelLinear( hidden_size, @@ -264,6 +266,8 @@ class DFlashQwen3Attention(nn.Module): q, k = self.rotary_emb(positions, q, k) + if self.v_scale is not None: + v = v * self.v_scale attn_output = self.attn(q, k, v) output, _ = self.o_proj(attn_output) return output @@ -312,6 +316,7 @@ class DFlashQwen3DecoderLayer(nn.Module): rms_norm_eps=config.rms_norm_eps, attention_bias=getattr(config, "attention_bias", False), add_swa_attention_sink_bias=add_swa_attention_sink_bias, + v_scale=dflash_config.get("attention_value_scale"), sliding_window=sliding_window, causal=causal, is_neox_style=is_neox_style, @@ -633,6 +638,10 @@ class DFlashQwen3Model(nn.Module): if context_slot_mapping is None: return + v_scale = getattr(self.layers[0].self_attn, "v_scale", None) + if v_scale is not None: + all_v.mul_(v_scale) + # --- Per-layer cache insert --- all_k_final = all_k_flat.view(L, num_ctx, nkv, hd) per_layer = isinstance(context_slot_mapping, (list, tuple))