From ca4cfd873163cc1911f98385e776fba6e7300f9f Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Wed, 10 Jun 2026 04:55:30 +0800 Subject: [PATCH] [Bugfix] fix qwen3.5 ep weight loading (#45002) Signed-off-by: zjy0516 --- .../gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml | 1 + vllm/model_executor/models/qwen3_5.py | 24 +++++++++++++------ vllm/model_executor/models/qwen3_5_mtp.py | 21 ++++++++++------ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml b/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml index 55a134ad9b..6c2dcad0e6 100644 --- a/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml +++ b/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml @@ -7,3 +7,4 @@ server_args: >- --max-model-len 4096 --data-parallel-size 2 --enable-expert-parallel + --no-enable-flashinfer-autotune \ No newline at end of file diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 95f6656523..43b9004638 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -36,6 +36,9 @@ from vllm.distributed import ( get_pp_group, ) from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import ( GemmaRMSNorm as Qwen3_5RMSNorm, ) @@ -294,13 +297,20 @@ class Qwen3_5Model(Qwen3NextModel): loaded_params: set[str] = set() expert_params_mapping = self.get_expert_mapping() is_fused_expert = False - base_layer = ( - "base_layer." if any(".base_layer." in name for name in params_dict) else "" - ) - fused_expert_params_mapping = [ - (f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"), - (f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"), - ] + fused_expert_params_mapping: list[tuple[str, str, int, str]] = [] + for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_up_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="gate_up_proj", + num_experts=1, + ): + if shard_id == "w3": + continue + parts = ckpt_name.split(".") + fused_expert_params_mapping.append( + (f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id) + ) num_experts = ( self.config.num_experts if hasattr(self.config, "num_experts") else 0 ) diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 0f76f3f5a2..7dd478d424 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -209,13 +209,20 @@ class Qwen3_5MultiTokenPredictor(nn.Module): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() is_fused_expert = False - base_layer = ( - "base_layer." if any(".base_layer." in name for name in params_dict) else "" - ) - fused_expert_params_mapping = [ - (f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"), - (f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"), - ] + fused_expert_params_mapping: list[tuple[str, str, int, str]] = [] + for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_up_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="gate_up_proj", + num_experts=1, + ): + if shard_id == "w3": + continue + parts = ckpt_name.split(".") + fused_expert_params_mapping.append( + (f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id) + ) num_experts = ( self.config.num_experts if hasattr(self.config, "num_experts") else 0 )