From 25abddc1a5cb5dbe51d629aada9360579acb8acf Mon Sep 17 00:00:00 2001 From: SoluMilken Date: Sun, 10 May 2026 01:20:44 +0800 Subject: [PATCH] [BugFix] Fix Gemma4 'layers.0.moe.experts.0.down_proj_packed' KeyError issue (#40708) Signed-off-by: SoluMilken --- tests/models/quantization/test_awq.py | 31 ++++++++++++++++++++ vllm/model_executor/models/gemma4.py | 42 +++++++++++++++------------ 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/tests/models/quantization/test_awq.py b/tests/models/quantization/test_awq.py index 6b34262d3e..25a63f6bd9 100644 --- a/tests/models/quantization/test_awq.py +++ b/tests/models/quantization/test_awq.py @@ -93,6 +93,37 @@ def run_awq_test( ) +@pytest.mark.parametrize( + ("model", "quantization", "dtype"), + [ + ("mattbucci/gemma-4-26B-AWQ", "awq", "float16"), + ("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"), + ], + ids=[ + "gemma4-moe-standard-awq-dot-suffix", + "gemma4-moe-compressed-tensors-underscore-suffix", + ], +) +@torch.inference_mode() +def test_awq_load( + vllm_runner: type[VllmRunner], + example_prompts: list[str], + model: str, + quantization: str, + dtype: str, +) -> None: + """Regression test: AWQ weight loading must not KeyError.""" + with vllm_runner( + model, + quantization=quantization, + dtype=dtype, + max_model_len=128, + enforce_eager=True, + ) as vllm_model: + outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32) + assert len(outputs) == 2 + + @pytest.mark.parametrize( ("source_model", "quant_model"), [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")], diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index 62b7f55766..1b50db61c2 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -40,6 +40,7 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, GateLinear, + fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -1368,30 +1369,35 @@ class Gemma4Model(nn.Module, EagleModelMixin): # moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13) # moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13) # moe.experts.{id}.down_proj → FusedMoE w2 - # - # Use prefix matching to handle both weights and - # quantization scale parameters. The param_name is a prefix ending - # in underscore, and weight_name ends with a dot, so that: - # "experts.0.gate_proj.weight_scale" -> "experts.w13_weight_scale" - # "experts.0.gate_proj.weight" -> "experts.w13_weight" num_experts = getattr(self.config, "num_experts", None) or 0 - expert_params_mapping = [ - # (param_name, weight_name, expert_id, shard_id) + # Strategy A: dot-separated suffix + # (standard AWQ/GPTQ e.g. .qweight, .scales, .weight) + dot_suffix_expert_params_mapping = fused_moe_make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=num_experts, + ) + # Strategy B: underscore-separated suffix + # (CompressedTensors-format AWQ/W4A16 _packed, _scale) + underscore_suffix_expert_params_mapping = [ ( - "experts.w13_" - if proj_name in ["gate_proj", "up_proj"] - else "experts.w2_", - f"experts.{expert_id}.{proj_name}.", + f"{param_name}weight_", + f"{weight_name.rstrip('.')}_", expert_id, shard_id, ) - for expert_id in range(num_experts) - for shard_id, proj_name in [ - ("w1", "gate_proj"), - ("w2", "down_proj"), - ("w3", "up_proj"), - ] + for ( + param_name, + weight_name, + expert_id, + shard_id, + ) in dot_suffix_expert_params_mapping ] + expert_params_mapping = ( + dot_suffix_expert_params_mapping + underscore_suffix_expert_params_mapping + ) params_dict = dict(self.named_parameters()) # Include buffers (e.g. layer_scalar) so they can be loaded too params_dict.update(dict(self.named_buffers()))