[BugFix] Fix Gemma4 'layers.0.moe.experts.0.down_proj_packed' KeyError issue (#40708)

Signed-off-by: SoluMilken <ypiheyn.imm02g@g2.nctu.edu.tw>
This commit is contained in:
SoluMilken
2026-05-09 17:20:44 +00:00
committed by GitHub
parent 171d59ae8d
commit 25abddc1a5
2 changed files with 55 additions and 18 deletions
+31
View File
@@ -93,6 +93,37 @@ def run_awq_test(
)
@pytest.mark.parametrize(
("model", "quantization", "dtype"),
[
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
],
ids=[
"gemma4-moe-standard-awq-dot-suffix",
"gemma4-moe-compressed-tensors-underscore-suffix",
],
)
@torch.inference_mode()
def test_awq_load(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: str,
quantization: str,
dtype: str,
) -> None:
"""Regression test: AWQ weight loading must not KeyError."""
with vllm_runner(
model,
quantization=quantization,
dtype=dtype,
max_model_len=128,
enforce_eager=True,
) as vllm_model:
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
assert len(outputs) == 2
@pytest.mark.parametrize(
("source_model", "quant_model"),
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
+24 -18
View File
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -1368,30 +1369,35 @@ class Gemma4Model(nn.Module, EagleModelMixin):
# moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13)
# moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13)
# moe.experts.{id}.down_proj → FusedMoE w2
#
# Use prefix matching to handle both weights and
# quantization scale parameters. The param_name is a prefix ending
# in underscore, and weight_name ends with a dot, so that:
# "experts.0.gate_proj.weight_scale" -> "experts.w13_weight_scale"
# "experts.0.gate_proj.weight" -> "experts.w13_weight"
num_experts = getattr(self.config, "num_experts", None) or 0
expert_params_mapping = [
# (param_name, weight_name, expert_id, shard_id)
# Strategy A: dot-separated suffix
# (standard AWQ/GPTQ e.g. .qweight, .scales, .weight)
dot_suffix_expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=num_experts,
)
# Strategy B: underscore-separated suffix
# (CompressedTensors-format AWQ/W4A16 _packed, _scale)
underscore_suffix_expert_params_mapping = [
(
"experts.w13_"
if proj_name in ["gate_proj", "up_proj"]
else "experts.w2_",
f"experts.{expert_id}.{proj_name}.",
f"{param_name}weight_",
f"{weight_name.rstrip('.')}_",
expert_id,
shard_id,
)
for expert_id in range(num_experts)
for shard_id, proj_name in [
("w1", "gate_proj"),
("w2", "down_proj"),
("w3", "up_proj"),
]
for (
param_name,
weight_name,
expert_id,
shard_id,
) in dot_suffix_expert_params_mapping
]
expert_params_mapping = (
dot_suffix_expert_params_mapping + underscore_suffix_expert_params_mapping
)
params_dict = dict(self.named_parameters())
# Include buffers (e.g. layer_scalar) so they can be loaded too
params_dict.update(dict(self.named_buffers()))