[BugFix] Fix Gemma4 'layers.0.moe.experts.0.down_proj_packed' KeyError issue (#40708)
Signed-off-by: SoluMilken <ypiheyn.imm02g@g2.nctu.edu.tw>
This commit is contained in:
@@ -93,6 +93,37 @@ def run_awq_test(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "quantization", "dtype"),
|
||||
[
|
||||
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
|
||||
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
|
||||
],
|
||||
ids=[
|
||||
"gemma4-moe-standard-awq-dot-suffix",
|
||||
"gemma4-moe-compressed-tensors-underscore-suffix",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_awq_load(
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: str,
|
||||
quantization: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Regression test: AWQ weight loading must not KeyError."""
|
||||
with vllm_runner(
|
||||
model,
|
||||
quantization=quantization,
|
||||
dtype=dtype,
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source_model", "quant_model"),
|
||||
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -1368,30 +1369,35 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
# moe.experts.{id}.gate_proj → FusedMoE w1 (shard of w13)
|
||||
# moe.experts.{id}.up_proj → FusedMoE w3 (shard of w13)
|
||||
# moe.experts.{id}.down_proj → FusedMoE w2
|
||||
#
|
||||
# Use prefix matching to handle both weights and
|
||||
# quantization scale parameters. The param_name is a prefix ending
|
||||
# in underscore, and weight_name ends with a dot, so that:
|
||||
# "experts.0.gate_proj.weight_scale" -> "experts.w13_weight_scale"
|
||||
# "experts.0.gate_proj.weight" -> "experts.w13_weight"
|
||||
num_experts = getattr(self.config, "num_experts", None) or 0
|
||||
expert_params_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
# Strategy A: dot-separated suffix
|
||||
# (standard AWQ/GPTQ e.g. .qweight, .scales, .weight)
|
||||
dot_suffix_expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=num_experts,
|
||||
)
|
||||
# Strategy B: underscore-separated suffix
|
||||
# (CompressedTensors-format AWQ/W4A16 _packed, _scale)
|
||||
underscore_suffix_expert_params_mapping = [
|
||||
(
|
||||
"experts.w13_"
|
||||
if proj_name in ["gate_proj", "up_proj"]
|
||||
else "experts.w2_",
|
||||
f"experts.{expert_id}.{proj_name}.",
|
||||
f"{param_name}weight_",
|
||||
f"{weight_name.rstrip('.')}_",
|
||||
expert_id,
|
||||
shard_id,
|
||||
)
|
||||
for expert_id in range(num_experts)
|
||||
for shard_id, proj_name in [
|
||||
("w1", "gate_proj"),
|
||||
("w2", "down_proj"),
|
||||
("w3", "up_proj"),
|
||||
]
|
||||
for (
|
||||
param_name,
|
||||
weight_name,
|
||||
expert_id,
|
||||
shard_id,
|
||||
) in dot_suffix_expert_params_mapping
|
||||
]
|
||||
expert_params_mapping = (
|
||||
dot_suffix_expert_params_mapping + underscore_suffix_expert_params_mapping
|
||||
)
|
||||
params_dict = dict(self.named_parameters())
|
||||
# Include buffers (e.g. layer_scalar) so they can be loaded too
|
||||
params_dict.update(dict(self.named_buffers()))
|
||||
|
||||
Reference in New Issue
Block a user