[BugFix] Fix Gemma4 'layers.0.moe.experts.0.down_proj_packed' KeyError issue (#40708)

Signed-off-by: SoluMilken <ypiheyn.imm02g@g2.nctu.edu.tw>
This commit is contained in:
SoluMilken
2026-05-09 17:20:44 +00:00
committed by GitHub
parent 171d59ae8d
commit 25abddc1a5
2 changed files with 55 additions and 18 deletions
+31
View File
@@ -93,6 +93,37 @@ def run_awq_test(
)
@pytest.mark.parametrize(
("model", "quantization", "dtype"),
[
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
],
ids=[
"gemma4-moe-standard-awq-dot-suffix",
"gemma4-moe-compressed-tensors-underscore-suffix",
],
)
@torch.inference_mode()
def test_awq_load(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: str,
quantization: str,
dtype: str,
) -> None:
"""Regression test: AWQ weight loading must not KeyError."""
with vllm_runner(
model,
quantization=quantization,
dtype=dtype,
max_model_len=128,
enforce_eager=True,
) as vllm_model:
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
assert len(outputs) == 2
@pytest.mark.parametrize(
("source_model", "quant_model"),
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],