[Model][Gemma4] Load Weights with AutoWeightsLoader (#55911)

Signed-off-by: Josephasafg <ajgard7@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Asaf Gardin
2026-09-18 12:07:30 +00:00
committed by GitHub
co-authored by Harry Mellor Claude Opus 5
parent 2b8a5495a8
commit 44dd18fe0b
8 changed files with 239 additions and 518 deletions
+65 -28
View File
@@ -14,7 +14,10 @@ from vllm.model_executor.models.gemma3n import (
Gemma3nTextModel,
_kv_sharing_weights_mapper,
)
from vllm.model_executor.models.gemma4 import Gemma4Model
from vllm.model_executor.models.gemma4 import (
Gemma4ForCausalLM,
_gemma4_layer_weights_mapper,
)
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
@@ -57,36 +60,70 @@ def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
@pytest.mark.cpu_test
def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
"""KV-shared layers have q_proj instead of a packed qkv_proj; their
redundant K/V tensors in original checkpoints have no parameter and are
skipped rather than failing the load."""
model = torch.nn.Module()
model.config = SimpleNamespace(num_experts=0)
model.start_layer, model.end_layer = 0, 2
model.layers = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
for layer in model.layers:
layer.self_attn = torch.nn.Module()
model.layers[0].self_attn.qkv_proj = torch.nn.Linear(2, 6, bias=False)
model.layers[1].self_attn.q_proj = torch.nn.Linear(2, 2, bias=False)
shards: list[str] = []
model.layers[0].self_attn.qkv_proj.weight.weight_loader = (
lambda param, weight, shard_id: shards.append(shard_id)
def test_gemma4_attention_mapper() -> None:
"""Layers with a qkv_proj pack q/k/v; `attention_k_eq_v` full-attention
layers also load K as the V shard, and leave any v_proj such a checkpoint
ships unmapped so it fails the load rather than silently overwriting V;
KV-shared layers keep q_proj and drop the K/V tensors original checkpoints
still ship for them."""
config = SimpleNamespace(
num_hidden_layers=3,
num_kv_shared_layers=1,
attention_k_eq_v=True,
layer_types=["sliding_attention", "full_attention", "sliding_attention"],
)
q_weight = torch.full((2, 2), 2.0)
weights = [
(f"layers.{i}.self_attn.{tensor}.weight", q_weight)
for i in (0, 1)
for tensor in ("q_proj", "k_proj", "v_proj")
] + [("layers.1.self_attn.k_norm.weight", torch.ones(2))]
loaded = Gemma4Model.load_weights(cast(Gemma4Model, model), weights)
(f"model.layers.{i}.self_attn.{tensor}.weight", torch.full((2, 2), i + 1.0))
for i in range(3)
for tensor in ("q_proj", "k_proj", "k_norm")
] + [
("model.layers.0.mlp.up_proj.weight", torch.empty(0)),
("model.layers.1.self_attn.v_proj.weight", torch.empty(0)),
]
assert shards == ["q", "k", "v"]
assert "layers.1.self_attn.q_proj.weight" in loaded
assert not any(name.startswith("layers.1.self_attn.k") for name in loaded)
assert not any(name.startswith("layers.1.self_attn.v") for name in loaded)
assert torch.equal(model.layers[1].self_attn.q_proj.weight, q_weight)
mapper = _gemma4_layer_weights_mapper(config)
mapped = list(mapper.apply(weights))
assert [(name, getattr(w, "shard_id", None)) for name, w in mapped] == [
("model.layers.0.self_attn.qkv_proj.weight", "q"),
("model.layers.0.self_attn.qkv_proj.weight", "k"),
("model.layers.0.self_attn.k_norm.weight", None),
("model.layers.1.self_attn.qkv_proj.weight", "q"),
("model.layers.1.self_attn.qkv_proj.weight", "k"),
("model.layers.1.self_attn.qkv_proj.weight", "v"),
("model.layers.1.self_attn.k_norm.weight", None),
("model.layers.2.self_attn.q_proj.weight", None),
("model.layers.0.mlp.gate_up_proj.weight", 1),
("model.layers.1.self_attn.v_proj.weight", None),
]
k_weight, v_weight = weights[4][1], mapped[5][1]
assert torch.equal(v_weight, k_weight) and v_weight is not k_weight
@pytest.mark.cpu_test
def test_gemma4_expert_names_strip_language_model_prefix() -> None:
"""The text-only path reuses the conditional wrapper's checkpoint naming,
so fused and per-expert tensors reach the experts under `model.*`."""
prefix = "model.language_model.layers.0."
weights = [
(prefix + name, torch.empty(0))
for name in (
"experts.gate_up_proj",
"experts.3.down_proj.weight_packed",
"router.per_expert_scale",
)
]
mapped = [
(name, getattr(w, "shard_id", None))
for name, w in Gemma4ForCausalLM.hf_to_vllm_mapper.apply(weights)
]
assert mapped == [
("model.layers.0.experts.gate_up_proj", None),
("model.layers.0.experts.3.down_proj.weight_packed", None),
("model.layers.0.router.per_expert_scale", None),
]
@pytest.mark.cpu_test
+22
View File
@@ -217,3 +217,25 @@ def test_get_rename_mapper_keeps_only_renames():
for name in ("drop_regex.w", "drop_substr.w", "drop_prefix.w", "w.drop_suffix"):
assert mapper._map_name(name) is None
assert renames._map_name(name) == name
def test_weights_mapper_stacks_one_weight_into_several_shards():
"""One checkpoint tensor can feed more than one shard of a stacked
parameter, and a dropped name stays dropped rather than reappearing as a
shard of the stacked one."""
mapper = WeightsMapper(
orig_to_new_substr={"layers.1.k_proj.": None},
orig_to_new_stacked={".k_proj.": (".qkv_proj.", ["k", "v"])},
)
weight = torch.ones(2)
weights = [(f"layers.{i}.k_proj.weight", weight) for i in (0, 1)]
mapped = list(mapper.apply(weights))
assert [(name, w.shard_id) for name, w in mapped] == [
("layers.0.qkv_proj.weight", "k"),
("layers.0.qkv_proj.weight", "v"),
]
# Each shard needs its own tensor object, but they alias one allocation.
assert mapped[0][1] is not mapped[1][1]
assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr() == weight.data_ptr()