[Model][Gemma4] Load Weights with AutoWeightsLoader (#55911)
Signed-off-by: Josephasafg <ajgard7@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Harry Mellor
Claude Opus 5
parent
2b8a5495a8
commit
44dd18fe0b
@@ -147,10 +147,10 @@ def test_gemma4_lora_weights_mapping():
|
||||
def test_gemma4_moe_lora_weights_mapping():
|
||||
mapper = Gemma4ForCausalLM.hf_to_vllm_mapper
|
||||
name = (
|
||||
"base_model.model.model.language_model.layers.9.moe.experts."
|
||||
"base_model.model.model.language_model.layers.9.experts."
|
||||
"gate_up_proj.lora_B.weight"
|
||||
)
|
||||
assert parse_fine_tuned_lora_name(name, mapper) == (
|
||||
"model.layers.9.moe.gate_up_proj",
|
||||
"model.layers.9.experts.gate_up_proj",
|
||||
False,
|
||||
)
|
||||
|
||||
@@ -14,7 +14,10 @@ from vllm.model_executor.models.gemma3n import (
|
||||
Gemma3nTextModel,
|
||||
_kv_sharing_weights_mapper,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4 import Gemma4Model
|
||||
from vllm.model_executor.models.gemma4 import (
|
||||
Gemma4ForCausalLM,
|
||||
_gemma4_layer_weights_mapper,
|
||||
)
|
||||
|
||||
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
|
||||
|
||||
@@ -57,36 +60,70 @@ def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
|
||||
"""KV-shared layers have q_proj instead of a packed qkv_proj; their
|
||||
redundant K/V tensors in original checkpoints have no parameter and are
|
||||
skipped rather than failing the load."""
|
||||
model = torch.nn.Module()
|
||||
model.config = SimpleNamespace(num_experts=0)
|
||||
model.start_layer, model.end_layer = 0, 2
|
||||
model.layers = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
|
||||
for layer in model.layers:
|
||||
layer.self_attn = torch.nn.Module()
|
||||
model.layers[0].self_attn.qkv_proj = torch.nn.Linear(2, 6, bias=False)
|
||||
model.layers[1].self_attn.q_proj = torch.nn.Linear(2, 2, bias=False)
|
||||
shards: list[str] = []
|
||||
model.layers[0].self_attn.qkv_proj.weight.weight_loader = (
|
||||
lambda param, weight, shard_id: shards.append(shard_id)
|
||||
def test_gemma4_attention_mapper() -> None:
|
||||
"""Layers with a qkv_proj pack q/k/v; `attention_k_eq_v` full-attention
|
||||
layers also load K as the V shard, and leave any v_proj such a checkpoint
|
||||
ships unmapped so it fails the load rather than silently overwriting V;
|
||||
KV-shared layers keep q_proj and drop the K/V tensors original checkpoints
|
||||
still ship for them."""
|
||||
config = SimpleNamespace(
|
||||
num_hidden_layers=3,
|
||||
num_kv_shared_layers=1,
|
||||
attention_k_eq_v=True,
|
||||
layer_types=["sliding_attention", "full_attention", "sliding_attention"],
|
||||
)
|
||||
|
||||
q_weight = torch.full((2, 2), 2.0)
|
||||
weights = [
|
||||
(f"layers.{i}.self_attn.{tensor}.weight", q_weight)
|
||||
for i in (0, 1)
|
||||
for tensor in ("q_proj", "k_proj", "v_proj")
|
||||
] + [("layers.1.self_attn.k_norm.weight", torch.ones(2))]
|
||||
loaded = Gemma4Model.load_weights(cast(Gemma4Model, model), weights)
|
||||
(f"model.layers.{i}.self_attn.{tensor}.weight", torch.full((2, 2), i + 1.0))
|
||||
for i in range(3)
|
||||
for tensor in ("q_proj", "k_proj", "k_norm")
|
||||
] + [
|
||||
("model.layers.0.mlp.up_proj.weight", torch.empty(0)),
|
||||
("model.layers.1.self_attn.v_proj.weight", torch.empty(0)),
|
||||
]
|
||||
|
||||
assert shards == ["q", "k", "v"]
|
||||
assert "layers.1.self_attn.q_proj.weight" in loaded
|
||||
assert not any(name.startswith("layers.1.self_attn.k") for name in loaded)
|
||||
assert not any(name.startswith("layers.1.self_attn.v") for name in loaded)
|
||||
assert torch.equal(model.layers[1].self_attn.q_proj.weight, q_weight)
|
||||
mapper = _gemma4_layer_weights_mapper(config)
|
||||
mapped = list(mapper.apply(weights))
|
||||
|
||||
assert [(name, getattr(w, "shard_id", None)) for name, w in mapped] == [
|
||||
("model.layers.0.self_attn.qkv_proj.weight", "q"),
|
||||
("model.layers.0.self_attn.qkv_proj.weight", "k"),
|
||||
("model.layers.0.self_attn.k_norm.weight", None),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "q"),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "k"),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "v"),
|
||||
("model.layers.1.self_attn.k_norm.weight", None),
|
||||
("model.layers.2.self_attn.q_proj.weight", None),
|
||||
("model.layers.0.mlp.gate_up_proj.weight", 1),
|
||||
("model.layers.1.self_attn.v_proj.weight", None),
|
||||
]
|
||||
k_weight, v_weight = weights[4][1], mapped[5][1]
|
||||
assert torch.equal(v_weight, k_weight) and v_weight is not k_weight
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma4_expert_names_strip_language_model_prefix() -> None:
|
||||
"""The text-only path reuses the conditional wrapper's checkpoint naming,
|
||||
so fused and per-expert tensors reach the experts under `model.*`."""
|
||||
prefix = "model.language_model.layers.0."
|
||||
weights = [
|
||||
(prefix + name, torch.empty(0))
|
||||
for name in (
|
||||
"experts.gate_up_proj",
|
||||
"experts.3.down_proj.weight_packed",
|
||||
"router.per_expert_scale",
|
||||
)
|
||||
]
|
||||
|
||||
mapped = [
|
||||
(name, getattr(w, "shard_id", None))
|
||||
for name, w in Gemma4ForCausalLM.hf_to_vllm_mapper.apply(weights)
|
||||
]
|
||||
|
||||
assert mapped == [
|
||||
("model.layers.0.experts.gate_up_proj", None),
|
||||
("model.layers.0.experts.3.down_proj.weight_packed", None),
|
||||
("model.layers.0.router.per_expert_scale", None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
|
||||
@@ -217,3 +217,25 @@ def test_get_rename_mapper_keeps_only_renames():
|
||||
for name in ("drop_regex.w", "drop_substr.w", "drop_prefix.w", "w.drop_suffix"):
|
||||
assert mapper._map_name(name) is None
|
||||
assert renames._map_name(name) == name
|
||||
|
||||
|
||||
def test_weights_mapper_stacks_one_weight_into_several_shards():
|
||||
"""One checkpoint tensor can feed more than one shard of a stacked
|
||||
parameter, and a dropped name stays dropped rather than reappearing as a
|
||||
shard of the stacked one."""
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_substr={"layers.1.k_proj.": None},
|
||||
orig_to_new_stacked={".k_proj.": (".qkv_proj.", ["k", "v"])},
|
||||
)
|
||||
weight = torch.ones(2)
|
||||
weights = [(f"layers.{i}.k_proj.weight", weight) for i in (0, 1)]
|
||||
|
||||
mapped = list(mapper.apply(weights))
|
||||
|
||||
assert [(name, w.shard_id) for name, w in mapped] == [
|
||||
("layers.0.qkv_proj.weight", "k"),
|
||||
("layers.0.qkv_proj.weight", "v"),
|
||||
]
|
||||
# Each shard needs its own tensor object, but they alias one allocation.
|
||||
assert mapped[0][1] is not mapped[1][1]
|
||||
assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr() == weight.data_ptr()
|
||||
|
||||
@@ -341,7 +341,7 @@ def test_modelopt_mixed_precision_composes_gemma4_mappers():
|
||||
"quant_algo": "NVFP4",
|
||||
"group_size": 16,
|
||||
},
|
||||
"model.language_model.layers.1.moe.experts.gate_up_proj": {
|
||||
"model.language_model.layers.1.experts.gate_up_proj": {
|
||||
"quant_algo": "NVFP4",
|
||||
"group_size": 16,
|
||||
},
|
||||
@@ -353,10 +353,10 @@ def test_modelopt_mixed_precision_composes_gemma4_mappers():
|
||||
)
|
||||
config.apply_vllm_mapper(Gemma4ForCausalLM.hf_to_vllm_mapper.get_rename_mapper())
|
||||
|
||||
expected_prefix = "language_model.model.layers.0.moe.experts"
|
||||
expected_prefix = "language_model.model.layers.0.experts"
|
||||
assert set(config.quantized_layers) == {
|
||||
expected_prefix,
|
||||
"language_model.model.layers.1.moe.gate_up_proj",
|
||||
"language_model.model.layers.1.experts.gate_up_proj",
|
||||
}
|
||||
assert config._resolve_quant_algo(expected_prefix) == "NVFP4"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user