[Model][Gemma4] Load Weights with AutoWeightsLoader (#55911)

Signed-off-by: Josephasafg <ajgard7@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Asaf Gardin
2026-09-18 12:07:30 +00:00
committed by GitHub
co-authored by Harry Mellor Claude Opus 5
parent 2b8a5495a8
commit 44dd18fe0b
8 changed files with 239 additions and 518 deletions
+11 -110
View File
@@ -46,7 +46,11 @@ from vllm.model_executor.models.gemma4_mm import (
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.transformers.utils import recursive_replace_linear
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.utils.torch_utils import async_tensor_h2d
@@ -160,14 +164,12 @@ class DiffusionGemmaForConditionalGeneration(
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"model.decoder.self_conditioning.": "self_conditioning.",
"model.decoder.": "model.",
"model.encoder.language_model.": "model.",
"model.encoder.vision_tower.": "vision_tower.",
"model.encoder.embed_vision.": "embed_vision.",
},
orig_to_new_substr={
".experts.": ".moe.experts.",
},
)
packed_modules_mapping = {
@@ -336,113 +338,12 @@ class DiffusionGemmaForConditionalGeneration(
logits = _softcap_logits(logits, self.final_logit_softcapping)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
"""Load weights from checkpoint.
Checkpoint layout (HF DiffusionGemma):
model.encoder.vision_tower.* → vision tower
model.encoder.embed_vision.* → vision embedder
model.encoder.language_model.layers.* → backbone
model.decoder.layers.* → backbone (tied)
model.decoder.embed_tokens.* → embeddings
model.decoder.self_conditioning.* → self-conditioning MLP
lm_head.* → LM head (tied)
We load encoder weights into our single ``Gemma4Model`` backbone,
skip duplicate decoder backbone weights, handle vision tower and
self-conditioning separately.
"""
sc_params = dict(
(n, p)
for n, p in self.named_parameters()
if n.startswith("self_conditioning.")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Some checkpoints carry a vestigial Gemma3n-style embedding table.
loader = AutoWeightsLoader(
self, ignore_unexpected_prefixes=["embed_vision.embedding."]
)
# Collect vision tower + embedder parameters AND buffers for manual
# loading. The HF vision tower registers std_bias / std_scale as
# buffers (not parameters) when config.standardize is True, so we
# must include named_buffers() to avoid "not found in model" warnings.
vision_params: dict[str, torch.Tensor] = {}
for n, p in self.named_parameters():
if n.startswith(("vision_tower.", "embed_vision.")):
vision_params[n] = p
for n, b in self.named_buffers():
if n.startswith(("vision_tower.", "embed_vision.")):
vision_params[n] = b
def _remap_weights():
# Use full weight names (including suffixes like .weight_scale,
# .weight_packed) for dedup instead of just the base layer name. Critical
# for quantized checkpoints where each weight has multiple tensors;
# tracking only base names skips scales as duplicates.
seen_weights: set[str] = set()
for name, weight in weights:
# Self-conditioning lives under model.decoder.self_conditioning.*
# in the checkpoint but at self_conditioning.* in our model.
if "self_conditioning" in name:
sc_name = name.split("self_conditioning.", 1)[1]
sc_name = "self_conditioning." + sc_name
if sc_name in sc_params:
sc_params[sc_name].data.copy_(weight)
continue
# Vision tower: model.encoder.vision_tower.* → vision_tower.*
# In HF, the vision tower is a sibling of language_model
# under the encoder module.
if name.startswith("model.encoder.vision_tower."):
vt_name = name[len("model.encoder.") :]
if vt_name in vision_params:
vision_params[vt_name].data.copy_(weight)
else:
logger.warning(
"Vision tower weight %s (mapped to %s) not found in model",
name,
vt_name,
)
continue
# Vision embedder: model.encoder.embed_vision.* → embed_vision.*
if name.startswith("model.encoder.embed_vision."):
ev_name = name[len("model.encoder.") :]
if ev_name in vision_params:
vision_params[ev_name].data.copy_(weight)
else:
logger.warning(
"Embed vision weight %s (mapped to %s) not found in model",
name,
ev_name,
)
continue
# Skip vestigial embed_vision.embedding weights.
if "embed_vision.embedding." in name:
continue
# Encoder backbone → model.*
if name.startswith("model.encoder.language_model."):
name = name.replace("model.encoder.language_model.", "model.")
# Decoder backbone → model.* (skip exact duplicates)
elif name.startswith("model.decoder."):
name = name.replace("model.decoder.", "model.")
# Skip only if we've seen the exact same weight name (including scales)
if name in seen_weights:
continue
seen_weights.add(name)
yield name, weight
# Delegate to Gemma4ForCausalLM.load_weights for the backbone,
# which handles stacked params, MoE, k_eq_v, etc.
# Temporarily set self.config to text_config since Gemma4's
# load_weights expects it (e.g. tie_word_embeddings, layer_types).
from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM
saved_config = self.config
self.config = self.model.config
try:
Gemma4ForCausalLM.load_weights(self, _remap_weights())
finally:
self.config = saved_config
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None: