Fix Gemma 4 for upcoming Transformers version (#49797)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-10 07:48:26 +00:00
committed by GitHub
parent 81840a172f
commit 70b84f0bcb
15 changed files with 569 additions and 105 deletions
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-layer config resolution for Gemma4 and its variants."""
from copy import copy
from transformers import PretrainedConfig
def gemma4_layer_config(
text_config: PretrainedConfig, layer_idx: int
) -> PretrainedConfig:
"""The Gemma4 text config as it applies to one layer.
Gemma4 uses a larger head dimension on its full attention layers than on its
sliding ones, and with `attention_k_eq_v` it uses more KV heads there too.
Transformers >= 5.15.0 says so in the config itself; before that the values
are flat attributes that `layer_types` picks between, so resolve them here.
The result is homogeneous either way, so callers read `head_dim` and
`num_key_value_heads` off it without caring which version they are on.
"""
if getattr(text_config, "is_heterogeneous", False):
return text_config.per_layer_config[layer_idx]
layer = copy(text_config)
if text_config.layer_types[layer_idx] == "full_attention":
global_head_dim = getattr(text_config, "global_head_dim", None)
layer.head_dim = global_head_dim or text_config.head_dim
if getattr(text_config, "attention_k_eq_v", False):
global_kv_heads = getattr(text_config, "num_global_key_value_heads", None)
layer.num_key_value_heads = (
global_kv_heads or text_config.num_key_value_heads
)
return layer
@@ -17,6 +17,7 @@ from vllm.transformers_utils.config import (
ConfigFormat,
get_safetensors_params_metadata,
)
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from vllm.utils.torch_utils import common_broadcastable_dtype
logger = init_logger(__name__)
@@ -27,6 +28,37 @@ class ModelArchConfigConvertorBase:
self.hf_config = hf_config
self.hf_text_config = hf_text_config
def get_per_layer_hf_configs(
self,
) -> list[tuple[PretrainedConfig, PretrainedConfig]] | None:
"""`(hf_config, hf_text_config)` per layer, or `None` if homogeneous.
This is the only place that decides whether a checkpoint is heterogeneous
and how many layers it has, so a convertor can declare variation that
Transformers does not express (see `Gemma4ModelArchConfigConvertor`).
Each pair is converted independently and the results are diffed, so a
convertor that overrides this must not also collapse a varying field in
its getters: the collapse would then be applied per layer and every layer
would report the same value.
"""
config_het = getattr(self.hf_config, "is_heterogeneous", False)
text_het = getattr(self.hf_text_config, "is_heterogeneous", False)
if not (config_het or text_het):
return None
# The text config decides the layer count when it is the heterogeneous one:
# its stack is what vLLM builds attention layers for.
source = self.hf_text_config if text_het else self.hf_config
return [
(
self.hf_config.per_layer_config[i] if config_het else self.hf_config,
self.hf_text_config.per_layer_config[i]
if text_het
else self.hf_text_config,
)
for i in range(len(source.per_layer_config))
]
def get_architectures(self) -> list[str]:
# Sometimes we get here from `vllm_config.with_hf_config(text_config)` where
# `text_config` is a sub-config from a multi-modal model. If this is the case,
@@ -382,6 +414,35 @@ class ModelArchConfigConvertorBase:
return derived_max_model_len, max_len_key
def convert(self, supports_multimodal: bool = True) -> ModelArchitectureConfig:
if (per_layer := self.get_per_layer_hf_configs()) is None:
return self.convert_layer(supports_multimodal)
if self.is_deepseek_mla():
raise NotImplementedError(
"Heterogeneous MLA models are not supported: `get_head_size` "
"patches the config in place for them, which a per-layer "
"conversion would apply to a throwaway copy."
)
# Convert each layer and let the result work out which fields differ.
# Reading a varying attribute off the global config would raise, so the
# whole-model config has to be built up from the layers.
return ModelArchitectureConfig.from_layers(
[
type(self)(*configs).convert_layer(supports_multimodal)
for configs in per_layer
]
)
def convert_layer(
self, supports_multimodal: bool = True
) -> ModelArchitectureConfig:
"""Convert one homogeneous config, without resolving per-layer values.
`convert` calls this once per layer, so it must not recurse back into
per-layer resolution: the configs it receives are already layer-specific,
and they can still carry the attributes that made the model look
heterogeneous in the first place.
"""
model_arch_config = ModelArchitectureConfig(
architectures=self.get_architectures(),
model_type=self.hf_config.model_type,
@@ -622,13 +683,26 @@ class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
== "vision"
)
def get_head_size(self) -> int:
# Gemma4 uses dual head dimensions: head_dim (sliding attention)
# and global_head_dim (full attention). Return the largest so
# that attention backends allocate buffers large enough for both.
head_dim = getattr(self.hf_text_config, "head_dim", 0)
global_head_dim = getattr(self.hf_text_config, "global_head_dim", 0)
return max(head_dim, global_head_dim) or super().get_head_size()
def get_per_layer_hf_configs(
self,
) -> list[tuple[PretrainedConfig, PretrainedConfig]] | None:
# Gemma4 uses a larger head dimension, and sometimes more KV heads, on its
# full attention layers than on its sliding ones. Transformers >= 5.15.0
# says so in the config; before that the values are flat attributes picked
# apart by `layer_types`, so build the per-layer configs here instead. Both
# then reach the base convertor as ordinary homogeneous configs.
if getattr(self.hf_text_config, "is_heterogeneous", False):
return super().get_per_layer_hf_configs()
text_config = self.hf_text_config
if not getattr(text_config, "layer_types", None):
return None
# `num_hidden_layers` is what the stack is built from, and it is what
# Transformers sizes `per_layer_config` by. `layer_types` can be longer,
# e.g. when a test truncates the layer count to build a small model.
return [
(self.hf_config, gemma4_layer_config(text_config, layer_idx))
for layer_idx in range(self.get_num_hidden_layers())
]
class MossAudioModelArchConfigConvertor(ModelArchConfigConvertorBase):