Fix Gemma 4 for upcoming Transformers version (#49797)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-layer config resolution for Gemma4 and its variants."""
|
||||
|
||||
from copy import copy
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
|
||||
def gemma4_layer_config(
|
||||
text_config: PretrainedConfig, layer_idx: int
|
||||
) -> PretrainedConfig:
|
||||
"""The Gemma4 text config as it applies to one layer.
|
||||
|
||||
Gemma4 uses a larger head dimension on its full attention layers than on its
|
||||
sliding ones, and with `attention_k_eq_v` it uses more KV heads there too.
|
||||
Transformers >= 5.15.0 says so in the config itself; before that the values
|
||||
are flat attributes that `layer_types` picks between, so resolve them here.
|
||||
The result is homogeneous either way, so callers read `head_dim` and
|
||||
`num_key_value_heads` off it without caring which version they are on.
|
||||
"""
|
||||
if getattr(text_config, "is_heterogeneous", False):
|
||||
return text_config.per_layer_config[layer_idx]
|
||||
|
||||
layer = copy(text_config)
|
||||
if text_config.layer_types[layer_idx] == "full_attention":
|
||||
global_head_dim = getattr(text_config, "global_head_dim", None)
|
||||
layer.head_dim = global_head_dim or text_config.head_dim
|
||||
if getattr(text_config, "attention_k_eq_v", False):
|
||||
global_kv_heads = getattr(text_config, "num_global_key_value_heads", None)
|
||||
layer.num_key_value_heads = (
|
||||
global_kv_heads or text_config.num_key_value_heads
|
||||
)
|
||||
return layer
|
||||
@@ -17,6 +17,7 @@ from vllm.transformers_utils.config import (
|
||||
ConfigFormat,
|
||||
get_safetensors_params_metadata,
|
||||
)
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
from vllm.utils.torch_utils import common_broadcastable_dtype
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -27,6 +28,37 @@ class ModelArchConfigConvertorBase:
|
||||
self.hf_config = hf_config
|
||||
self.hf_text_config = hf_text_config
|
||||
|
||||
def get_per_layer_hf_configs(
|
||||
self,
|
||||
) -> list[tuple[PretrainedConfig, PretrainedConfig]] | None:
|
||||
"""`(hf_config, hf_text_config)` per layer, or `None` if homogeneous.
|
||||
|
||||
This is the only place that decides whether a checkpoint is heterogeneous
|
||||
and how many layers it has, so a convertor can declare variation that
|
||||
Transformers does not express (see `Gemma4ModelArchConfigConvertor`).
|
||||
|
||||
Each pair is converted independently and the results are diffed, so a
|
||||
convertor that overrides this must not also collapse a varying field in
|
||||
its getters: the collapse would then be applied per layer and every layer
|
||||
would report the same value.
|
||||
"""
|
||||
config_het = getattr(self.hf_config, "is_heterogeneous", False)
|
||||
text_het = getattr(self.hf_text_config, "is_heterogeneous", False)
|
||||
if not (config_het or text_het):
|
||||
return None
|
||||
# The text config decides the layer count when it is the heterogeneous one:
|
||||
# its stack is what vLLM builds attention layers for.
|
||||
source = self.hf_text_config if text_het else self.hf_config
|
||||
return [
|
||||
(
|
||||
self.hf_config.per_layer_config[i] if config_het else self.hf_config,
|
||||
self.hf_text_config.per_layer_config[i]
|
||||
if text_het
|
||||
else self.hf_text_config,
|
||||
)
|
||||
for i in range(len(source.per_layer_config))
|
||||
]
|
||||
|
||||
def get_architectures(self) -> list[str]:
|
||||
# Sometimes we get here from `vllm_config.with_hf_config(text_config)` where
|
||||
# `text_config` is a sub-config from a multi-modal model. If this is the case,
|
||||
@@ -382,6 +414,35 @@ class ModelArchConfigConvertorBase:
|
||||
return derived_max_model_len, max_len_key
|
||||
|
||||
def convert(self, supports_multimodal: bool = True) -> ModelArchitectureConfig:
|
||||
if (per_layer := self.get_per_layer_hf_configs()) is None:
|
||||
return self.convert_layer(supports_multimodal)
|
||||
|
||||
if self.is_deepseek_mla():
|
||||
raise NotImplementedError(
|
||||
"Heterogeneous MLA models are not supported: `get_head_size` "
|
||||
"patches the config in place for them, which a per-layer "
|
||||
"conversion would apply to a throwaway copy."
|
||||
)
|
||||
# Convert each layer and let the result work out which fields differ.
|
||||
# Reading a varying attribute off the global config would raise, so the
|
||||
# whole-model config has to be built up from the layers.
|
||||
return ModelArchitectureConfig.from_layers(
|
||||
[
|
||||
type(self)(*configs).convert_layer(supports_multimodal)
|
||||
for configs in per_layer
|
||||
]
|
||||
)
|
||||
|
||||
def convert_layer(
|
||||
self, supports_multimodal: bool = True
|
||||
) -> ModelArchitectureConfig:
|
||||
"""Convert one homogeneous config, without resolving per-layer values.
|
||||
|
||||
`convert` calls this once per layer, so it must not recurse back into
|
||||
per-layer resolution: the configs it receives are already layer-specific,
|
||||
and they can still carry the attributes that made the model look
|
||||
heterogeneous in the first place.
|
||||
"""
|
||||
model_arch_config = ModelArchitectureConfig(
|
||||
architectures=self.get_architectures(),
|
||||
model_type=self.hf_config.model_type,
|
||||
@@ -622,13 +683,26 @@ class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
== "vision"
|
||||
)
|
||||
|
||||
def get_head_size(self) -> int:
|
||||
# Gemma4 uses dual head dimensions: head_dim (sliding attention)
|
||||
# and global_head_dim (full attention). Return the largest so
|
||||
# that attention backends allocate buffers large enough for both.
|
||||
head_dim = getattr(self.hf_text_config, "head_dim", 0)
|
||||
global_head_dim = getattr(self.hf_text_config, "global_head_dim", 0)
|
||||
return max(head_dim, global_head_dim) or super().get_head_size()
|
||||
def get_per_layer_hf_configs(
|
||||
self,
|
||||
) -> list[tuple[PretrainedConfig, PretrainedConfig]] | None:
|
||||
# Gemma4 uses a larger head dimension, and sometimes more KV heads, on its
|
||||
# full attention layers than on its sliding ones. Transformers >= 5.15.0
|
||||
# says so in the config; before that the values are flat attributes picked
|
||||
# apart by `layer_types`, so build the per-layer configs here instead. Both
|
||||
# then reach the base convertor as ordinary homogeneous configs.
|
||||
if getattr(self.hf_text_config, "is_heterogeneous", False):
|
||||
return super().get_per_layer_hf_configs()
|
||||
text_config = self.hf_text_config
|
||||
if not getattr(text_config, "layer_types", None):
|
||||
return None
|
||||
# `num_hidden_layers` is what the stack is built from, and it is what
|
||||
# Transformers sizes `per_layer_config` by. `layer_types` can be longer,
|
||||
# e.g. when a test truncates the layer count to build a small model.
|
||||
return [
|
||||
(self.hf_config, gemma4_layer_config(text_config, layer_idx))
|
||||
for layer_idx in range(self.get_num_hidden_layers())
|
||||
]
|
||||
|
||||
|
||||
class MossAudioModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
|
||||
Reference in New Issue
Block a user