Fix Gemma 4 for upcoming Transformers version (#49797)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-10 07:48:26 +00:00
committed by GitHub
parent 81840a172f
commit 70b84f0bcb
15 changed files with 569 additions and 105 deletions
@@ -522,9 +522,6 @@ class Base(
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
self.model_config.model_arch_config.head_size = qk_head_dim
num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
head_size = self.model_config.get_head_size()
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
pp_rank = self.pp_group.rank_in_group
@@ -532,11 +529,22 @@ class Base(
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
for i in range(start, end):
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
arch_config = self.model_config.model_arch_config[i]
num_heads = self.model_config.get_num_attention_heads(
self.parallel_config, arch_config
)
head_size = arch_config.head_size
# Default to Llama scale, maybe updated in vllm_attention_forward
scale = head_size**-0.5
num_kv_heads = self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
)
kwargs = dict(
num_heads=num_heads,
# NOTE: We use Llama scale as default, if it's set by
# Transformers, it's updated in vllm_attention_forward
scale=head_size**-0.5,
scale=scale,
cache_config=self.cache_config,
quant_config=self.quant_config,
prefix=f"{i}.attn",