Fix Gemma 4 for upcoming Transformers version (#49797)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -522,9 +522,6 @@ class Base(
|
||||
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
|
||||
self.model_config.model_arch_config.head_size = qk_head_dim
|
||||
|
||||
num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
|
||||
head_size = self.model_config.get_head_size()
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
|
||||
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
@@ -532,11 +529,22 @@ class Base(
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
for i in range(start, end):
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
arch_config = self.model_config.model_arch_config[i]
|
||||
num_heads = self.model_config.get_num_attention_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
head_size = arch_config.head_size
|
||||
# Default to Llama scale, maybe updated in vllm_attention_forward
|
||||
scale = head_size**-0.5
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
num_heads=num_heads,
|
||||
# NOTE: We use Llama scale as default, if it's set by
|
||||
# Transformers, it's updated in vllm_attention_forward
|
||||
scale=head_size**-0.5,
|
||||
scale=scale,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{i}.attn",
|
||||
|
||||
Reference in New Issue
Block a user