Fix Gemma 4 for upcoming Transformers version (#49797)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -522,9 +522,6 @@ class Base(
|
||||
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
|
||||
self.model_config.model_arch_config.head_size = qk_head_dim
|
||||
|
||||
num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
|
||||
head_size = self.model_config.get_head_size()
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
|
||||
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
@@ -532,11 +529,22 @@ class Base(
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
for i in range(start, end):
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
arch_config = self.model_config.model_arch_config[i]
|
||||
num_heads = self.model_config.get_num_attention_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
head_size = arch_config.head_size
|
||||
# Default to Llama scale, maybe updated in vllm_attention_forward
|
||||
scale = head_size**-0.5
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
num_heads=num_heads,
|
||||
# NOTE: We use Llama scale as default, if it's set by
|
||||
# Transformers, it's updated in vllm_attention_forward
|
||||
scale=head_size**-0.5,
|
||||
scale=scale,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{i}.attn",
|
||||
|
||||
@@ -57,6 +57,17 @@ class BaseFuser(ABC):
|
||||
return {}
|
||||
|
||||
|
||||
def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int:
|
||||
"""The head size of `module`, which head counts are derived from.
|
||||
|
||||
Prefer the module's own `head_dim`, which Transformers sets per instance:
|
||||
the model-wide value is the largest head size across layers, so on a
|
||||
heterogeneous checkpoint it would divide out the wrong number of heads, and
|
||||
it is the text head size, so it does not describe a vision tower at all.
|
||||
"""
|
||||
return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size()
|
||||
|
||||
|
||||
def local_output_sizes(merged_name: str) -> str:
|
||||
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
|
||||
merged = f"self.{merged_name}"
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
RewriteFuser,
|
||||
fused_head_size,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
@@ -122,7 +123,7 @@ class PackedQKVFuser(RewriteFuser):
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible with a head-sharded packed GEMM."""
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
compatible = (
|
||||
self.q_size % head_size == 0
|
||||
@@ -137,7 +138,7 @@ class PackedQKVFuser(RewriteFuser):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
merged = QKVParallelLinear(
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
StackedFuser,
|
||||
fused_head_size,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
@@ -158,7 +159,7 @@ class QKVFuser(StackedFuser):
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
v = module.get_submodule(self.v_name)
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
compatible = (
|
||||
q.in_features == k.in_features == v.in_features
|
||||
and len({proj.bias is None for proj in (q, k, v)}) == 1
|
||||
@@ -174,7 +175,7 @@ class QKVFuser(StackedFuser):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
merged = QKVParallelLinear(
|
||||
|
||||
Reference in New Issue
Block a user