Fix Gemma 4 for upcoming Transformers version (#49797)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-10 07:48:26 +00:00
committed by GitHub
parent 81840a172f
commit 70b84f0bcb
15 changed files with 569 additions and 105 deletions
@@ -57,6 +57,17 @@ class BaseFuser(ABC):
return {}
def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int:
"""The head size of `module`, which head counts are derived from.
Prefer the module's own `head_dim`, which Transformers sets per instance:
the model-wide value is the largest head size across layers, so on a
heterogeneous checkpoint it would divide out the wrong number of heads, and
it is the text head size, so it does not describe a vision tower at all.
"""
return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size()
def local_output_sizes(merged_name: str) -> str:
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
merged = f"self.{merged_name}"
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import (
RewriteFuser,
fused_head_size,
local_output_sizes,
)
from vllm.model_executor.models.transformers.fx_utils import (
@@ -122,7 +123,7 @@ class PackedQKVFuser(RewriteFuser):
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible with a head-sharded packed GEMM."""
head_size = vllm_config.model_config.get_head_size()
head_size = fused_head_size(module, vllm_config)
qkv = module.get_submodule(self.qkv_name)
compatible = (
self.q_size % head_size == 0
@@ -137,7 +138,7 @@ class PackedQKVFuser(RewriteFuser):
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
head_size = vllm_config.model_config.get_head_size()
head_size = fused_head_size(module, vllm_config)
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
qkv = module.get_submodule(self.qkv_name)
merged = QKVParallelLinear(
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import (
StackedFuser,
fused_head_size,
local_output_sizes,
)
from vllm.model_executor.models.transformers.fx_utils import (
@@ -158,7 +159,7 @@ class QKVFuser(StackedFuser):
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
v = module.get_submodule(self.v_name)
head_size = vllm_config.model_config.get_head_size()
head_size = fused_head_size(module, vllm_config)
compatible = (
q.in_features == k.in_features == v.in_features
and len({proj.bias is None for proj in (q, k, v)}) == 1
@@ -174,7 +175,7 @@ class QKVFuser(StackedFuser):
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
head_size = vllm_config.model_config.get_head_size()
head_size = fused_head_size(module, vllm_config)
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
merged = QKVParallelLinear(