Fix Gemma 4 for upcoming Transformers version (#49797)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -260,12 +260,8 @@ def get_quant_config(
|
||||
and hf_quant_config.get("quant_method") == "compressed-tensors"
|
||||
and "config_groups" in hf_quant_config
|
||||
):
|
||||
if hf_text_config is not None:
|
||||
n_heads = getattr(hf_text_config, "num_attention_heads", None)
|
||||
n_kv_heads = getattr(hf_text_config, "num_key_value_heads", None)
|
||||
else:
|
||||
n_heads = getattr(model_config.hf_config, "num_attention_heads", None)
|
||||
n_kv_heads = getattr(model_config.hf_config, "num_key_value_heads", None)
|
||||
n_heads = model_config.model_arch_config.total_num_attention_heads
|
||||
n_kv_heads = model_config.model_arch_config.total_num_kv_heads
|
||||
|
||||
hf_quant_config["total_num_heads"] = n_heads
|
||||
hf_quant_config["total_num_kv_heads"] = (
|
||||
|
||||
@@ -200,26 +200,29 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
"""Configure attention for heterogeneous head dimensions.
|
||||
|
||||
Gemma4 uses different head dimensions for sliding window
|
||||
(head_dim) vs full attention (global_head_dim) layers. The
|
||||
default FA3 on Hopper cannot handle head_dim > 256, which
|
||||
causes mixed backend selection and numerical divergence.
|
||||
Gemma4 uses different head dimensions for sliding window vs full attention
|
||||
layers. The default FA3 on Hopper cannot handle head_dim > 256, which causes
|
||||
mixed backend selection and numerical divergence.
|
||||
|
||||
When FA4 is available we force it for ALL layers, giving a
|
||||
uniform kernel path and avoiding the mixed FA3+FA4 penalty.
|
||||
When FA4 is not available we fall back to Triton.
|
||||
When FA4 is available we force it for ALL layers, giving a uniform kernel path
|
||||
and avoiding the mixed FA3+FA4 penalty. When FA4 is not available we fall back
|
||||
to Triton.
|
||||
"""
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
head_dim = getattr(hf_text_config, "head_dim", None)
|
||||
global_head_dim = getattr(hf_text_config, "global_head_dim", None)
|
||||
model_config = vllm_config.model_config
|
||||
arch_config = model_config.model_arch_config
|
||||
layer_types = getattr(model_config.hf_text_config, "layer_types", None) or []
|
||||
head_dims = {
|
||||
layer_types[i]: arch_config[i].head_size
|
||||
for i in range(min(arch_config.total_num_hidden_layers, len(layer_types)))
|
||||
}
|
||||
|
||||
if head_dim is None or global_head_dim is None or head_dim == global_head_dim:
|
||||
if len(set(head_dims.values())) <= 1:
|
||||
return
|
||||
|
||||
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
max_head_dim = max(head_dim, global_head_dim)
|
||||
max_head_dim = max(head_dims.values())
|
||||
|
||||
if is_fa_version_supported(4) and max_head_dim <= 512:
|
||||
if (
|
||||
@@ -229,20 +232,16 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
):
|
||||
vllm_config.attention_config.flash_attn_version = 4
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions "
|
||||
"(head_dim=%d, global_head_dim=%d). Using FA4 for "
|
||||
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 for "
|
||||
"all layers to avoid mixed FA3/FA4 penalty.",
|
||||
head_dim,
|
||||
global_head_dim,
|
||||
head_dims,
|
||||
)
|
||||
elif vllm_config.attention_config.backend is None:
|
||||
vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions "
|
||||
"(head_dim=%d, global_head_dim=%d). FA4 not available, "
|
||||
"forcing TRITON_ATTN backend.",
|
||||
head_dim,
|
||||
global_head_dim,
|
||||
"%s. FA4 not available, forcing TRITON_ATTN backend.",
|
||||
head_dims,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
|
||||
|
||||
@@ -380,7 +381,6 @@ class Gemma4Attention(nn.Module):
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
max_position_embeddings: int,
|
||||
use_k_eq_v: bool = False,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
attn_logits_soft_cap: float | None = None,
|
||||
@@ -389,7 +389,6 @@ class Gemma4Attention(nn.Module):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = hidden_size
|
||||
self.use_k_eq_v = use_k_eq_v
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
@@ -570,27 +569,9 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
# Gemma4 uses different head dimensions for sliding vs full attention
|
||||
layer_type = config.layer_types[layer_idx]
|
||||
self.is_full_attention = layer_type == "full_attention"
|
||||
if self.is_full_attention:
|
||||
head_dim = getattr(config, "global_head_dim", config.head_dim)
|
||||
else:
|
||||
head_dim = config.head_dim
|
||||
|
||||
# Determine if this full-attention layer uses k_eq_v
|
||||
# (laptop variant: no v_proj, K reused as V on full attention layers)
|
||||
use_k_eq_v = self.is_full_attention and getattr(
|
||||
config, "attention_k_eq_v", False
|
||||
)
|
||||
|
||||
# For k_eq_v full-attention layers, use num_global_key_value_heads
|
||||
# as the KV head count when k_eq_v is enabled.
|
||||
if use_k_eq_v:
|
||||
num_kv_heads = getattr(
|
||||
config, "num_global_key_value_heads", config.num_key_value_heads
|
||||
)
|
||||
else:
|
||||
num_kv_heads = config.num_key_value_heads
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
|
||||
self.self_attn = Gemma4Attention(
|
||||
config=config,
|
||||
@@ -599,7 +580,6 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
max_position_embeddings=config.max_position_embeddings,
|
||||
use_k_eq_v=use_k_eq_v,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
attn_logits_soft_cap=getattr(config, "attn_logit_softcapping", None),
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
|
||||
from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer
|
||||
from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal
|
||||
@@ -38,18 +39,14 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
|
||||
prefix: str,
|
||||
) -> None:
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
is_full = config.layer_types[layer_idx] == "full_attention"
|
||||
head_dim = (
|
||||
getattr(config, "global_head_dim", config.head_dim)
|
||||
if is_full
|
||||
else config.head_dim
|
||||
)
|
||||
use_k_eq_v = is_full and getattr(config, "attention_k_eq_v", False)
|
||||
num_kv_heads = (
|
||||
getattr(config, "num_global_key_value_heads", config.num_key_value_heads)
|
||||
if use_k_eq_v
|
||||
else config.num_key_value_heads
|
||||
layer_type = config.layer_types[layer_idx]
|
||||
self.use_k_eq_v = layer_type == "full_attention" and getattr(
|
||||
config, "attention_k_eq_v", False
|
||||
)
|
||||
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
super().__init__(
|
||||
config=config,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -64,7 +61,6 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
|
||||
)
|
||||
self.is_kv_shared_layer = False
|
||||
self.causal = _dflash_layer_causal(config, layer_idx)
|
||||
self.use_k_eq_v = use_k_eq_v
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
attn_bias = getattr(config, "attention_bias", False)
|
||||
self.k_proj = ColumnParallelLinear(
|
||||
@@ -76,7 +72,7 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
|
||||
)
|
||||
self.v_proj = (
|
||||
None
|
||||
if use_k_eq_v
|
||||
if self.use_k_eq_v
|
||||
else ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
self.total_num_kv_heads * self.head_dim,
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
|
||||
from .gemma4 import Gemma4MLP, _get_text_config
|
||||
from .utils import (
|
||||
@@ -271,21 +272,9 @@ class Gemma4MTPDecoderLayer(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
layer_type = config.layer_types[layer_idx]
|
||||
is_full_attention = layer_type == "full_attention"
|
||||
head_dim = (
|
||||
getattr(config, "global_head_dim", config.head_dim)
|
||||
if is_full_attention
|
||||
else config.head_dim
|
||||
)
|
||||
|
||||
use_k_eq_v = is_full_attention and getattr(config, "attention_k_eq_v", False)
|
||||
if use_k_eq_v:
|
||||
num_kv_heads = getattr(
|
||||
config, "num_global_key_value_heads", config.num_key_value_heads
|
||||
)
|
||||
else:
|
||||
num_kv_heads = config.num_key_value_heads
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
|
||||
self.self_attn = Gemma4MTPAttention(
|
||||
config=config,
|
||||
|
||||
@@ -522,9 +522,6 @@ class Base(
|
||||
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
|
||||
self.model_config.model_arch_config.head_size = qk_head_dim
|
||||
|
||||
num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
|
||||
head_size = self.model_config.get_head_size()
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
|
||||
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
@@ -532,11 +529,22 @@ class Base(
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
for i in range(start, end):
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
arch_config = self.model_config.model_arch_config[i]
|
||||
num_heads = self.model_config.get_num_attention_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
head_size = arch_config.head_size
|
||||
# Default to Llama scale, maybe updated in vllm_attention_forward
|
||||
scale = head_size**-0.5
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
num_heads=num_heads,
|
||||
# NOTE: We use Llama scale as default, if it's set by
|
||||
# Transformers, it's updated in vllm_attention_forward
|
||||
scale=head_size**-0.5,
|
||||
scale=scale,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{i}.attn",
|
||||
|
||||
@@ -57,6 +57,17 @@ class BaseFuser(ABC):
|
||||
return {}
|
||||
|
||||
|
||||
def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int:
|
||||
"""The head size of `module`, which head counts are derived from.
|
||||
|
||||
Prefer the module's own `head_dim`, which Transformers sets per instance:
|
||||
the model-wide value is the largest head size across layers, so on a
|
||||
heterogeneous checkpoint it would divide out the wrong number of heads, and
|
||||
it is the text head size, so it does not describe a vision tower at all.
|
||||
"""
|
||||
return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size()
|
||||
|
||||
|
||||
def local_output_sizes(merged_name: str) -> str:
|
||||
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
|
||||
merged = f"self.{merged_name}"
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
RewriteFuser,
|
||||
fused_head_size,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
@@ -122,7 +123,7 @@ class PackedQKVFuser(RewriteFuser):
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible with a head-sharded packed GEMM."""
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
compatible = (
|
||||
self.q_size % head_size == 0
|
||||
@@ -137,7 +138,7 @@ class PackedQKVFuser(RewriteFuser):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
merged = QKVParallelLinear(
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
StackedFuser,
|
||||
fused_head_size,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
@@ -158,7 +159,7 @@ class QKVFuser(StackedFuser):
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
v = module.get_submodule(self.v_name)
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
compatible = (
|
||||
q.in_features == k.in_features == v.in_features
|
||||
and len({proj.bias is None for proj in (q, k, v)}) == 1
|
||||
@@ -174,7 +175,7 @@ class QKVFuser(StackedFuser):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
merged = QKVParallelLinear(
|
||||
|
||||
Reference in New Issue
Block a user