Fix Gemma 4 for upcoming Transformers version (#49797)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -200,26 +200,29 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
"""Configure attention for heterogeneous head dimensions.
|
||||
|
||||
Gemma4 uses different head dimensions for sliding window
|
||||
(head_dim) vs full attention (global_head_dim) layers. The
|
||||
default FA3 on Hopper cannot handle head_dim > 256, which
|
||||
causes mixed backend selection and numerical divergence.
|
||||
Gemma4 uses different head dimensions for sliding window vs full attention
|
||||
layers. The default FA3 on Hopper cannot handle head_dim > 256, which causes
|
||||
mixed backend selection and numerical divergence.
|
||||
|
||||
When FA4 is available we force it for ALL layers, giving a
|
||||
uniform kernel path and avoiding the mixed FA3+FA4 penalty.
|
||||
When FA4 is not available we fall back to Triton.
|
||||
When FA4 is available we force it for ALL layers, giving a uniform kernel path
|
||||
and avoiding the mixed FA3+FA4 penalty. When FA4 is not available we fall back
|
||||
to Triton.
|
||||
"""
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
head_dim = getattr(hf_text_config, "head_dim", None)
|
||||
global_head_dim = getattr(hf_text_config, "global_head_dim", None)
|
||||
model_config = vllm_config.model_config
|
||||
arch_config = model_config.model_arch_config
|
||||
layer_types = getattr(model_config.hf_text_config, "layer_types", None) or []
|
||||
head_dims = {
|
||||
layer_types[i]: arch_config[i].head_size
|
||||
for i in range(min(arch_config.total_num_hidden_layers, len(layer_types)))
|
||||
}
|
||||
|
||||
if head_dim is None or global_head_dim is None or head_dim == global_head_dim:
|
||||
if len(set(head_dims.values())) <= 1:
|
||||
return
|
||||
|
||||
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
max_head_dim = max(head_dim, global_head_dim)
|
||||
max_head_dim = max(head_dims.values())
|
||||
|
||||
if is_fa_version_supported(4) and max_head_dim <= 512:
|
||||
if (
|
||||
@@ -229,20 +232,16 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
):
|
||||
vllm_config.attention_config.flash_attn_version = 4
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions "
|
||||
"(head_dim=%d, global_head_dim=%d). Using FA4 for "
|
||||
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 for "
|
||||
"all layers to avoid mixed FA3/FA4 penalty.",
|
||||
head_dim,
|
||||
global_head_dim,
|
||||
head_dims,
|
||||
)
|
||||
elif vllm_config.attention_config.backend is None:
|
||||
vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions "
|
||||
"(head_dim=%d, global_head_dim=%d). FA4 not available, "
|
||||
"forcing TRITON_ATTN backend.",
|
||||
head_dim,
|
||||
global_head_dim,
|
||||
"%s. FA4 not available, forcing TRITON_ATTN backend.",
|
||||
head_dims,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user