[Core][Model] Gemma4: Unified FA4 for all layers + FlashAttention mm_prefix support (#42175)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Luciano Martins
2026-06-09 17:45:39 -07:00
committed by GitHub
co-authored by Luciano Martins Lucas Wilkinson Matthew Bonanni
parent d82ac00923
commit 6deb05e0e4
14 changed files with 222 additions and 142 deletions
+32 -33
View File
@@ -57,50 +57,49 @@ class Gemma3TextModelConfig(VerifyAndUpdateConfig):
class Gemma4Config(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
"""Force unified attention backend for models with heterogeneous
head dimensions.
"""Configure attention for heterogeneous head dimensions.
Some Gemma4 variants use different head dimensions for
sliding window (head_dim) vs full attention (global_head_dim) layers.
When global_head_dim > 256, FlashAttention rejects those layers
(head_size <= 256 kernel limit), causing vLLM to select a different
backend for each layer type. This mixed-backend execution produces
numerical divergence and output corruption.
Gemma4 uses different head dimensions for sliding window
(head_dim) vs full attention (global_head_dim) layers. The
default FA3 on Hopper cannot handle head_dim > 256, which
causes mixed backend selection and numerical divergence.
The fix detects heterogeneous head dimensions from the model config
and forces TRITON_ATTN (which has no head_size ceiling) for all
layers when the user hasn't explicitly chosen a backend.
TODO: Heterogeneous head_sizes (head_dim != global_head_dim)
require NixlConnector changes to support per-layer KV transfer
with different head dimensions for prefill-decode disaggregation.
When FA4 is available we force it for ALL layers, giving a
uniform kernel path and avoiding the mixed FA3+FA4 penalty.
When FA4 is not available we fall back to Triton.
"""
hf_text_config = vllm_config.model_config.hf_text_config
head_dim = getattr(hf_text_config, "head_dim", None)
global_head_dim = getattr(hf_text_config, "global_head_dim", None)
# Only force Triton when head dimensions actually differ AND the
# larger one exceeds FlashAttention's kernel limit (head_size <= 256).
# This avoids unnecessary backend forcing on smaller models where
# the config carries global_head_dim but all layers can still use
# the same FA backend.
max_head_dim = max(head_dim or 0, global_head_dim or 0)
if (
head_dim is not None
and global_head_dim is not None
and head_dim != global_head_dim
and max_head_dim > 256
and vllm_config.attention_config.backend is None
):
from vllm.v1.attention.backends.registry import (
AttentionBackendEnum,
)
if head_dim is None or global_head_dim is None or head_dim == global_head_dim:
return
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
from vllm.v1.attention.backends.registry import AttentionBackendEnum
max_head_dim = max(head_dim, global_head_dim)
if is_fa_version_supported(4) and max_head_dim <= 512:
if (
vllm_config.attention_config.flash_attn_version is None
and vllm_config.attention_config.backend
in (None, AttentionBackendEnum.FLASH_ATTN)
):
vllm_config.attention_config.flash_attn_version = 4
logger.info(
"Gemma4 model has heterogeneous head dimensions "
"(head_dim=%d, global_head_dim=%d). Using FA4 for "
"all layers to avoid mixed FA3/FA4 penalty.",
head_dim,
global_head_dim,
)
elif vllm_config.attention_config.backend is None:
vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
logger.info(
"Gemma4 model has heterogeneous head dimensions "
"(head_dim=%d, global_head_dim=%d). Forcing TRITON_ATTN "
"backend to prevent mixed-backend numerical divergence.",
"(head_dim=%d, global_head_dim=%d). FA4 not available, "
"forcing TRITON_ATTN backend.",
head_dim,
global_head_dim,
)