[Core][Model] Gemma4: Unified FA4 for all layers + FlashAttention mm_prefix support (#42175)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Luciano Martins
Lucas Wilkinson
Matthew Bonanni
parent
d82ac00923
commit
6deb05e0e4
@@ -57,50 +57,49 @@ class Gemma3TextModelConfig(VerifyAndUpdateConfig):
|
||||
class Gemma4Config(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
"""Force unified attention backend for models with heterogeneous
|
||||
head dimensions.
|
||||
"""Configure attention for heterogeneous head dimensions.
|
||||
|
||||
Some Gemma4 variants use different head dimensions for
|
||||
sliding window (head_dim) vs full attention (global_head_dim) layers.
|
||||
When global_head_dim > 256, FlashAttention rejects those layers
|
||||
(head_size <= 256 kernel limit), causing vLLM to select a different
|
||||
backend for each layer type. This mixed-backend execution produces
|
||||
numerical divergence and output corruption.
|
||||
Gemma4 uses different head dimensions for sliding window
|
||||
(head_dim) vs full attention (global_head_dim) layers. The
|
||||
default FA3 on Hopper cannot handle head_dim > 256, which
|
||||
causes mixed backend selection and numerical divergence.
|
||||
|
||||
The fix detects heterogeneous head dimensions from the model config
|
||||
and forces TRITON_ATTN (which has no head_size ceiling) for all
|
||||
layers when the user hasn't explicitly chosen a backend.
|
||||
|
||||
TODO: Heterogeneous head_sizes (head_dim != global_head_dim)
|
||||
require NixlConnector changes to support per-layer KV transfer
|
||||
with different head dimensions for prefill-decode disaggregation.
|
||||
When FA4 is available we force it for ALL layers, giving a
|
||||
uniform kernel path and avoiding the mixed FA3+FA4 penalty.
|
||||
When FA4 is not available we fall back to Triton.
|
||||
"""
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
head_dim = getattr(hf_text_config, "head_dim", None)
|
||||
global_head_dim = getattr(hf_text_config, "global_head_dim", None)
|
||||
|
||||
# Only force Triton when head dimensions actually differ AND the
|
||||
# larger one exceeds FlashAttention's kernel limit (head_size <= 256).
|
||||
# This avoids unnecessary backend forcing on smaller models where
|
||||
# the config carries global_head_dim but all layers can still use
|
||||
# the same FA backend.
|
||||
max_head_dim = max(head_dim or 0, global_head_dim or 0)
|
||||
if (
|
||||
head_dim is not None
|
||||
and global_head_dim is not None
|
||||
and head_dim != global_head_dim
|
||||
and max_head_dim > 256
|
||||
and vllm_config.attention_config.backend is None
|
||||
):
|
||||
from vllm.v1.attention.backends.registry import (
|
||||
AttentionBackendEnum,
|
||||
)
|
||||
if head_dim is None or global_head_dim is None or head_dim == global_head_dim:
|
||||
return
|
||||
|
||||
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
max_head_dim = max(head_dim, global_head_dim)
|
||||
|
||||
if is_fa_version_supported(4) and max_head_dim <= 512:
|
||||
if (
|
||||
vllm_config.attention_config.flash_attn_version is None
|
||||
and vllm_config.attention_config.backend
|
||||
in (None, AttentionBackendEnum.FLASH_ATTN)
|
||||
):
|
||||
vllm_config.attention_config.flash_attn_version = 4
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions "
|
||||
"(head_dim=%d, global_head_dim=%d). Using FA4 for "
|
||||
"all layers to avoid mixed FA3/FA4 penalty.",
|
||||
head_dim,
|
||||
global_head_dim,
|
||||
)
|
||||
elif vllm_config.attention_config.backend is None:
|
||||
vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions "
|
||||
"(head_dim=%d, global_head_dim=%d). Forcing TRITON_ATTN "
|
||||
"backend to prevent mixed-backend numerical divergence.",
|
||||
"(head_dim=%d, global_head_dim=%d). FA4 not available, "
|
||||
"forcing TRITON_ATTN backend.",
|
||||
head_dim,
|
||||
global_head_dim,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user