[Kernel] Resubmit PR 48666 - Gemma4 FP8 KV FA4 head dim 512 backend selection (#53175)

Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
Jhao-Ting Chen
2026-09-23 20:29:06 -07:00
committed by GitHub
parent e78e367c6e
commit 4fb767ff0a
47 changed files with 241 additions and 103 deletions
+25 -11
View File
@@ -212,12 +212,15 @@ class Gemma4Config(VerifyAndUpdateConfig):
"""Configure attention for heterogeneous head dimensions.
Gemma4 uses different head dimensions for sliding window vs full attention
layers. The default FA3 on Hopper cannot handle head_dim > 256, which causes
mixed backend selection and numerical divergence.
layers. The default FA3 on Hopper cannot handle head_dim > 256.
When FA4 is available we force it for ALL layers, giving a uniform kernel path
and avoiding the mixed FA3+FA4 penalty. When FA4 is not available we fall back
to Triton.
On SM90 with FP8 KV cache, use FA3 for supported layers and let the generic
FlashAttention selector upgrade larger head dimensions to FA4.
The multimodal-prefix composite routes image masks to Triton and causal
requests to this per-layer FA3/FA4 selection. For other configurations,
force FA4 for all layers to avoid the mixed
FA3+FA4 penalty.
When FA4 is not available, fall back to Triton.
"""
model_config = vllm_config.model_config
arch_config = model_config.model_arch_config
@@ -230,6 +233,7 @@ class Gemma4Config(VerifyAndUpdateConfig):
if len(set(head_dims.values())) <= 1:
return
from vllm.platforms import current_platform
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -241,12 +245,22 @@ class Gemma4Config(VerifyAndUpdateConfig):
and vllm_config.attention_config.backend
in (None, AttentionBackendEnum.FLASH_ATTN)
):
vllm_config.attention_config.flash_attn_version = 4
logger.info(
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 for "
"all layers to avoid mixed FA3/FA4 penalty.",
head_dims,
)
use_per_layer_fa = current_platform.is_device_capability_family(
90
) and vllm_config.cache_config.cache_dtype.startswith("fp8")
if use_per_layer_fa:
logger.info(
"Gemma4 model has heterogeneous head dimensions %s. Using "
"per-layer FA3/FA4 selection for FP8 KV cache on SM90.",
head_dims,
)
else:
vllm_config.attention_config.flash_attn_version = 4
logger.info(
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 "
"for all layers to avoid mixed FA3/FA4 penalty.",
head_dims,
)
elif vllm_config.attention_config.backend is None:
vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
logger.info(