[Kernel] Resubmit PR 48666 - Gemma4 FP8 KV FA4 head dim 512 backend selection (#53175)

Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
Jhao-Ting Chen
2026-09-23 20:29:06 -07:00
committed by GitHub
parent e78e367c6e
commit 4fb767ff0a
47 changed files with 241 additions and 103 deletions
@@ -8,7 +8,10 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, get_current_vllm_config
from vllm.config import (
CacheConfig,
get_current_vllm_config,
)
from vllm.config.vllm import VllmConfig
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
@@ -100,6 +103,7 @@ def _largest_kernel_block_within(
per_token_bytes: int,
page_budget: int,
fallback: int,
kv_cache_spec: KVCacheSpec | None = None,
) -> int:
"""Largest supported kernel block size whose page fits in ``page_budget``.
@@ -111,7 +115,7 @@ def _largest_kernel_block_within(
"""
from vllm.v1.attention.backend import MultipleOf
sizes = attn_backend.get_supported_kernel_block_sizes()
sizes = attn_backend.get_supported_kernel_block_sizes(kv_cache_spec)
max_block_size = page_budget // per_token_bytes
candidates = [s for s in sizes if isinstance(s, int)]
candidates.extend(
@@ -632,7 +636,7 @@ class Attention(nn.Module, AttentionLayerBase):
# ``unify`` scales it up by an integer ratio.
shared_page = vllm_config.cache_config.skip_page_size_padded
# The backend owns its packing
sw_per_token = self.attn_backend.customize_spec(
kv_cache_spec = self.attn_backend.customize_spec(
SlidingWindowSpec(
block_size=1,
num_kv_heads=self.num_kv_heads,
@@ -642,10 +646,15 @@ class Attention(nn.Module, AttentionLayerBase):
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
)
).real_page_size_bytes
)
sw_per_token = kv_cache_spec.real_page_size_bytes
page_budget = shared_page or sw_per_token * block_size
sw_block_size = _largest_kernel_block_within(
self.attn_backend, sw_per_token, page_budget, block_size
self.attn_backend,
sw_per_token,
page_budget,
block_size,
kv_cache_spec,
)
return SlidingWindowSpec(
block_size=sw_block_size,
+25 -11
View File
@@ -212,12 +212,15 @@ class Gemma4Config(VerifyAndUpdateConfig):
"""Configure attention for heterogeneous head dimensions.
Gemma4 uses different head dimensions for sliding window vs full attention
layers. The default FA3 on Hopper cannot handle head_dim > 256, which causes
mixed backend selection and numerical divergence.
layers. The default FA3 on Hopper cannot handle head_dim > 256.
When FA4 is available we force it for ALL layers, giving a uniform kernel path
and avoiding the mixed FA3+FA4 penalty. When FA4 is not available we fall back
to Triton.
On SM90 with FP8 KV cache, use FA3 for supported layers and let the generic
FlashAttention selector upgrade larger head dimensions to FA4.
The multimodal-prefix composite routes image masks to Triton and causal
requests to this per-layer FA3/FA4 selection. For other configurations,
force FA4 for all layers to avoid the mixed
FA3+FA4 penalty.
When FA4 is not available, fall back to Triton.
"""
model_config = vllm_config.model_config
arch_config = model_config.model_arch_config
@@ -230,6 +233,7 @@ class Gemma4Config(VerifyAndUpdateConfig):
if len(set(head_dims.values())) <= 1:
return
from vllm.platforms import current_platform
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -241,12 +245,22 @@ class Gemma4Config(VerifyAndUpdateConfig):
and vllm_config.attention_config.backend
in (None, AttentionBackendEnum.FLASH_ATTN)
):
vllm_config.attention_config.flash_attn_version = 4
logger.info(
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 for "
"all layers to avoid mixed FA3/FA4 penalty.",
head_dims,
)
use_per_layer_fa = current_platform.is_device_capability_family(
90
) and vllm_config.cache_config.cache_dtype.startswith("fp8")
if use_per_layer_fa:
logger.info(
"Gemma4 model has heterogeneous head dimensions %s. Using "
"per-layer FA3/FA4 selection for FP8 KV cache on SM90.",
head_dims,
)
else:
vllm_config.attention_config.flash_attn_version = 4
logger.info(
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 "
"for all layers to avoid mixed FA3/FA4 penalty.",
head_dims,
)
elif vllm_config.attention_config.backend is None:
vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
logger.info(