[Kernel] Resubmit PR 48666 - Gemma4 FP8 KV FA4 head dim 512 backend selection (#53175)
Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
@@ -8,7 +8,10 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, get_current_vllm_config
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -100,6 +103,7 @@ def _largest_kernel_block_within(
|
||||
per_token_bytes: int,
|
||||
page_budget: int,
|
||||
fallback: int,
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
) -> int:
|
||||
"""Largest supported kernel block size whose page fits in ``page_budget``.
|
||||
|
||||
@@ -111,7 +115,7 @@ def _largest_kernel_block_within(
|
||||
"""
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
sizes = attn_backend.get_supported_kernel_block_sizes()
|
||||
sizes = attn_backend.get_supported_kernel_block_sizes(kv_cache_spec)
|
||||
max_block_size = page_budget // per_token_bytes
|
||||
candidates = [s for s in sizes if isinstance(s, int)]
|
||||
candidates.extend(
|
||||
@@ -632,7 +636,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# ``unify`` scales it up by an integer ratio.
|
||||
shared_page = vllm_config.cache_config.skip_page_size_padded
|
||||
# The backend owns its packing
|
||||
sw_per_token = self.attn_backend.customize_spec(
|
||||
kv_cache_spec = self.attn_backend.customize_spec(
|
||||
SlidingWindowSpec(
|
||||
block_size=1,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
@@ -642,10 +646,15 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
).real_page_size_bytes
|
||||
)
|
||||
sw_per_token = kv_cache_spec.real_page_size_bytes
|
||||
page_budget = shared_page or sw_per_token * block_size
|
||||
sw_block_size = _largest_kernel_block_within(
|
||||
self.attn_backend, sw_per_token, page_budget, block_size
|
||||
self.attn_backend,
|
||||
sw_per_token,
|
||||
page_budget,
|
||||
block_size,
|
||||
kv_cache_spec,
|
||||
)
|
||||
return SlidingWindowSpec(
|
||||
block_size=sw_block_size,
|
||||
|
||||
@@ -212,12 +212,15 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
"""Configure attention for heterogeneous head dimensions.
|
||||
|
||||
Gemma4 uses different head dimensions for sliding window vs full attention
|
||||
layers. The default FA3 on Hopper cannot handle head_dim > 256, which causes
|
||||
mixed backend selection and numerical divergence.
|
||||
layers. The default FA3 on Hopper cannot handle head_dim > 256.
|
||||
|
||||
When FA4 is available we force it for ALL layers, giving a uniform kernel path
|
||||
and avoiding the mixed FA3+FA4 penalty. When FA4 is not available we fall back
|
||||
to Triton.
|
||||
On SM90 with FP8 KV cache, use FA3 for supported layers and let the generic
|
||||
FlashAttention selector upgrade larger head dimensions to FA4.
|
||||
The multimodal-prefix composite routes image masks to Triton and causal
|
||||
requests to this per-layer FA3/FA4 selection. For other configurations,
|
||||
force FA4 for all layers to avoid the mixed
|
||||
FA3+FA4 penalty.
|
||||
When FA4 is not available, fall back to Triton.
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
arch_config = model_config.model_arch_config
|
||||
@@ -230,6 +233,7 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
if len(set(head_dims.values())) <= 1:
|
||||
return
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.fa_utils import is_fa_version_supported
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -241,12 +245,22 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
and vllm_config.attention_config.backend
|
||||
in (None, AttentionBackendEnum.FLASH_ATTN)
|
||||
):
|
||||
vllm_config.attention_config.flash_attn_version = 4
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 for "
|
||||
"all layers to avoid mixed FA3/FA4 penalty.",
|
||||
head_dims,
|
||||
)
|
||||
use_per_layer_fa = current_platform.is_device_capability_family(
|
||||
90
|
||||
) and vllm_config.cache_config.cache_dtype.startswith("fp8")
|
||||
if use_per_layer_fa:
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions %s. Using "
|
||||
"per-layer FA3/FA4 selection for FP8 KV cache on SM90.",
|
||||
head_dims,
|
||||
)
|
||||
else:
|
||||
vllm_config.attention_config.flash_attn_version = 4
|
||||
logger.info(
|
||||
"Gemma4 model has heterogeneous head dimensions %s. Using FA4 "
|
||||
"for all layers to avoid mixed FA3/FA4 penalty.",
|
||||
head_dims,
|
||||
)
|
||||
elif vllm_config.attention_config.backend is None:
|
||||
vllm_config.attention_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user