From 186a1e62221c02cdf8ef0433eb4ad33d1b48c7f7 Mon Sep 17 00:00:00 2001 From: "Roberto L. Castro" <38211239+LopezCastroRoberto@users.noreply.github.com> Date: Sun, 13 Sep 2026 22:47:21 +0200 Subject: [PATCH] [Warmup] Gemma 4 de-JITification (#55768) Signed-off-by: LopezCastroRoberto Co-authored-by: Codex --- vllm/v1/attention/backends/flash_attn.py | 239 ++++++++++++++++++- vllm/v1/worker/gpu/attn_utils.py | 5 +- vllm/v1/worker/gpu/model_runner.py | 18 +- vllm/vllm_flash_attn/flash_attn_interface.py | 54 ++++- 4 files changed, 288 insertions(+), 28 deletions(-) diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 9109a9c6d7..eb2268cc62 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -5,12 +5,13 @@ import copy import functools from dataclasses import dataclass -from typing import ClassVar +from typing import Any, ClassVar import numpy as np import torch from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, WarmupChoices from vllm.platforms import current_platform from vllm.utils.torch_utils import ( PIN_MEMORY, @@ -46,6 +47,7 @@ from vllm.v1.worker.workspace import current_workspace_manager if is_flash_attn_varlen_func_available(): from vllm.v1.attention.backends.fa_utils import ( + compile_flash_attn_varlen_func_from_specs, flash_attn_supports_sinks, flash_attn_varlen_func, get_scheduler_metadata, @@ -67,7 +69,13 @@ from vllm.v1.attention.backend import ( AttentionMetadataBuilder, CommonAttentionMetadata, ) -from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + FullAttentionSpec, + KVCacheSpec, + KVQuantMode, + SlidingWindowSpec, +) from vllm.v1.worker.cp_utils import ( run_split_fa2_dcp_context_attention, should_skip_dcp_context_attention, @@ -77,6 +85,200 @@ from vllm.v1.worker.cp_utils import ( logger = init_logger(__name__) +FA4_DENSE_FLOAT_DTYPES = (torch.bfloat16, torch.float16) +FA4_DENSE_Q_TILE = 128 +FA4_DENSE_NUM_BLOCKS = 256 +FA4_DENSE_MAX_SEQLEN_K = 8192 + + +class FA4DenseAttentionKernel(VllmJitKernel["FA4DenseAttentionKernel.CompileKey"]): + """Warm paged, causal FA4 on Blackwell and Hopper.""" + + @dataclass(frozen=True) + class CompileKey: + q_stage: int + dtype: torch.dtype + qhead_per_kvhead: int + head_dim: int + page_size: int + has_window_left: bool + has_window_right: bool + softcap: float + is_single_batch: bool + num_splits: int + + @staticmethod + def kernel(*args: Any, **kwargs: Any) -> Any: + assert flash_attn_varlen_func is not None + return flash_attn_varlen_func(*args, **kwargs) + + def dispatch( # type: ignore[override] + self, + *, + q_stage: int, + dtype: torch.dtype, + num_qo_heads: int, + num_kv_heads: int, + head_dim: int, + page_size: int, + window_size: tuple[int, int], + softcap: float, + is_single_batch: bool = True, + num_splits: int = 1, + ) -> CompileKey: + return self.CompileKey( + q_stage=q_stage, + dtype=dtype, + qhead_per_kvhead=num_qo_heads // num_kv_heads, + head_dim=head_dim, + page_size=page_size, + has_window_left=window_size[0] >= 0, + has_window_right=window_size[1] >= 0, + softcap=softcap, + is_single_batch=is_single_batch, + num_splits=num_splits, + ) + + def get_warmup_keys( + self, + *, + vllm_config: VllmConfig, + kv_cache_spec: AttentionSpec, + num_qo_heads: int, + ) -> list[CompileKey]: + capability = current_platform.get_device_capability() + major = capability.major if capability is not None else None + if ( + vllm_config.parallel_config.decode_context_parallel_size != 1 + or vllm_config.model_config.rswa_window is not None + or kv_cache_spec.kv_quant_mode != KVQuantMode.NONE + or kv_cache_spec.dtype not in FA4_DENSE_FLOAT_DTYPES + or not isinstance(kv_cache_spec, (FullAttentionSpec, SlidingWindowSpec)) + ): + return [] + head_dim = kv_cache_spec.head_size + page_size = kv_cache_spec.block_size + sliding_window = kv_cache_spec.sliding_window + window_size = ( + (sliding_window - 1, 0) if sliding_window is not None else (-1, -1) + ) + softcap = float( + getattr( + vllm_config.model_config.hf_text_config, "attn_logit_softcapping", 0 + ) + or 0 + ) + if ( + major in (10, 11) + and uses_fa4_hd256_kernel(head_dim) + and page_size == FA4_HD256_PAGE_SIZE + ): + q_stages = WarmupChoices(1, 2) + single_batches = WarmupChoices(True, False) + split_counts = WarmupChoices(1) + elif ( + major == 9 and head_dim == 512 and window_size == (-1, -1) and softcap == 0 + ): + # SM90 forward has split/non-split variants; its transitive + # combine specializes at 32/64/128/256 splits. + q_stages = WarmupChoices(1) + single_batches = WarmupChoices(False) + split_counts = WarmupChoices(1, 32, 64, 128, 256) + else: + return [] + + return self._trace_dispatch(self.dispatch)( + q_stage=q_stages, + is_single_batch=single_batches, + num_splits=split_counts, + dtype=vllm_config.model_config.dtype, + num_qo_heads=num_qo_heads, + num_kv_heads=kv_cache_spec.num_kv_heads, + head_dim=head_dim, + page_size=page_size, + window_size=WarmupChoices(window_size), + softcap=softcap, + ) + + def compile(self, compile_key: CompileKey) -> None: + assert compile_flash_attn_varlen_func_from_specs is not None + max_seqlen_q = FA4_DENSE_Q_TILE + 1 if compile_key.q_stage == 2 else 1 + batch_size = 1 if compile_key.is_single_batch else 2 + kv_shape = ( + FA4_DENSE_NUM_BLOCKS, + compile_key.page_size, + 1, + compile_key.head_dim, + ) + kv_stride = ( + 2 * compile_key.page_size * compile_key.head_dim, + 2 * compile_key.head_dim, + 2 * compile_key.head_dim, + 1, + ) + compile_flash_attn_varlen_func_from_specs( + q_shape=( + batch_size * max_seqlen_q, + compile_key.qhead_per_kvhead, + compile_key.head_dim, + ), + k_shape=kv_shape, + v_shape=kv_shape, + k_stride=kv_stride, + v_stride=kv_stride, + q_dtype=compile_key.dtype, + cu_seqlens_q_shape=(batch_size + 1,), + seqused_k_shape=(batch_size,), + page_table_shape=( + batch_size, + FA4_DENSE_MAX_SEQLEN_K // compile_key.page_size, + ), + max_seqlen_q=max_seqlen_q, + max_seqlen_k=FA4_DENSE_MAX_SEQLEN_K, + causal=True, + window_size=[ + 1 if compile_key.has_window_left else -1, + 1 if compile_key.has_window_right else -1, + ], + softcap=compile_key.softcap, + num_splits=2 if compile_key.num_splits > 1 else 1, + fa_version=4, + return_softmax_lse=False, + ) + if compile_key.head_dim == 512 and compile_key.num_splits > 1: + from torch._subclasses.fake_tensor import FakeTensorMode + + from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd_combine + + # FA4's forward compile-only entry returns before its transitive + # SplitKV combine. Call the native combine entry in FakeTensorMode: + # it populates the native cache without launching the kernel. + with FakeTensorMode(): + _flash_attn_fwd_combine( + torch.empty( + (compile_key.num_splits, 2, 1, 512), + dtype=torch.float32, + device="cuda", + ), + torch.empty( + (compile_key.num_splits, 2, 1), + dtype=torch.float32, + device="cuda", + ), + torch.empty((2, 1, 512), dtype=compile_key.dtype, device="cuda"), + cu_seqlens=torch.empty((3,), dtype=torch.int32, device="cuda"), + ) + + def __call__( + self, + *, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + **kwargs: Any, + ) -> Any: + return self.kernel(q=q, k=k, v=v, **kwargs) + class FlashAttentionBackend(AttentionBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] @@ -447,15 +649,27 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad self.max_num_splits = 0 # No upper bound on the number of splits. self.aot_schedule = get_flash_attn_version() == 3 - - self.fa4_hd256 = uses_fa4_hd256_kernel(self.headdim) and ( - get_flash_attn_version( - head_size=self.headdim, - kv_cache_block_size=self.block_size, - supports_fa4_hd256=True, - ) - == 4 + head_size_v = getattr(kv_cache_spec, "head_size_v", None) + fa_version = get_flash_attn_version( + head_size=self.headdim, + head_size_v=head_size_v, + kv_cache_block_size=self.block_size, + supports_fa4_hd256=True, ) + self.fa4_hd256 = fa_version == 4 and uses_fa4_hd256_kernel( + self.headdim, head_size_v + ) + + if ( + vllm_config.kernel_config.enable_jit_warmup + and self.model_config.hf_config.model_type in ("gemma4", "gemma4_unified") + and fa_version == 4 + ): + _FA4_DENSE_ATTENTION_KERNEL.register_warmup( + vllm_config=vllm_config, + kv_cache_spec=kv_cache_spec, + num_qo_heads=self.num_heads_q, + ) try: from vllm.distributed.parallel_state import get_dcp_group @@ -1164,7 +1378,7 @@ class FlashAttentionImpl(AttentionImpl): block_table = block_table[:, :num_pages] num_splits = 1 - flash_attn_varlen_func( + _FA4_DENSE_ATTENTION_KERNEL( q=query[:num_actual_tokens], k=key_cache, v=value_cache, @@ -1837,3 +2051,6 @@ def cascade_attention( # Merge prefix and suffix outputs, and store the result in output. merge_attn_states(output, prefix_output, prefix_lse, suffix_output, suffix_lse) + + +_FA4_DENSE_ATTENTION_KERNEL = FA4DenseAttentionKernel() diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index a3f6541b08..eccfdbabcd 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -8,7 +8,10 @@ from typing import TYPE_CHECKING, Any, cast import numpy as np import torch -from vllm.config import VllmConfig, get_layers_from_vllm_config +from vllm.config import ( + VllmConfig, + get_layers_from_vllm_config, +) from vllm.config.compilation import CUDAGraphMode from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index fe423d6e38..9d7918126f 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -632,12 +632,18 @@ class GPUModelRunner(LoRAModelRunnerMixin): draft_attn_layer_names = None if isinstance(self.speculator, DraftModelSpeculator): draft_attn_layer_names = self.speculator.draft_attn_layer_names - self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend( - self.kv_cache_config, - self.vllm_config, - self.device, - draft_layer_names=draft_attn_layer_names, - ) + # Metadata builders select attention kernels that need JIT warmup. + with self.jit_warmup_registry.activate(): + ( + self.attn_groups, + attn_cg_support, + self.kernel_block_sizes, + ) = init_attn_backend( + self.kv_cache_config, + self.vllm_config, + self.device, + draft_layer_names=draft_attn_layer_names, + ) additional_attn_cg_support = self.model_state.get_additional_cg_support() attn_cg_support = attn_cg_support.narrow(*additional_attn_cg_support) # The speculator clears the flag at load time when the checkpoint has diff --git a/vllm/vllm_flash_attn/flash_attn_interface.py b/vllm/vllm_flash_attn/flash_attn_interface.py index 81768e1209..4ba4692f67 100644 --- a/vllm/vllm_flash_attn/flash_attn_interface.py +++ b/vllm/vllm_flash_attn/flash_attn_interface.py @@ -445,6 +445,7 @@ def compile_flash_attn_varlen_func_from_specs( v_shape: tuple[int, ...], q_dtype: torch.dtype, v_stride: tuple[int, ...] | None = None, + k_stride: tuple[int, ...] | None = None, cu_seqlens_q_shape: tuple[int, ...] | None = None, cu_seqlens_k_shape: tuple[int, ...] | None = None, max_seqlen_q: int | None = None, @@ -457,6 +458,9 @@ def compile_flash_attn_varlen_func_from_specs( return_softmax_lse=False, num_splits: int = 0, fa_version: int = DEFAULT_FA_VERSION, + seqused_k_shape: tuple[int, ...] | None = None, + page_table_shape: tuple[int, ...] | None = None, + softcap: float | None = None, ) -> None: if fa_version != 4: raise ValueError( @@ -467,7 +471,8 @@ def compile_flash_attn_varlen_func_from_specs( del deterministic from vllm.vllm_flash_attn.cute.interface import ( - compile_flash_attn_varlen_func_from_specs as _fa4_compile_flash_attn_varlen_func_from_specs, + _flash_attn_fwd, + _make_compile_only_tensor_spec, ) real_window_size: tuple[int, int] @@ -480,21 +485,50 @@ def compile_flash_attn_varlen_func_from_specs( if softmax_scale is None: softmax_scale = q_shape[-1] ** (-0.5) - return _fa4_compile_flash_attn_varlen_func_from_specs( - q_shape=q_shape, - k_shape=k_shape, - v_shape=v_shape, - q_dtype=q_dtype, - v_stride=v_stride, - cu_seqlens_q_shape=cu_seqlens_q_shape, - cu_seqlens_k_shape=cu_seqlens_k_shape, + q = _make_compile_only_tensor_spec(q_shape, q_dtype) + k = _make_compile_only_tensor_spec(k_shape, q_dtype, stride=k_stride) + v = _make_compile_only_tensor_spec(v_shape, q_dtype, stride=v_stride) + cu_seqlens_q = _make_compile_only_tensor_spec(cu_seqlens_q_shape, torch.int32, 4) + out = _make_compile_only_tensor_spec( + (*q_shape[:-1], v_shape[-1]), + q_dtype, + ) + lse = None + if return_softmax_lse: + assert q is not None + if cu_seqlens_q_shape is None: + lse_shape = (*q.shape[:-3], q.shape[-2], q.shape[-3]) + lse_stride = None + else: + lse_shape = (q.shape[-2], q.shape[0]) + lse_stride = (None, 1) + lse = _make_compile_only_tensor_spec( + lse_shape, + torch.float32, + 4, + stride=lse_stride, + ) + + _flash_attn_fwd( + q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=_make_compile_only_tensor_spec(cu_seqlens_k_shape, torch.int32, 4), + seqused_k=_make_compile_only_tensor_spec(seqused_k_shape, torch.int32, 4), + page_table=_make_compile_only_tensor_spec(page_table_shape, torch.int32, 4), max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, softmax_scale=softmax_scale, causal=causal, - window_size=real_window_size, + softcap=softcap, + window_size_left=real_window_size[0], + window_size_right=real_window_size[1], num_splits=num_splits, return_lse=return_softmax_lse, + out=out, + lse=lse, + compile_only=True, )