[Warmup] Gemma 4 de-JITification (#55768)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-13 16:47:21 -04:00
committed by GitHub
co-authored by Codex
parent 586f652f8d
commit 186a1e6222
4 changed files with 288 additions and 28 deletions
+228 -11
View File
@@ -5,12 +5,13 @@
import copy
import functools
from dataclasses import dataclass
from typing import ClassVar
from typing import Any, ClassVar
import numpy as np
import torch
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, WarmupChoices
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
PIN_MEMORY,
@@ -46,6 +47,7 @@ from vllm.v1.worker.workspace import current_workspace_manager
if is_flash_attn_varlen_func_available():
from vllm.v1.attention.backends.fa_utils import (
compile_flash_attn_varlen_func_from_specs,
flash_attn_supports_sinks,
flash_attn_varlen_func,
get_scheduler_metadata,
@@ -67,7 +69,13 @@ from vllm.v1.attention.backend import (
AttentionMetadataBuilder,
CommonAttentionMetadata,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
from vllm.v1.kv_cache_interface import (
AttentionSpec,
FullAttentionSpec,
KVCacheSpec,
KVQuantMode,
SlidingWindowSpec,
)
from vllm.v1.worker.cp_utils import (
run_split_fa2_dcp_context_attention,
should_skip_dcp_context_attention,
@@ -77,6 +85,200 @@ from vllm.v1.worker.cp_utils import (
logger = init_logger(__name__)
FA4_DENSE_FLOAT_DTYPES = (torch.bfloat16, torch.float16)
FA4_DENSE_Q_TILE = 128
FA4_DENSE_NUM_BLOCKS = 256
FA4_DENSE_MAX_SEQLEN_K = 8192
class FA4DenseAttentionKernel(VllmJitKernel["FA4DenseAttentionKernel.CompileKey"]):
"""Warm paged, causal FA4 on Blackwell and Hopper."""
@dataclass(frozen=True)
class CompileKey:
q_stage: int
dtype: torch.dtype
qhead_per_kvhead: int
head_dim: int
page_size: int
has_window_left: bool
has_window_right: bool
softcap: float
is_single_batch: bool
num_splits: int
@staticmethod
def kernel(*args: Any, **kwargs: Any) -> Any:
assert flash_attn_varlen_func is not None
return flash_attn_varlen_func(*args, **kwargs)
def dispatch( # type: ignore[override]
self,
*,
q_stage: int,
dtype: torch.dtype,
num_qo_heads: int,
num_kv_heads: int,
head_dim: int,
page_size: int,
window_size: tuple[int, int],
softcap: float,
is_single_batch: bool = True,
num_splits: int = 1,
) -> CompileKey:
return self.CompileKey(
q_stage=q_stage,
dtype=dtype,
qhead_per_kvhead=num_qo_heads // num_kv_heads,
head_dim=head_dim,
page_size=page_size,
has_window_left=window_size[0] >= 0,
has_window_right=window_size[1] >= 0,
softcap=softcap,
is_single_batch=is_single_batch,
num_splits=num_splits,
)
def get_warmup_keys(
self,
*,
vllm_config: VllmConfig,
kv_cache_spec: AttentionSpec,
num_qo_heads: int,
) -> list[CompileKey]:
capability = current_platform.get_device_capability()
major = capability.major if capability is not None else None
if (
vllm_config.parallel_config.decode_context_parallel_size != 1
or vllm_config.model_config.rswa_window is not None
or kv_cache_spec.kv_quant_mode != KVQuantMode.NONE
or kv_cache_spec.dtype not in FA4_DENSE_FLOAT_DTYPES
or not isinstance(kv_cache_spec, (FullAttentionSpec, SlidingWindowSpec))
):
return []
head_dim = kv_cache_spec.head_size
page_size = kv_cache_spec.block_size
sliding_window = kv_cache_spec.sliding_window
window_size = (
(sliding_window - 1, 0) if sliding_window is not None else (-1, -1)
)
softcap = float(
getattr(
vllm_config.model_config.hf_text_config, "attn_logit_softcapping", 0
)
or 0
)
if (
major in (10, 11)
and uses_fa4_hd256_kernel(head_dim)
and page_size == FA4_HD256_PAGE_SIZE
):
q_stages = WarmupChoices(1, 2)
single_batches = WarmupChoices(True, False)
split_counts = WarmupChoices(1)
elif (
major == 9 and head_dim == 512 and window_size == (-1, -1) and softcap == 0
):
# SM90 forward has split/non-split variants; its transitive
# combine specializes at 32/64/128/256 splits.
q_stages = WarmupChoices(1)
single_batches = WarmupChoices(False)
split_counts = WarmupChoices(1, 32, 64, 128, 256)
else:
return []
return self._trace_dispatch(self.dispatch)(
q_stage=q_stages,
is_single_batch=single_batches,
num_splits=split_counts,
dtype=vllm_config.model_config.dtype,
num_qo_heads=num_qo_heads,
num_kv_heads=kv_cache_spec.num_kv_heads,
head_dim=head_dim,
page_size=page_size,
window_size=WarmupChoices(window_size),
softcap=softcap,
)
def compile(self, compile_key: CompileKey) -> None:
assert compile_flash_attn_varlen_func_from_specs is not None
max_seqlen_q = FA4_DENSE_Q_TILE + 1 if compile_key.q_stage == 2 else 1
batch_size = 1 if compile_key.is_single_batch else 2
kv_shape = (
FA4_DENSE_NUM_BLOCKS,
compile_key.page_size,
1,
compile_key.head_dim,
)
kv_stride = (
2 * compile_key.page_size * compile_key.head_dim,
2 * compile_key.head_dim,
2 * compile_key.head_dim,
1,
)
compile_flash_attn_varlen_func_from_specs(
q_shape=(
batch_size * max_seqlen_q,
compile_key.qhead_per_kvhead,
compile_key.head_dim,
),
k_shape=kv_shape,
v_shape=kv_shape,
k_stride=kv_stride,
v_stride=kv_stride,
q_dtype=compile_key.dtype,
cu_seqlens_q_shape=(batch_size + 1,),
seqused_k_shape=(batch_size,),
page_table_shape=(
batch_size,
FA4_DENSE_MAX_SEQLEN_K // compile_key.page_size,
),
max_seqlen_q=max_seqlen_q,
max_seqlen_k=FA4_DENSE_MAX_SEQLEN_K,
causal=True,
window_size=[
1 if compile_key.has_window_left else -1,
1 if compile_key.has_window_right else -1,
],
softcap=compile_key.softcap,
num_splits=2 if compile_key.num_splits > 1 else 1,
fa_version=4,
return_softmax_lse=False,
)
if compile_key.head_dim == 512 and compile_key.num_splits > 1:
from torch._subclasses.fake_tensor import FakeTensorMode
from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd_combine
# FA4's forward compile-only entry returns before its transitive
# SplitKV combine. Call the native combine entry in FakeTensorMode:
# it populates the native cache without launching the kernel.
with FakeTensorMode():
_flash_attn_fwd_combine(
torch.empty(
(compile_key.num_splits, 2, 1, 512),
dtype=torch.float32,
device="cuda",
),
torch.empty(
(compile_key.num_splits, 2, 1),
dtype=torch.float32,
device="cuda",
),
torch.empty((2, 1, 512), dtype=compile_key.dtype, device="cuda"),
cu_seqlens=torch.empty((3,), dtype=torch.int32, device="cuda"),
)
def __call__(
self,
*,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
**kwargs: Any,
) -> Any:
return self.kernel(q=q, k=k, v=v, **kwargs)
class FlashAttentionBackend(AttentionBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
@@ -447,15 +649,27 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
self.max_num_splits = 0 # No upper bound on the number of splits.
self.aot_schedule = get_flash_attn_version() == 3
self.fa4_hd256 = uses_fa4_hd256_kernel(self.headdim) and (
get_flash_attn_version(
head_size=self.headdim,
kv_cache_block_size=self.block_size,
supports_fa4_hd256=True,
)
== 4
head_size_v = getattr(kv_cache_spec, "head_size_v", None)
fa_version = get_flash_attn_version(
head_size=self.headdim,
head_size_v=head_size_v,
kv_cache_block_size=self.block_size,
supports_fa4_hd256=True,
)
self.fa4_hd256 = fa_version == 4 and uses_fa4_hd256_kernel(
self.headdim, head_size_v
)
if (
vllm_config.kernel_config.enable_jit_warmup
and self.model_config.hf_config.model_type in ("gemma4", "gemma4_unified")
and fa_version == 4
):
_FA4_DENSE_ATTENTION_KERNEL.register_warmup(
vllm_config=vllm_config,
kv_cache_spec=kv_cache_spec,
num_qo_heads=self.num_heads_q,
)
try:
from vllm.distributed.parallel_state import get_dcp_group
@@ -1164,7 +1378,7 @@ class FlashAttentionImpl(AttentionImpl):
block_table = block_table[:, :num_pages]
num_splits = 1
flash_attn_varlen_func(
_FA4_DENSE_ATTENTION_KERNEL(
q=query[:num_actual_tokens],
k=key_cache,
v=value_cache,
@@ -1837,3 +2051,6 @@ def cascade_attention(
# Merge prefix and suffix outputs, and store the result in output.
merge_attn_states(output, prefix_output, prefix_lse, suffix_output, suffix_lse)
_FA4_DENSE_ATTENTION_KERNEL = FA4DenseAttentionKernel()
+4 -1
View File
@@ -8,7 +8,10 @@ from typing import TYPE_CHECKING, Any, cast
import numpy as np
import torch
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config import (
VllmConfig,
get_layers_from_vllm_config,
)
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
+12 -6
View File
@@ -632,12 +632,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
draft_attn_layer_names = None
if isinstance(self.speculator, DraftModelSpeculator):
draft_attn_layer_names = self.speculator.draft_attn_layer_names
self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend(
self.kv_cache_config,
self.vllm_config,
self.device,
draft_layer_names=draft_attn_layer_names,
)
# Metadata builders select attention kernels that need JIT warmup.
with self.jit_warmup_registry.activate():
(
self.attn_groups,
attn_cg_support,
self.kernel_block_sizes,
) = init_attn_backend(
self.kv_cache_config,
self.vllm_config,
self.device,
draft_layer_names=draft_attn_layer_names,
)
additional_attn_cg_support = self.model_state.get_additional_cg_support()
attn_cg_support = attn_cg_support.narrow(*additional_attn_cg_support)
# The speculator clears the flag at load time when the checkpoint has
+44 -10
View File
@@ -445,6 +445,7 @@ def compile_flash_attn_varlen_func_from_specs(
v_shape: tuple[int, ...],
q_dtype: torch.dtype,
v_stride: tuple[int, ...] | None = None,
k_stride: tuple[int, ...] | None = None,
cu_seqlens_q_shape: tuple[int, ...] | None = None,
cu_seqlens_k_shape: tuple[int, ...] | None = None,
max_seqlen_q: int | None = None,
@@ -457,6 +458,9 @@ def compile_flash_attn_varlen_func_from_specs(
return_softmax_lse=False,
num_splits: int = 0,
fa_version: int = DEFAULT_FA_VERSION,
seqused_k_shape: tuple[int, ...] | None = None,
page_table_shape: tuple[int, ...] | None = None,
softcap: float | None = None,
) -> None:
if fa_version != 4:
raise ValueError(
@@ -467,7 +471,8 @@ def compile_flash_attn_varlen_func_from_specs(
del deterministic
from vllm.vllm_flash_attn.cute.interface import (
compile_flash_attn_varlen_func_from_specs as _fa4_compile_flash_attn_varlen_func_from_specs,
_flash_attn_fwd,
_make_compile_only_tensor_spec,
)
real_window_size: tuple[int, int]
@@ -480,21 +485,50 @@ def compile_flash_attn_varlen_func_from_specs(
if softmax_scale is None:
softmax_scale = q_shape[-1] ** (-0.5)
return _fa4_compile_flash_attn_varlen_func_from_specs(
q_shape=q_shape,
k_shape=k_shape,
v_shape=v_shape,
q_dtype=q_dtype,
v_stride=v_stride,
cu_seqlens_q_shape=cu_seqlens_q_shape,
cu_seqlens_k_shape=cu_seqlens_k_shape,
q = _make_compile_only_tensor_spec(q_shape, q_dtype)
k = _make_compile_only_tensor_spec(k_shape, q_dtype, stride=k_stride)
v = _make_compile_only_tensor_spec(v_shape, q_dtype, stride=v_stride)
cu_seqlens_q = _make_compile_only_tensor_spec(cu_seqlens_q_shape, torch.int32, 4)
out = _make_compile_only_tensor_spec(
(*q_shape[:-1], v_shape[-1]),
q_dtype,
)
lse = None
if return_softmax_lse:
assert q is not None
if cu_seqlens_q_shape is None:
lse_shape = (*q.shape[:-3], q.shape[-2], q.shape[-3])
lse_stride = None
else:
lse_shape = (q.shape[-2], q.shape[0])
lse_stride = (None, 1)
lse = _make_compile_only_tensor_spec(
lse_shape,
torch.float32,
4,
stride=lse_stride,
)
_flash_attn_fwd(
q=q,
k=k,
v=v,
cu_seqlens_q=cu_seqlens_q,
cu_seqlens_k=_make_compile_only_tensor_spec(cu_seqlens_k_shape, torch.int32, 4),
seqused_k=_make_compile_only_tensor_spec(seqused_k_shape, torch.int32, 4),
page_table=_make_compile_only_tensor_spec(page_table_shape, torch.int32, 4),
max_seqlen_q=max_seqlen_q,
max_seqlen_k=max_seqlen_k,
softmax_scale=softmax_scale,
causal=causal,
window_size=real_window_size,
softcap=softcap,
window_size_left=real_window_size[0],
window_size_right=real_window_size[1],
num_splits=num_splits,
return_lse=return_softmax_lse,
out=out,
lse=lse,
compile_only=True,
)