[Warmup] Gemma 4 de-JITification (#55768)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
parent
586f652f8d
commit
186a1e6222
@@ -5,12 +5,13 @@
|
||||
import copy
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, WarmupChoices
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
@@ -46,6 +47,7 @@ from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if is_flash_attn_varlen_func_available():
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
compile_flash_attn_varlen_func_from_specs,
|
||||
flash_attn_supports_sinks,
|
||||
flash_attn_varlen_func,
|
||||
get_scheduler_metadata,
|
||||
@@ -67,7 +69,13 @@ from vllm.v1.attention.backend import (
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
KVQuantMode,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.worker.cp_utils import (
|
||||
run_split_fa2_dcp_context_attention,
|
||||
should_skip_dcp_context_attention,
|
||||
@@ -77,6 +85,200 @@ from vllm.v1.worker.cp_utils import (
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
FA4_DENSE_FLOAT_DTYPES = (torch.bfloat16, torch.float16)
|
||||
FA4_DENSE_Q_TILE = 128
|
||||
FA4_DENSE_NUM_BLOCKS = 256
|
||||
FA4_DENSE_MAX_SEQLEN_K = 8192
|
||||
|
||||
|
||||
class FA4DenseAttentionKernel(VllmJitKernel["FA4DenseAttentionKernel.CompileKey"]):
|
||||
"""Warm paged, causal FA4 on Blackwell and Hopper."""
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
q_stage: int
|
||||
dtype: torch.dtype
|
||||
qhead_per_kvhead: int
|
||||
head_dim: int
|
||||
page_size: int
|
||||
has_window_left: bool
|
||||
has_window_right: bool
|
||||
softcap: float
|
||||
is_single_batch: bool
|
||||
num_splits: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(*args: Any, **kwargs: Any) -> Any:
|
||||
assert flash_attn_varlen_func is not None
|
||||
return flash_attn_varlen_func(*args, **kwargs)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
q_stage: int,
|
||||
dtype: torch.dtype,
|
||||
num_qo_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
page_size: int,
|
||||
window_size: tuple[int, int],
|
||||
softcap: float,
|
||||
is_single_batch: bool = True,
|
||||
num_splits: int = 1,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
q_stage=q_stage,
|
||||
dtype=dtype,
|
||||
qhead_per_kvhead=num_qo_heads // num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
page_size=page_size,
|
||||
has_window_left=window_size[0] >= 0,
|
||||
has_window_right=window_size[1] >= 0,
|
||||
softcap=softcap,
|
||||
is_single_batch=is_single_batch,
|
||||
num_splits=num_splits,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
num_qo_heads: int,
|
||||
) -> list[CompileKey]:
|
||||
capability = current_platform.get_device_capability()
|
||||
major = capability.major if capability is not None else None
|
||||
if (
|
||||
vllm_config.parallel_config.decode_context_parallel_size != 1
|
||||
or vllm_config.model_config.rswa_window is not None
|
||||
or kv_cache_spec.kv_quant_mode != KVQuantMode.NONE
|
||||
or kv_cache_spec.dtype not in FA4_DENSE_FLOAT_DTYPES
|
||||
or not isinstance(kv_cache_spec, (FullAttentionSpec, SlidingWindowSpec))
|
||||
):
|
||||
return []
|
||||
head_dim = kv_cache_spec.head_size
|
||||
page_size = kv_cache_spec.block_size
|
||||
sliding_window = kv_cache_spec.sliding_window
|
||||
window_size = (
|
||||
(sliding_window - 1, 0) if sliding_window is not None else (-1, -1)
|
||||
)
|
||||
softcap = float(
|
||||
getattr(
|
||||
vllm_config.model_config.hf_text_config, "attn_logit_softcapping", 0
|
||||
)
|
||||
or 0
|
||||
)
|
||||
if (
|
||||
major in (10, 11)
|
||||
and uses_fa4_hd256_kernel(head_dim)
|
||||
and page_size == FA4_HD256_PAGE_SIZE
|
||||
):
|
||||
q_stages = WarmupChoices(1, 2)
|
||||
single_batches = WarmupChoices(True, False)
|
||||
split_counts = WarmupChoices(1)
|
||||
elif (
|
||||
major == 9 and head_dim == 512 and window_size == (-1, -1) and softcap == 0
|
||||
):
|
||||
# SM90 forward has split/non-split variants; its transitive
|
||||
# combine specializes at 32/64/128/256 splits.
|
||||
q_stages = WarmupChoices(1)
|
||||
single_batches = WarmupChoices(False)
|
||||
split_counts = WarmupChoices(1, 32, 64, 128, 256)
|
||||
else:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
q_stage=q_stages,
|
||||
is_single_batch=single_batches,
|
||||
num_splits=split_counts,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
page_size=page_size,
|
||||
window_size=WarmupChoices(window_size),
|
||||
softcap=softcap,
|
||||
)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
assert compile_flash_attn_varlen_func_from_specs is not None
|
||||
max_seqlen_q = FA4_DENSE_Q_TILE + 1 if compile_key.q_stage == 2 else 1
|
||||
batch_size = 1 if compile_key.is_single_batch else 2
|
||||
kv_shape = (
|
||||
FA4_DENSE_NUM_BLOCKS,
|
||||
compile_key.page_size,
|
||||
1,
|
||||
compile_key.head_dim,
|
||||
)
|
||||
kv_stride = (
|
||||
2 * compile_key.page_size * compile_key.head_dim,
|
||||
2 * compile_key.head_dim,
|
||||
2 * compile_key.head_dim,
|
||||
1,
|
||||
)
|
||||
compile_flash_attn_varlen_func_from_specs(
|
||||
q_shape=(
|
||||
batch_size * max_seqlen_q,
|
||||
compile_key.qhead_per_kvhead,
|
||||
compile_key.head_dim,
|
||||
),
|
||||
k_shape=kv_shape,
|
||||
v_shape=kv_shape,
|
||||
k_stride=kv_stride,
|
||||
v_stride=kv_stride,
|
||||
q_dtype=compile_key.dtype,
|
||||
cu_seqlens_q_shape=(batch_size + 1,),
|
||||
seqused_k_shape=(batch_size,),
|
||||
page_table_shape=(
|
||||
batch_size,
|
||||
FA4_DENSE_MAX_SEQLEN_K // compile_key.page_size,
|
||||
),
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
max_seqlen_k=FA4_DENSE_MAX_SEQLEN_K,
|
||||
causal=True,
|
||||
window_size=[
|
||||
1 if compile_key.has_window_left else -1,
|
||||
1 if compile_key.has_window_right else -1,
|
||||
],
|
||||
softcap=compile_key.softcap,
|
||||
num_splits=2 if compile_key.num_splits > 1 else 1,
|
||||
fa_version=4,
|
||||
return_softmax_lse=False,
|
||||
)
|
||||
if compile_key.head_dim == 512 and compile_key.num_splits > 1:
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from vllm.vllm_flash_attn.cute.interface import _flash_attn_fwd_combine
|
||||
|
||||
# FA4's forward compile-only entry returns before its transitive
|
||||
# SplitKV combine. Call the native combine entry in FakeTensorMode:
|
||||
# it populates the native cache without launching the kernel.
|
||||
with FakeTensorMode():
|
||||
_flash_attn_fwd_combine(
|
||||
torch.empty(
|
||||
(compile_key.num_splits, 2, 1, 512),
|
||||
dtype=torch.float32,
|
||||
device="cuda",
|
||||
),
|
||||
torch.empty(
|
||||
(compile_key.num_splits, 2, 1),
|
||||
dtype=torch.float32,
|
||||
device="cuda",
|
||||
),
|
||||
torch.empty((2, 1, 512), dtype=compile_key.dtype, device="cuda"),
|
||||
cu_seqlens=torch.empty((3,), dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
return self.kernel(q=q, k=k, v=v, **kwargs)
|
||||
|
||||
|
||||
class FlashAttentionBackend(AttentionBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
@@ -447,15 +649,27 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
|
||||
self.max_num_splits = 0 # No upper bound on the number of splits.
|
||||
self.aot_schedule = get_flash_attn_version() == 3
|
||||
|
||||
self.fa4_hd256 = uses_fa4_hd256_kernel(self.headdim) and (
|
||||
get_flash_attn_version(
|
||||
head_size=self.headdim,
|
||||
kv_cache_block_size=self.block_size,
|
||||
supports_fa4_hd256=True,
|
||||
)
|
||||
== 4
|
||||
head_size_v = getattr(kv_cache_spec, "head_size_v", None)
|
||||
fa_version = get_flash_attn_version(
|
||||
head_size=self.headdim,
|
||||
head_size_v=head_size_v,
|
||||
kv_cache_block_size=self.block_size,
|
||||
supports_fa4_hd256=True,
|
||||
)
|
||||
self.fa4_hd256 = fa_version == 4 and uses_fa4_hd256_kernel(
|
||||
self.headdim, head_size_v
|
||||
)
|
||||
|
||||
if (
|
||||
vllm_config.kernel_config.enable_jit_warmup
|
||||
and self.model_config.hf_config.model_type in ("gemma4", "gemma4_unified")
|
||||
and fa_version == 4
|
||||
):
|
||||
_FA4_DENSE_ATTENTION_KERNEL.register_warmup(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_spec=kv_cache_spec,
|
||||
num_qo_heads=self.num_heads_q,
|
||||
)
|
||||
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
@@ -1164,7 +1378,7 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
block_table = block_table[:, :num_pages]
|
||||
num_splits = 1
|
||||
|
||||
flash_attn_varlen_func(
|
||||
_FA4_DENSE_ATTENTION_KERNEL(
|
||||
q=query[:num_actual_tokens],
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
@@ -1837,3 +2051,6 @@ def cascade_attention(
|
||||
|
||||
# Merge prefix and suffix outputs, and store the result in output.
|
||||
merge_attn_states(output, prefix_output, prefix_lse, suffix_output, suffix_lse)
|
||||
|
||||
|
||||
_FA4_DENSE_ATTENTION_KERNEL = FA4DenseAttentionKernel()
|
||||
|
||||
@@ -8,7 +8,10 @@ from typing import TYPE_CHECKING, Any, cast
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_layers_from_vllm_config,
|
||||
)
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
|
||||
@@ -632,12 +632,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
draft_attn_layer_names = None
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
draft_attn_layer_names = self.speculator.draft_attn_layer_names
|
||||
self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend(
|
||||
self.kv_cache_config,
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
draft_layer_names=draft_attn_layer_names,
|
||||
)
|
||||
# Metadata builders select attention kernels that need JIT warmup.
|
||||
with self.jit_warmup_registry.activate():
|
||||
(
|
||||
self.attn_groups,
|
||||
attn_cg_support,
|
||||
self.kernel_block_sizes,
|
||||
) = init_attn_backend(
|
||||
self.kv_cache_config,
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
draft_layer_names=draft_attn_layer_names,
|
||||
)
|
||||
additional_attn_cg_support = self.model_state.get_additional_cg_support()
|
||||
attn_cg_support = attn_cg_support.narrow(*additional_attn_cg_support)
|
||||
# The speculator clears the flag at load time when the checkpoint has
|
||||
|
||||
@@ -445,6 +445,7 @@ def compile_flash_attn_varlen_func_from_specs(
|
||||
v_shape: tuple[int, ...],
|
||||
q_dtype: torch.dtype,
|
||||
v_stride: tuple[int, ...] | None = None,
|
||||
k_stride: tuple[int, ...] | None = None,
|
||||
cu_seqlens_q_shape: tuple[int, ...] | None = None,
|
||||
cu_seqlens_k_shape: tuple[int, ...] | None = None,
|
||||
max_seqlen_q: int | None = None,
|
||||
@@ -457,6 +458,9 @@ def compile_flash_attn_varlen_func_from_specs(
|
||||
return_softmax_lse=False,
|
||||
num_splits: int = 0,
|
||||
fa_version: int = DEFAULT_FA_VERSION,
|
||||
seqused_k_shape: tuple[int, ...] | None = None,
|
||||
page_table_shape: tuple[int, ...] | None = None,
|
||||
softcap: float | None = None,
|
||||
) -> None:
|
||||
if fa_version != 4:
|
||||
raise ValueError(
|
||||
@@ -467,7 +471,8 @@ def compile_flash_attn_varlen_func_from_specs(
|
||||
del deterministic
|
||||
|
||||
from vllm.vllm_flash_attn.cute.interface import (
|
||||
compile_flash_attn_varlen_func_from_specs as _fa4_compile_flash_attn_varlen_func_from_specs,
|
||||
_flash_attn_fwd,
|
||||
_make_compile_only_tensor_spec,
|
||||
)
|
||||
|
||||
real_window_size: tuple[int, int]
|
||||
@@ -480,21 +485,50 @@ def compile_flash_attn_varlen_func_from_specs(
|
||||
if softmax_scale is None:
|
||||
softmax_scale = q_shape[-1] ** (-0.5)
|
||||
|
||||
return _fa4_compile_flash_attn_varlen_func_from_specs(
|
||||
q_shape=q_shape,
|
||||
k_shape=k_shape,
|
||||
v_shape=v_shape,
|
||||
q_dtype=q_dtype,
|
||||
v_stride=v_stride,
|
||||
cu_seqlens_q_shape=cu_seqlens_q_shape,
|
||||
cu_seqlens_k_shape=cu_seqlens_k_shape,
|
||||
q = _make_compile_only_tensor_spec(q_shape, q_dtype)
|
||||
k = _make_compile_only_tensor_spec(k_shape, q_dtype, stride=k_stride)
|
||||
v = _make_compile_only_tensor_spec(v_shape, q_dtype, stride=v_stride)
|
||||
cu_seqlens_q = _make_compile_only_tensor_spec(cu_seqlens_q_shape, torch.int32, 4)
|
||||
out = _make_compile_only_tensor_spec(
|
||||
(*q_shape[:-1], v_shape[-1]),
|
||||
q_dtype,
|
||||
)
|
||||
lse = None
|
||||
if return_softmax_lse:
|
||||
assert q is not None
|
||||
if cu_seqlens_q_shape is None:
|
||||
lse_shape = (*q.shape[:-3], q.shape[-2], q.shape[-3])
|
||||
lse_stride = None
|
||||
else:
|
||||
lse_shape = (q.shape[-2], q.shape[0])
|
||||
lse_stride = (None, 1)
|
||||
lse = _make_compile_only_tensor_spec(
|
||||
lse_shape,
|
||||
torch.float32,
|
||||
4,
|
||||
stride=lse_stride,
|
||||
)
|
||||
|
||||
_flash_attn_fwd(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
cu_seqlens_k=_make_compile_only_tensor_spec(cu_seqlens_k_shape, torch.int32, 4),
|
||||
seqused_k=_make_compile_only_tensor_spec(seqused_k_shape, torch.int32, 4),
|
||||
page_table=_make_compile_only_tensor_spec(page_table_shape, torch.int32, 4),
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
window_size=real_window_size,
|
||||
softcap=softcap,
|
||||
window_size_left=real_window_size[0],
|
||||
window_size_right=real_window_size[1],
|
||||
num_splits=num_splits,
|
||||
return_lse=return_softmax_lse,
|
||||
out=out,
|
||||
lse=lse,
|
||||
compile_only=True,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user