[Feat][1/N] CuTeDSL warmup infrastructure, FA4 MLA (#46182)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Roberto L. Castro
2026-06-30 12:17:34 -07:00
committed by GitHub
co-authored by Lucas Wilkinson
parent 11b26c5528
commit 248d1fbb71
9 changed files with 518 additions and 2 deletions
@@ -39,7 +39,7 @@ else()
FetchContent_Declare(
vllm-flash-attn
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
GIT_TAG b3964b1d8b95d8e8447435668ab169a2700bab65
GIT_TAG 2c839c33742309ec41e620bf837495ec9926c56e
GIT_PROGRESS TRUE
# Don't share the vllm-flash-attn build between build types
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
+9 -1
View File
@@ -172,6 +172,9 @@ class KernelConfig:
enable_flashinfer_autotune: bool = None # type: ignore[assignment]
"""If True, run FlashInfer autotuning during kernel warmup."""
enable_cutedsl_warmup: bool = True
"""If True, run CuTeDSL compile warmup during kernel warmup."""
moe_backend: MoEBackend = "auto"
"""Backend for MoE expert computation kernels. Available options:
@@ -237,6 +240,7 @@ class KernelConfig:
Any future fields that don't affect compilation should be excluded.
"""
ignored_factors = {
"enable_cutedsl_warmup",
"enable_flashinfer_autotune",
"ir_op_priority", # handled separately below
}
@@ -244,7 +248,11 @@ class KernelConfig:
factors["ir_op_priority"] = self.ir_op_priority.compute_hash()
return hash_factors(factors)
@field_validator("enable_flashinfer_autotune", mode="wrap")
@field_validator(
"enable_flashinfer_autotune",
"enable_cutedsl_warmup",
mode="wrap",
)
@classmethod
def _skip_none_validation(cls, value: Any, handler: Callable) -> Any:
"""Skip validation if the value is `None` when initialization is delayed."""
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Run registered CuTeDSL warmup compile units."""
from __future__ import annotations
import time
import weakref
from collections.abc import Callable, Hashable, Iterable
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.tracing import instrument
logger = init_logger(__name__)
CuTeDSLCompileFn = Callable[[], None]
@dataclass(frozen=True)
class CuTeDSLCompileUnit:
name: str
key: Hashable
compile: CuTeDSLCompileFn
_CUTEDSL_WARMUP_PROVIDERS: weakref.WeakSet[object] = weakref.WeakSet()
def register_cutedsl_warmup_provider(provider: object) -> None:
"""Register an object that can expose CuTeDSL warmup compile units."""
_CUTEDSL_WARMUP_PROVIDERS.add(provider)
# Yield compile units from registered providers.
def _iter_cutedsl_warmup_compile_units() -> Iterable[CuTeDSLCompileUnit]:
for provider in tuple(_CUTEDSL_WARMUP_PROVIDERS):
get_units = getattr(provider, "get_cutedsl_warmup_compile_units", None)
if not callable(get_units):
continue
compile_units = get_units()
if compile_units is None:
continue
for unit in compile_units:
if not isinstance(unit, CuTeDSLCompileUnit):
raise TypeError(
"get_cutedsl_warmup_compile_units must return "
"CuTeDSLCompileUnit objects"
)
yield unit
# Drop duplicate compile units across providers.
def _collect_unique_compile_units(
compile_units: Iterable[CuTeDSLCompileUnit],
) -> list[CuTeDSLCompileUnit]:
seen: set[Hashable] = set()
unique_compile_units: list[CuTeDSLCompileUnit] = []
for unit in compile_units:
if unit.key in seen:
continue
seen.add(unit.key)
unique_compile_units.append(unit)
return unique_compile_units
# Execute compile units under inference mode.
def _compile_cutedsl_warmup_units(
compile_units: Iterable[CuTeDSLCompileUnit],
) -> int:
compiled = 0
with torch.inference_mode():
for unit in compile_units:
unit.compile()
compiled += 1
torch.accelerator.synchronize()
return compiled
# Run all CuTeDSL warmup before serving.
@instrument(span_name="CuTeDSL warmup")
def cutedsl_warmup() -> None:
"""Run CuTeDSL compile providers before serving."""
if not current_platform.is_cuda():
logger.info("Skipping CuTeDSL warmup on non-CUDA platform.")
return
compile_units = _collect_unique_compile_units(_iter_cutedsl_warmup_compile_units())
if not compile_units:
logger.info("Skipping CuTeDSL warmup because no compile units were requested.")
return
unit_names = list(dict.fromkeys(unit.name for unit in compile_units))
logger.info(
"Warming up CuTeDSL compile_units=%d names=%s.",
len(compile_units),
unit_names,
)
start_time = time.perf_counter()
compiled_count = _compile_cutedsl_warmup_units(compile_units)
logger.info(
"CuTeDSL warmup compiled %d units in %.2f s.",
compiled_count,
time.perf_counter() - start_time,
)
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FA4 MLA prefill CuTeDSL compile warmup config."""
from __future__ import annotations
from collections.abc import Hashable, Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Literal
import torch
if TYPE_CHECKING:
from vllm.v1.attention.backends.fa_utils import (
FlashAttentionCuTeDSLCompileSpec,
)
FA4ArchitectureFamily = Literal["sm90", "sm100f", "sm120"]
FA4_STANDARD_DTYPES = (torch.bfloat16, torch.float16)
# Current vLLM MLA prefill expands K/V to num_heads before FA4, so this plan
# covers qhead_per_kvhead=1.
# Batch is not a current FA4 MLA-prefill key field. Use b1 for compile-only
# specs because it is the conservative case for Split-KV shape heuristics.
# TODO(roberto): FA4 also has direct-GQA and qv/top-k absorbed-MLA paths, but vLLM
# does not use them in this backend yet; they need a separate
# num_kv_heads/qv/top-k-aware warmup plan if wired in later.
FA4_MLA_PREFILL_COMPILE_BATCH_SIZE = 1
FA4_MLA_PREFILL_Q_TILE = 128
FA4_MLA_PREFILL_K_TILE = 128
FA4_MLA_PREFILL_LONG_K_BLOCKS = 32
FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS = 64
FA4_MLA_PREFILL_CAUSAL_OPTIONS = (False, True)
FA4_MLA_PREFILL_LSE_OPTIONS = (False, True)
@dataclass(frozen=True)
class FA4MLAPrefillCompileContext:
dtype: torch.dtype
num_heads: int
qk_head_dim: int
v_head_dim: int
kv_nope_head_dim: int
requires_v_padding: bool
scale: float
num_splits: int
fa_version: int
# Return the V head dim FA4 sees.
@property
def effective_v_head_dim(self) -> int:
if self.requires_v_padding:
return self.qk_head_dim
return self.v_head_dim
@dataclass(frozen=True)
class FA4MLAPrefillCompileRequest:
"""One compile-only FA4 MLA prefill request."""
key: Hashable
compile_spec: FlashAttentionCuTeDSLCompileSpec
# Compile this request.
def compile(self) -> None:
self.compile_spec.compile()
# Yield deduped compile requests.
def iter_fa4_mla_prefill_compile_requests(
ctx: FA4MLAPrefillCompileContext,
) -> Iterator[FA4MLAPrefillCompileRequest]:
"""Yield compile requests for this fixed MLA backend.
FA4 dedupes duplicate atomic kernel selections in its own JIT cache.
"""
seen: set[Hashable] = set()
for compile_spec in iter_fa4_mla_prefill_compile_specs(ctx):
key = compile_spec.request_key()
if key in seen:
continue
seen.add(key)
yield FA4MLAPrefillCompileRequest(
key=key,
compile_spec=compile_spec,
)
# Build compile specs for this setup.
def iter_fa4_mla_prefill_compile_specs(
ctx: FA4MLAPrefillCompileContext,
) -> Iterator[FlashAttentionCuTeDSLCompileSpec]:
"""Yield compile-only FA4 MLA prefill requests for this fixed setup."""
arch_family = _fa4_architecture_family_from_compute_capability(
*torch.cuda.get_device_capability()
)
if not _supports_fa4_mla_prefill(ctx, arch_family):
return
from vllm.v1.attention.backends.fa_utils import (
FlashAttentionCuTeDSLCompileSpec,
)
batch_size = FA4_MLA_PREFILL_COMPILE_BATCH_SIZE
v_stride = None
if not ctx.requires_v_padding:
v_stride = (
ctx.num_heads * ctx.kv_nope_head_dim,
ctx.kv_nope_head_dim,
1,
)
for _, max_seqlen_q, max_seqlen_k in _shape_probes_for_context(ctx, arch_family):
total_q_tokens = batch_size * max_seqlen_q
total_kv_tokens = batch_size * max_seqlen_k
for causal in FA4_MLA_PREFILL_CAUSAL_OPTIONS:
for return_lse in FA4_MLA_PREFILL_LSE_OPTIONS:
yield FlashAttentionCuTeDSLCompileSpec(
q_shape=(total_q_tokens, ctx.num_heads, ctx.qk_head_dim),
k_shape=(total_kv_tokens, ctx.num_heads, ctx.qk_head_dim),
v_shape=(
total_kv_tokens,
ctx.num_heads,
ctx.effective_v_head_dim,
),
v_stride=v_stride,
q_dtype=ctx.dtype,
cu_seqlens_q_shape=(batch_size + 1,),
cu_seqlens_k_shape=(batch_size + 1,),
max_seqlen_q=max_seqlen_q,
max_seqlen_k=max_seqlen_k,
softmax_scale=ctx.scale,
causal=causal,
return_softmax_lse=return_lse,
num_splits=ctx.num_splits,
fa_version=ctx.fa_version,
)
# Pick one q/k point per current FA4 MLA-prefill shape regime.
def _shape_probes_for_context(
ctx: FA4MLAPrefillCompileContext,
arch_family: FA4ArchitectureFamily,
) -> tuple[tuple[str, int, int], ...]:
q_stage1_q = 1
q_stage2_q = FA4_MLA_PREFILL_Q_TILE + 1
# FA4 never auto-splits when ceil(max_seqlen_k / tile_n) <= 4.
no_split_k = 4 * FA4_MLA_PREFILL_K_TILE
long_k = FA4_MLA_PREFILL_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
# Diff-head-dim Blackwell Split-KV switches tile_n at 64 K blocks.
very_long_k = FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
base_probes = (
("q_stage1", q_stage1_q, FA4_MLA_PREFILL_K_TILE),
("q_stage2", q_stage2_q, no_split_k),
)
# SM120 currently rejects Split-KV in FA4; num_splits=1 also has no split
# shape regimes on any architecture.
if ctx.num_splits == 1 or arch_family == "sm120":
return base_probes
long_k_probes = (
("q_stage1_long_k", q_stage1_q, long_k),
("q_stage2_long_k", q_stage2_q, long_k),
)
# SM90 does not have the SM100 q_stage or diff-head-dim tile_n=64 branch.
# Same-dim SM100-family MLA also does not need the very-long-K probe.
if arch_family == "sm90" or ctx.qk_head_dim == ctx.effective_v_head_dim:
return (*base_probes, *long_k_probes)
very_long_k_probes = (
("q_stage1_very_long_k", q_stage1_q, very_long_k),
("q_stage2_very_long_k", q_stage2_q, very_long_k),
)
return (*base_probes, *long_k_probes, *very_long_k_probes)
# Check whether this setup can use FA4 MLA prefill.
def _supports_fa4_mla_prefill(
ctx: FA4MLAPrefillCompileContext,
arch_family: FA4ArchitectureFamily,
) -> bool:
return (
ctx.dtype in FA4_STANDARD_DTYPES
and ctx.num_heads > 0
and (arch_family != "sm120" or ctx.num_splits == 1)
)
# Map CUDA capability to the FA4 arch family used by warmup checks.
def _fa4_architecture_family_from_compute_capability(
major: int,
minor: int,
) -> FA4ArchitectureFamily:
if (major, minor) == (9, 0):
return "sm90"
if major == 10:
return "sm100f"
if (major, minor) == (12, 0):
return "sm120"
raise ValueError(f"FA4 warmup does not know CUDA capability {major}.{minor}")
@@ -12,6 +12,7 @@ import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
deepseek_v4_mhc_warmup,
@@ -125,6 +126,9 @@ def kernel_warmup(worker: "Worker"):
create_mixed_batch=True,
)
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
cutedsl_warmup()
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"""
+76
View File
@@ -1,8 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.platforms import current_platform
@@ -18,6 +21,7 @@ _ROCM_FLASH_ATTN_AVAILABLE = False
if current_platform.is_cuda():
from vllm._custom_ops import reshape_and_cache_flash
from vllm.vllm_flash_attn import ( # type: ignore[attr-defined]
compile_flash_attn_varlen_func_from_specs,
flash_attn_varlen_func,
get_scheduler_metadata,
)
@@ -28,11 +32,14 @@ elif current_platform.is_xpu():
reshape_and_cache_flash = ops.reshape_and_cache_flash
flash_attn_varlen_func = xpu_ops.flash_attn_varlen_func # type: ignore[assignment]
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
get_scheduler_metadata = xpu_ops.get_scheduler_metadata # type: ignore[assignment]
elif current_platform.is_rocm():
try:
from flash_attn import flash_attn_varlen_func # type: ignore[no-redef]
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
# Mark that upstream flash-attn is available on ROCm
_ROCM_FLASH_ATTN_AVAILABLE = True
except ImportError:
@@ -43,6 +50,8 @@ elif current_platform.is_rocm():
"to be installed. Please install flash-attn first."
)
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
# ROCm doesn't use scheduler metadata (FA3 feature), provide stub
def get_scheduler_metadata(*args: Any, **kwargs: Any) -> None: # type: ignore[misc]
return None
@@ -53,6 +62,73 @@ elif current_platform.is_rocm():
reshape_and_cache_flash = ops.reshape_and_cache_flash
@dataclass(frozen=True)
class FlashAttentionCuTeDSLCompileSpec:
"""High-level FA4 compile-only request used by vLLM warmup.
This is not the CuTeDSL cache key. FA4 owns the selector that maps these
serving inputs to the actual compile-static fields: tile sizes, q_stage,
Split-KV, scheduler choice, layout-presence booleans, dtype/head dims,
arch, and related fields.
"""
q_shape: tuple[int, ...]
k_shape: tuple[int, ...]
v_shape: tuple[int, ...]
q_dtype: torch.dtype
max_seqlen_q: int
max_seqlen_k: int
softmax_scale: float
causal: bool
fa_version: int
v_stride: tuple[int, ...] | None = None
cu_seqlens_q_shape: tuple[int, ...] | None = None
cu_seqlens_k_shape: tuple[int, ...] | None = None
window_size: tuple[int, int] | None = None
return_softmax_lse: bool = False
num_splits: int = 0
def compile(self) -> None:
assert compile_flash_attn_varlen_func_from_specs is not None
window_size = list(self.window_size) if self.window_size is not None else None
compile_flash_attn_varlen_func_from_specs(
q_shape=self.q_shape,
k_shape=self.k_shape,
v_shape=self.v_shape,
q_dtype=self.q_dtype,
v_stride=self.v_stride,
cu_seqlens_q_shape=self.cu_seqlens_q_shape,
cu_seqlens_k_shape=self.cu_seqlens_k_shape,
max_seqlen_q=self.max_seqlen_q,
max_seqlen_k=self.max_seqlen_k,
softmax_scale=self.softmax_scale,
causal=self.causal,
window_size=window_size,
return_softmax_lse=self.return_softmax_lse,
fa_version=self.fa_version,
num_splits=self.num_splits,
)
def request_key(self) -> tuple[object, ...]:
return (
self.q_shape,
self.k_shape,
self.v_shape,
self.q_dtype,
self.max_seqlen_q,
self.max_seqlen_k,
self.softmax_scale,
self.causal,
self.fa_version,
self.v_stride,
self.cu_seqlens_q_shape,
self.cu_seqlens_k_shape,
self.window_size,
self.return_softmax_lse,
self.num_splits,
)
def get_flash_attn_version(
requires_alibi: bool = False,
head_size: int | None = None,
@@ -11,8 +11,17 @@ import vllm.envs as envs
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.model_executor.warmup.cutedsl_warmup import (
CuTeDSLCompileUnit,
register_cutedsl_warmup_provider,
)
from vllm.model_executor.warmup.fa4_cutedsl_config import (
FA4MLAPrefillCompileContext,
iter_fa4_mla_prefill_compile_requests,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.fa_utils import (
compile_flash_attn_varlen_func_from_specs,
get_flash_attn_version,
is_flash_attn_varlen_func_available,
)
@@ -90,6 +99,46 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
# Track whether we're using vllm's FA or upstream (for ROCm)
self._is_vllm_fa = current_platform.is_cuda() or current_platform.is_xpu()
if self.vllm_flash_attn_version == 4:
register_cutedsl_warmup_provider(self)
def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]:
if self.vllm_flash_attn_version != 4:
return ()
if compile_flash_attn_varlen_func_from_specs is None:
raise RuntimeError(
"FA4 compile-only API is unavailable; CuTeDSL warmup does not "
"fall back to synthetic forward passes."
)
dtype = self.vllm_config.model_config.dtype
if dtype not in self.supported_dtypes:
dtype = torch.bfloat16
qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
ctx = FA4MLAPrefillCompileContext(
dtype=dtype,
num_heads=self.num_heads,
qk_head_dim=qk_head_dim,
v_head_dim=self.v_head_dim,
kv_nope_head_dim=self.qk_nope_head_dim + self.v_head_dim,
requires_v_padding=self.requires_v_padding,
scale=self.scale,
num_splits=1 if envs.VLLM_BATCH_INVARIANT else 0,
fa_version=self.vllm_flash_attn_version,
)
compile_requests = tuple(iter_fa4_mla_prefill_compile_requests(ctx))
if not compile_requests:
return ()
return tuple(
CuTeDSLCompileUnit(
name="fa4_mla_prefill",
key=request.key,
compile=request.compile,
)
for request in compile_requests
)
def supports_quant_output(self, quant_key: "QuantKey") -> bool:
device_capability = current_platform.get_device_capability()
+2
View File
@@ -23,6 +23,7 @@ if os.path.islink(_cute_dir) and "flash_attn" not in sys.modules:
from vllm.vllm_flash_attn.flash_attn_interface import ( # noqa: E402
FA2_AVAILABLE,
FA3_AVAILABLE,
compile_flash_attn_varlen_func_from_specs,
fa_version_unsupported_reason,
flash_attn_varlen_func,
get_scheduler_metadata,
@@ -36,6 +37,7 @@ if not (FA2_AVAILABLE or FA3_AVAILABLE):
)
__all__ = [
"compile_flash_attn_varlen_func_from_specs",
"fa_version_unsupported_reason",
"flash_attn_varlen_func",
"get_scheduler_metadata",
@@ -417,6 +417,66 @@ def flash_attn_varlen_func(
return (out, softmax_lse) if return_softmax_lse else out
def compile_flash_attn_varlen_func_from_specs(
*,
q_shape: tuple[int, ...],
k_shape: tuple[int, ...],
v_shape: tuple[int, ...],
q_dtype: torch.dtype,
v_stride: tuple[int, ...] | None = None,
cu_seqlens_q_shape: tuple[int, ...] | None = None,
cu_seqlens_k_shape: tuple[int, ...] | None = None,
max_seqlen_q: int | None = None,
max_seqlen_k: int | None = None,
dropout_p: float = 0.0,
softmax_scale=None,
causal=False,
window_size: list[int] | None = None,
deterministic=False,
return_softmax_lse=False,
num_splits: int = 0,
fa_version: int = DEFAULT_FA_VERSION,
) -> None:
if fa_version != 4:
raise ValueError(
f"Compile-only FlashAttention is only supported for FA4, got FA{fa_version}"
)
if dropout_p != 0.0:
raise NotImplementedError("FA4 compile-only wrapper does not support dropout")
del deterministic
from vllm.vllm_flash_attn.cute.interface import (
compile_flash_attn_varlen_func_from_specs as _fa4_compile_flash_attn_varlen_func_from_specs,
)
real_window_size: tuple[int, int]
if window_size is None:
real_window_size = (-1, -1)
else:
assert len(window_size) == 2
real_window_size = (window_size[0], window_size[1])
if softmax_scale is None:
softmax_scale = q_shape[-1] ** (-0.5)
return _fa4_compile_flash_attn_varlen_func_from_specs(
q_shape=q_shape,
k_shape=k_shape,
v_shape=v_shape,
q_dtype=q_dtype,
v_stride=v_stride,
cu_seqlens_q_shape=cu_seqlens_q_shape,
cu_seqlens_k_shape=cu_seqlens_k_shape,
max_seqlen_q=max_seqlen_q,
max_seqlen_k=max_seqlen_k,
softmax_scale=softmax_scale,
causal=causal,
window_size=real_window_size,
num_splits=num_splits,
return_lse=return_softmax_lse,
)
def sparse_attn_func(
q,
k,