[Feat][1/N] CuTeDSL warmup infrastructure, FA4 MLA (#46182)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
11b26c5528
commit
248d1fbb71
@@ -39,7 +39,7 @@ else()
|
||||
FetchContent_Declare(
|
||||
vllm-flash-attn
|
||||
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
|
||||
GIT_TAG b3964b1d8b95d8e8447435668ab169a2700bab65
|
||||
GIT_TAG 2c839c33742309ec41e620bf837495ec9926c56e
|
||||
GIT_PROGRESS TRUE
|
||||
# Don't share the vllm-flash-attn build between build types
|
||||
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
|
||||
|
||||
@@ -172,6 +172,9 @@ class KernelConfig:
|
||||
enable_flashinfer_autotune: bool = None # type: ignore[assignment]
|
||||
"""If True, run FlashInfer autotuning during kernel warmup."""
|
||||
|
||||
enable_cutedsl_warmup: bool = True
|
||||
"""If True, run CuTeDSL compile warmup during kernel warmup."""
|
||||
|
||||
moe_backend: MoEBackend = "auto"
|
||||
"""Backend for MoE expert computation kernels. Available options:
|
||||
|
||||
@@ -237,6 +240,7 @@ class KernelConfig:
|
||||
Any future fields that don't affect compilation should be excluded.
|
||||
"""
|
||||
ignored_factors = {
|
||||
"enable_cutedsl_warmup",
|
||||
"enable_flashinfer_autotune",
|
||||
"ir_op_priority", # handled separately below
|
||||
}
|
||||
@@ -244,7 +248,11 @@ class KernelConfig:
|
||||
factors["ir_op_priority"] = self.ir_op_priority.compute_hash()
|
||||
return hash_factors(factors)
|
||||
|
||||
@field_validator("enable_flashinfer_autotune", mode="wrap")
|
||||
@field_validator(
|
||||
"enable_flashinfer_autotune",
|
||||
"enable_cutedsl_warmup",
|
||||
mode="wrap",
|
||||
)
|
||||
@classmethod
|
||||
def _skip_none_validation(cls, value: Any, handler: Callable) -> Any:
|
||||
"""Skip validation if the value is `None` when initialization is delayed."""
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Run registered CuTeDSL warmup compile units."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import Callable, Hashable, Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tracing import instrument
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
CuTeDSLCompileFn = Callable[[], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CuTeDSLCompileUnit:
|
||||
name: str
|
||||
key: Hashable
|
||||
compile: CuTeDSLCompileFn
|
||||
|
||||
|
||||
_CUTEDSL_WARMUP_PROVIDERS: weakref.WeakSet[object] = weakref.WeakSet()
|
||||
|
||||
|
||||
def register_cutedsl_warmup_provider(provider: object) -> None:
|
||||
"""Register an object that can expose CuTeDSL warmup compile units."""
|
||||
_CUTEDSL_WARMUP_PROVIDERS.add(provider)
|
||||
|
||||
|
||||
# Yield compile units from registered providers.
|
||||
def _iter_cutedsl_warmup_compile_units() -> Iterable[CuTeDSLCompileUnit]:
|
||||
for provider in tuple(_CUTEDSL_WARMUP_PROVIDERS):
|
||||
get_units = getattr(provider, "get_cutedsl_warmup_compile_units", None)
|
||||
if not callable(get_units):
|
||||
continue
|
||||
|
||||
compile_units = get_units()
|
||||
if compile_units is None:
|
||||
continue
|
||||
for unit in compile_units:
|
||||
if not isinstance(unit, CuTeDSLCompileUnit):
|
||||
raise TypeError(
|
||||
"get_cutedsl_warmup_compile_units must return "
|
||||
"CuTeDSLCompileUnit objects"
|
||||
)
|
||||
yield unit
|
||||
|
||||
|
||||
# Drop duplicate compile units across providers.
|
||||
def _collect_unique_compile_units(
|
||||
compile_units: Iterable[CuTeDSLCompileUnit],
|
||||
) -> list[CuTeDSLCompileUnit]:
|
||||
seen: set[Hashable] = set()
|
||||
unique_compile_units: list[CuTeDSLCompileUnit] = []
|
||||
|
||||
for unit in compile_units:
|
||||
if unit.key in seen:
|
||||
continue
|
||||
|
||||
seen.add(unit.key)
|
||||
unique_compile_units.append(unit)
|
||||
|
||||
return unique_compile_units
|
||||
|
||||
|
||||
# Execute compile units under inference mode.
|
||||
def _compile_cutedsl_warmup_units(
|
||||
compile_units: Iterable[CuTeDSLCompileUnit],
|
||||
) -> int:
|
||||
compiled = 0
|
||||
with torch.inference_mode():
|
||||
for unit in compile_units:
|
||||
unit.compile()
|
||||
compiled += 1
|
||||
torch.accelerator.synchronize()
|
||||
return compiled
|
||||
|
||||
|
||||
# Run all CuTeDSL warmup before serving.
|
||||
@instrument(span_name="CuTeDSL warmup")
|
||||
def cutedsl_warmup() -> None:
|
||||
"""Run CuTeDSL compile providers before serving."""
|
||||
if not current_platform.is_cuda():
|
||||
logger.info("Skipping CuTeDSL warmup on non-CUDA platform.")
|
||||
return
|
||||
|
||||
compile_units = _collect_unique_compile_units(_iter_cutedsl_warmup_compile_units())
|
||||
if not compile_units:
|
||||
logger.info("Skipping CuTeDSL warmup because no compile units were requested.")
|
||||
return
|
||||
|
||||
unit_names = list(dict.fromkeys(unit.name for unit in compile_units))
|
||||
logger.info(
|
||||
"Warming up CuTeDSL compile_units=%d names=%s.",
|
||||
len(compile_units),
|
||||
unit_names,
|
||||
)
|
||||
|
||||
start_time = time.perf_counter()
|
||||
compiled_count = _compile_cutedsl_warmup_units(compile_units)
|
||||
logger.info(
|
||||
"CuTeDSL warmup compiled %d units in %.2f s.",
|
||||
compiled_count,
|
||||
time.perf_counter() - start_time,
|
||||
)
|
||||
@@ -0,0 +1,204 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FA4 MLA prefill CuTeDSL compile warmup config."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Hashable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
FlashAttentionCuTeDSLCompileSpec,
|
||||
)
|
||||
|
||||
FA4ArchitectureFamily = Literal["sm90", "sm100f", "sm120"]
|
||||
|
||||
FA4_STANDARD_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
# Current vLLM MLA prefill expands K/V to num_heads before FA4, so this plan
|
||||
# covers qhead_per_kvhead=1.
|
||||
# Batch is not a current FA4 MLA-prefill key field. Use b1 for compile-only
|
||||
# specs because it is the conservative case for Split-KV shape heuristics.
|
||||
# TODO(roberto): FA4 also has direct-GQA and qv/top-k absorbed-MLA paths, but vLLM
|
||||
# does not use them in this backend yet; they need a separate
|
||||
# num_kv_heads/qv/top-k-aware warmup plan if wired in later.
|
||||
FA4_MLA_PREFILL_COMPILE_BATCH_SIZE = 1
|
||||
FA4_MLA_PREFILL_Q_TILE = 128
|
||||
FA4_MLA_PREFILL_K_TILE = 128
|
||||
FA4_MLA_PREFILL_LONG_K_BLOCKS = 32
|
||||
FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS = 64
|
||||
FA4_MLA_PREFILL_CAUSAL_OPTIONS = (False, True)
|
||||
FA4_MLA_PREFILL_LSE_OPTIONS = (False, True)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FA4MLAPrefillCompileContext:
|
||||
dtype: torch.dtype
|
||||
num_heads: int
|
||||
qk_head_dim: int
|
||||
v_head_dim: int
|
||||
kv_nope_head_dim: int
|
||||
requires_v_padding: bool
|
||||
scale: float
|
||||
num_splits: int
|
||||
fa_version: int
|
||||
|
||||
# Return the V head dim FA4 sees.
|
||||
@property
|
||||
def effective_v_head_dim(self) -> int:
|
||||
if self.requires_v_padding:
|
||||
return self.qk_head_dim
|
||||
return self.v_head_dim
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FA4MLAPrefillCompileRequest:
|
||||
"""One compile-only FA4 MLA prefill request."""
|
||||
|
||||
key: Hashable
|
||||
compile_spec: FlashAttentionCuTeDSLCompileSpec
|
||||
|
||||
# Compile this request.
|
||||
def compile(self) -> None:
|
||||
self.compile_spec.compile()
|
||||
|
||||
|
||||
# Yield deduped compile requests.
|
||||
def iter_fa4_mla_prefill_compile_requests(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
) -> Iterator[FA4MLAPrefillCompileRequest]:
|
||||
"""Yield compile requests for this fixed MLA backend.
|
||||
|
||||
FA4 dedupes duplicate atomic kernel selections in its own JIT cache.
|
||||
"""
|
||||
seen: set[Hashable] = set()
|
||||
for compile_spec in iter_fa4_mla_prefill_compile_specs(ctx):
|
||||
key = compile_spec.request_key()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
yield FA4MLAPrefillCompileRequest(
|
||||
key=key,
|
||||
compile_spec=compile_spec,
|
||||
)
|
||||
|
||||
|
||||
# Build compile specs for this setup.
|
||||
def iter_fa4_mla_prefill_compile_specs(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
) -> Iterator[FlashAttentionCuTeDSLCompileSpec]:
|
||||
"""Yield compile-only FA4 MLA prefill requests for this fixed setup."""
|
||||
|
||||
arch_family = _fa4_architecture_family_from_compute_capability(
|
||||
*torch.cuda.get_device_capability()
|
||||
)
|
||||
if not _supports_fa4_mla_prefill(ctx, arch_family):
|
||||
return
|
||||
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
FlashAttentionCuTeDSLCompileSpec,
|
||||
)
|
||||
|
||||
batch_size = FA4_MLA_PREFILL_COMPILE_BATCH_SIZE
|
||||
v_stride = None
|
||||
if not ctx.requires_v_padding:
|
||||
v_stride = (
|
||||
ctx.num_heads * ctx.kv_nope_head_dim,
|
||||
ctx.kv_nope_head_dim,
|
||||
1,
|
||||
)
|
||||
|
||||
for _, max_seqlen_q, max_seqlen_k in _shape_probes_for_context(ctx, arch_family):
|
||||
total_q_tokens = batch_size * max_seqlen_q
|
||||
total_kv_tokens = batch_size * max_seqlen_k
|
||||
for causal in FA4_MLA_PREFILL_CAUSAL_OPTIONS:
|
||||
for return_lse in FA4_MLA_PREFILL_LSE_OPTIONS:
|
||||
yield FlashAttentionCuTeDSLCompileSpec(
|
||||
q_shape=(total_q_tokens, ctx.num_heads, ctx.qk_head_dim),
|
||||
k_shape=(total_kv_tokens, ctx.num_heads, ctx.qk_head_dim),
|
||||
v_shape=(
|
||||
total_kv_tokens,
|
||||
ctx.num_heads,
|
||||
ctx.effective_v_head_dim,
|
||||
),
|
||||
v_stride=v_stride,
|
||||
q_dtype=ctx.dtype,
|
||||
cu_seqlens_q_shape=(batch_size + 1,),
|
||||
cu_seqlens_k_shape=(batch_size + 1,),
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
softmax_scale=ctx.scale,
|
||||
causal=causal,
|
||||
return_softmax_lse=return_lse,
|
||||
num_splits=ctx.num_splits,
|
||||
fa_version=ctx.fa_version,
|
||||
)
|
||||
|
||||
|
||||
# Pick one q/k point per current FA4 MLA-prefill shape regime.
|
||||
def _shape_probes_for_context(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
arch_family: FA4ArchitectureFamily,
|
||||
) -> tuple[tuple[str, int, int], ...]:
|
||||
q_stage1_q = 1
|
||||
q_stage2_q = FA4_MLA_PREFILL_Q_TILE + 1
|
||||
# FA4 never auto-splits when ceil(max_seqlen_k / tile_n) <= 4.
|
||||
no_split_k = 4 * FA4_MLA_PREFILL_K_TILE
|
||||
long_k = FA4_MLA_PREFILL_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
|
||||
# Diff-head-dim Blackwell Split-KV switches tile_n at 64 K blocks.
|
||||
very_long_k = FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
|
||||
|
||||
base_probes = (
|
||||
("q_stage1", q_stage1_q, FA4_MLA_PREFILL_K_TILE),
|
||||
("q_stage2", q_stage2_q, no_split_k),
|
||||
)
|
||||
# SM120 currently rejects Split-KV in FA4; num_splits=1 also has no split
|
||||
# shape regimes on any architecture.
|
||||
if ctx.num_splits == 1 or arch_family == "sm120":
|
||||
return base_probes
|
||||
|
||||
long_k_probes = (
|
||||
("q_stage1_long_k", q_stage1_q, long_k),
|
||||
("q_stage2_long_k", q_stage2_q, long_k),
|
||||
)
|
||||
|
||||
# SM90 does not have the SM100 q_stage or diff-head-dim tile_n=64 branch.
|
||||
# Same-dim SM100-family MLA also does not need the very-long-K probe.
|
||||
if arch_family == "sm90" or ctx.qk_head_dim == ctx.effective_v_head_dim:
|
||||
return (*base_probes, *long_k_probes)
|
||||
|
||||
very_long_k_probes = (
|
||||
("q_stage1_very_long_k", q_stage1_q, very_long_k),
|
||||
("q_stage2_very_long_k", q_stage2_q, very_long_k),
|
||||
)
|
||||
return (*base_probes, *long_k_probes, *very_long_k_probes)
|
||||
|
||||
|
||||
# Check whether this setup can use FA4 MLA prefill.
|
||||
def _supports_fa4_mla_prefill(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
arch_family: FA4ArchitectureFamily,
|
||||
) -> bool:
|
||||
return (
|
||||
ctx.dtype in FA4_STANDARD_DTYPES
|
||||
and ctx.num_heads > 0
|
||||
and (arch_family != "sm120" or ctx.num_splits == 1)
|
||||
)
|
||||
|
||||
|
||||
# Map CUDA capability to the FA4 arch family used by warmup checks.
|
||||
def _fa4_architecture_family_from_compute_capability(
|
||||
major: int,
|
||||
minor: int,
|
||||
) -> FA4ArchitectureFamily:
|
||||
if (major, minor) == (9, 0):
|
||||
return "sm90"
|
||||
if major == 10:
|
||||
return "sm100f"
|
||||
if (major, minor) == (12, 0):
|
||||
return "sm120"
|
||||
raise ValueError(f"FA4 warmup does not know CUDA capability {major}.{minor}")
|
||||
@@ -12,6 +12,7 @@ import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
|
||||
deepseek_v4_mhc_warmup,
|
||||
@@ -125,6 +126,9 @@ def kernel_warmup(worker: "Worker"):
|
||||
create_mixed_batch=True,
|
||||
)
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
|
||||
cutedsl_warmup()
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
@@ -18,6 +21,7 @@ _ROCM_FLASH_ATTN_AVAILABLE = False
|
||||
if current_platform.is_cuda():
|
||||
from vllm._custom_ops import reshape_and_cache_flash
|
||||
from vllm.vllm_flash_attn import ( # type: ignore[attr-defined]
|
||||
compile_flash_attn_varlen_func_from_specs,
|
||||
flash_attn_varlen_func,
|
||||
get_scheduler_metadata,
|
||||
)
|
||||
@@ -28,11 +32,14 @@ elif current_platform.is_xpu():
|
||||
|
||||
reshape_and_cache_flash = ops.reshape_and_cache_flash
|
||||
flash_attn_varlen_func = xpu_ops.flash_attn_varlen_func # type: ignore[assignment]
|
||||
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
|
||||
get_scheduler_metadata = xpu_ops.get_scheduler_metadata # type: ignore[assignment]
|
||||
elif current_platform.is_rocm():
|
||||
try:
|
||||
from flash_attn import flash_attn_varlen_func # type: ignore[no-redef]
|
||||
|
||||
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
|
||||
|
||||
# Mark that upstream flash-attn is available on ROCm
|
||||
_ROCM_FLASH_ATTN_AVAILABLE = True
|
||||
except ImportError:
|
||||
@@ -43,6 +50,8 @@ elif current_platform.is_rocm():
|
||||
"to be installed. Please install flash-attn first."
|
||||
)
|
||||
|
||||
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
|
||||
|
||||
# ROCm doesn't use scheduler metadata (FA3 feature), provide stub
|
||||
def get_scheduler_metadata(*args: Any, **kwargs: Any) -> None: # type: ignore[misc]
|
||||
return None
|
||||
@@ -53,6 +62,73 @@ elif current_platform.is_rocm():
|
||||
reshape_and_cache_flash = ops.reshape_and_cache_flash
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashAttentionCuTeDSLCompileSpec:
|
||||
"""High-level FA4 compile-only request used by vLLM warmup.
|
||||
|
||||
This is not the CuTeDSL cache key. FA4 owns the selector that maps these
|
||||
serving inputs to the actual compile-static fields: tile sizes, q_stage,
|
||||
Split-KV, scheduler choice, layout-presence booleans, dtype/head dims,
|
||||
arch, and related fields.
|
||||
"""
|
||||
|
||||
q_shape: tuple[int, ...]
|
||||
k_shape: tuple[int, ...]
|
||||
v_shape: tuple[int, ...]
|
||||
q_dtype: torch.dtype
|
||||
max_seqlen_q: int
|
||||
max_seqlen_k: int
|
||||
softmax_scale: float
|
||||
causal: bool
|
||||
fa_version: int
|
||||
v_stride: tuple[int, ...] | None = None
|
||||
cu_seqlens_q_shape: tuple[int, ...] | None = None
|
||||
cu_seqlens_k_shape: tuple[int, ...] | None = None
|
||||
window_size: tuple[int, int] | None = None
|
||||
return_softmax_lse: bool = False
|
||||
num_splits: int = 0
|
||||
|
||||
def compile(self) -> None:
|
||||
assert compile_flash_attn_varlen_func_from_specs is not None
|
||||
window_size = list(self.window_size) if self.window_size is not None else None
|
||||
compile_flash_attn_varlen_func_from_specs(
|
||||
q_shape=self.q_shape,
|
||||
k_shape=self.k_shape,
|
||||
v_shape=self.v_shape,
|
||||
q_dtype=self.q_dtype,
|
||||
v_stride=self.v_stride,
|
||||
cu_seqlens_q_shape=self.cu_seqlens_q_shape,
|
||||
cu_seqlens_k_shape=self.cu_seqlens_k_shape,
|
||||
max_seqlen_q=self.max_seqlen_q,
|
||||
max_seqlen_k=self.max_seqlen_k,
|
||||
softmax_scale=self.softmax_scale,
|
||||
causal=self.causal,
|
||||
window_size=window_size,
|
||||
return_softmax_lse=self.return_softmax_lse,
|
||||
fa_version=self.fa_version,
|
||||
num_splits=self.num_splits,
|
||||
)
|
||||
|
||||
def request_key(self) -> tuple[object, ...]:
|
||||
return (
|
||||
self.q_shape,
|
||||
self.k_shape,
|
||||
self.v_shape,
|
||||
self.q_dtype,
|
||||
self.max_seqlen_q,
|
||||
self.max_seqlen_k,
|
||||
self.softmax_scale,
|
||||
self.causal,
|
||||
self.fa_version,
|
||||
self.v_stride,
|
||||
self.cu_seqlens_q_shape,
|
||||
self.cu_seqlens_k_shape,
|
||||
self.window_size,
|
||||
self.return_softmax_lse,
|
||||
self.num_splits,
|
||||
)
|
||||
|
||||
|
||||
def get_flash_attn_version(
|
||||
requires_alibi: bool = False,
|
||||
head_size: int | None = None,
|
||||
|
||||
@@ -11,8 +11,17 @@ import vllm.envs as envs
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import (
|
||||
CuTeDSLCompileUnit,
|
||||
register_cutedsl_warmup_provider,
|
||||
)
|
||||
from vllm.model_executor.warmup.fa4_cutedsl_config import (
|
||||
FA4MLAPrefillCompileContext,
|
||||
iter_fa4_mla_prefill_compile_requests,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
compile_flash_attn_varlen_func_from_specs,
|
||||
get_flash_attn_version,
|
||||
is_flash_attn_varlen_func_available,
|
||||
)
|
||||
@@ -90,6 +99,46 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
|
||||
|
||||
# Track whether we're using vllm's FA or upstream (for ROCm)
|
||||
self._is_vllm_fa = current_platform.is_cuda() or current_platform.is_xpu()
|
||||
if self.vllm_flash_attn_version == 4:
|
||||
register_cutedsl_warmup_provider(self)
|
||||
|
||||
def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]:
|
||||
if self.vllm_flash_attn_version != 4:
|
||||
return ()
|
||||
if compile_flash_attn_varlen_func_from_specs is None:
|
||||
raise RuntimeError(
|
||||
"FA4 compile-only API is unavailable; CuTeDSL warmup does not "
|
||||
"fall back to synthetic forward passes."
|
||||
)
|
||||
|
||||
dtype = self.vllm_config.model_config.dtype
|
||||
if dtype not in self.supported_dtypes:
|
||||
dtype = torch.bfloat16
|
||||
|
||||
qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
ctx = FA4MLAPrefillCompileContext(
|
||||
dtype=dtype,
|
||||
num_heads=self.num_heads,
|
||||
qk_head_dim=qk_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
kv_nope_head_dim=self.qk_nope_head_dim + self.v_head_dim,
|
||||
requires_v_padding=self.requires_v_padding,
|
||||
scale=self.scale,
|
||||
num_splits=1 if envs.VLLM_BATCH_INVARIANT else 0,
|
||||
fa_version=self.vllm_flash_attn_version,
|
||||
)
|
||||
compile_requests = tuple(iter_fa4_mla_prefill_compile_requests(ctx))
|
||||
if not compile_requests:
|
||||
return ()
|
||||
|
||||
return tuple(
|
||||
CuTeDSLCompileUnit(
|
||||
name="fa4_mla_prefill",
|
||||
key=request.key,
|
||||
compile=request.compile,
|
||||
)
|
||||
for request in compile_requests
|
||||
)
|
||||
|
||||
def supports_quant_output(self, quant_key: "QuantKey") -> bool:
|
||||
device_capability = current_platform.get_device_capability()
|
||||
|
||||
@@ -23,6 +23,7 @@ if os.path.islink(_cute_dir) and "flash_attn" not in sys.modules:
|
||||
from vllm.vllm_flash_attn.flash_attn_interface import ( # noqa: E402
|
||||
FA2_AVAILABLE,
|
||||
FA3_AVAILABLE,
|
||||
compile_flash_attn_varlen_func_from_specs,
|
||||
fa_version_unsupported_reason,
|
||||
flash_attn_varlen_func,
|
||||
get_scheduler_metadata,
|
||||
@@ -36,6 +37,7 @@ if not (FA2_AVAILABLE or FA3_AVAILABLE):
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"compile_flash_attn_varlen_func_from_specs",
|
||||
"fa_version_unsupported_reason",
|
||||
"flash_attn_varlen_func",
|
||||
"get_scheduler_metadata",
|
||||
|
||||
@@ -417,6 +417,66 @@ def flash_attn_varlen_func(
|
||||
return (out, softmax_lse) if return_softmax_lse else out
|
||||
|
||||
|
||||
def compile_flash_attn_varlen_func_from_specs(
|
||||
*,
|
||||
q_shape: tuple[int, ...],
|
||||
k_shape: tuple[int, ...],
|
||||
v_shape: tuple[int, ...],
|
||||
q_dtype: torch.dtype,
|
||||
v_stride: tuple[int, ...] | None = None,
|
||||
cu_seqlens_q_shape: tuple[int, ...] | None = None,
|
||||
cu_seqlens_k_shape: tuple[int, ...] | None = None,
|
||||
max_seqlen_q: int | None = None,
|
||||
max_seqlen_k: int | None = None,
|
||||
dropout_p: float = 0.0,
|
||||
softmax_scale=None,
|
||||
causal=False,
|
||||
window_size: list[int] | None = None,
|
||||
deterministic=False,
|
||||
return_softmax_lse=False,
|
||||
num_splits: int = 0,
|
||||
fa_version: int = DEFAULT_FA_VERSION,
|
||||
) -> None:
|
||||
if fa_version != 4:
|
||||
raise ValueError(
|
||||
f"Compile-only FlashAttention is only supported for FA4, got FA{fa_version}"
|
||||
)
|
||||
if dropout_p != 0.0:
|
||||
raise NotImplementedError("FA4 compile-only wrapper does not support dropout")
|
||||
del deterministic
|
||||
|
||||
from vllm.vllm_flash_attn.cute.interface import (
|
||||
compile_flash_attn_varlen_func_from_specs as _fa4_compile_flash_attn_varlen_func_from_specs,
|
||||
)
|
||||
|
||||
real_window_size: tuple[int, int]
|
||||
if window_size is None:
|
||||
real_window_size = (-1, -1)
|
||||
else:
|
||||
assert len(window_size) == 2
|
||||
real_window_size = (window_size[0], window_size[1])
|
||||
|
||||
if softmax_scale is None:
|
||||
softmax_scale = q_shape[-1] ** (-0.5)
|
||||
|
||||
return _fa4_compile_flash_attn_varlen_func_from_specs(
|
||||
q_shape=q_shape,
|
||||
k_shape=k_shape,
|
||||
v_shape=v_shape,
|
||||
q_dtype=q_dtype,
|
||||
v_stride=v_stride,
|
||||
cu_seqlens_q_shape=cu_seqlens_q_shape,
|
||||
cu_seqlens_k_shape=cu_seqlens_k_shape,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
window_size=real_window_size,
|
||||
num_splits=num_splits,
|
||||
return_lse=return_softmax_lse,
|
||||
)
|
||||
|
||||
|
||||
def sparse_attn_func(
|
||||
q,
|
||||
k,
|
||||
|
||||
Reference in New Issue
Block a user