diff --git a/cmake/external_projects/vllm_flash_attn.cmake b/cmake/external_projects/vllm_flash_attn.cmake index c8b1d68918..728f5e247f 100644 --- a/cmake/external_projects/vllm_flash_attn.cmake +++ b/cmake/external_projects/vllm_flash_attn.cmake @@ -39,7 +39,7 @@ else() FetchContent_Declare( vllm-flash-attn GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git - GIT_TAG b3964b1d8b95d8e8447435668ab169a2700bab65 + GIT_TAG 2c839c33742309ec41e620bf837495ec9926c56e GIT_PROGRESS TRUE # Don't share the vllm-flash-attn build between build types BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 770daad1ce..1d5b421715 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -172,6 +172,9 @@ class KernelConfig: enable_flashinfer_autotune: bool = None # type: ignore[assignment] """If True, run FlashInfer autotuning during kernel warmup.""" + enable_cutedsl_warmup: bool = True + """If True, run CuTeDSL compile warmup during kernel warmup.""" + moe_backend: MoEBackend = "auto" """Backend for MoE expert computation kernels. Available options: @@ -237,6 +240,7 @@ class KernelConfig: Any future fields that don't affect compilation should be excluded. """ ignored_factors = { + "enable_cutedsl_warmup", "enable_flashinfer_autotune", "ir_op_priority", # handled separately below } @@ -244,7 +248,11 @@ class KernelConfig: factors["ir_op_priority"] = self.ir_op_priority.compute_hash() return hash_factors(factors) - @field_validator("enable_flashinfer_autotune", mode="wrap") + @field_validator( + "enable_flashinfer_autotune", + "enable_cutedsl_warmup", + mode="wrap", + ) @classmethod def _skip_none_validation(cls, value: Any, handler: Callable) -> Any: """Skip validation if the value is `None` when initialization is delayed.""" diff --git a/vllm/model_executor/warmup/cutedsl_warmup.py b/vllm/model_executor/warmup/cutedsl_warmup.py new file mode 100644 index 0000000000..5978e91a6e --- /dev/null +++ b/vllm/model_executor/warmup/cutedsl_warmup.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Run registered CuTeDSL warmup compile units.""" + +from __future__ import annotations + +import time +import weakref +from collections.abc import Callable, Hashable, Iterable +from dataclasses import dataclass + +import torch + +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.tracing import instrument + +logger = init_logger(__name__) + +CuTeDSLCompileFn = Callable[[], None] + + +@dataclass(frozen=True) +class CuTeDSLCompileUnit: + name: str + key: Hashable + compile: CuTeDSLCompileFn + + +_CUTEDSL_WARMUP_PROVIDERS: weakref.WeakSet[object] = weakref.WeakSet() + + +def register_cutedsl_warmup_provider(provider: object) -> None: + """Register an object that can expose CuTeDSL warmup compile units.""" + _CUTEDSL_WARMUP_PROVIDERS.add(provider) + + +# Yield compile units from registered providers. +def _iter_cutedsl_warmup_compile_units() -> Iterable[CuTeDSLCompileUnit]: + for provider in tuple(_CUTEDSL_WARMUP_PROVIDERS): + get_units = getattr(provider, "get_cutedsl_warmup_compile_units", None) + if not callable(get_units): + continue + + compile_units = get_units() + if compile_units is None: + continue + for unit in compile_units: + if not isinstance(unit, CuTeDSLCompileUnit): + raise TypeError( + "get_cutedsl_warmup_compile_units must return " + "CuTeDSLCompileUnit objects" + ) + yield unit + + +# Drop duplicate compile units across providers. +def _collect_unique_compile_units( + compile_units: Iterable[CuTeDSLCompileUnit], +) -> list[CuTeDSLCompileUnit]: + seen: set[Hashable] = set() + unique_compile_units: list[CuTeDSLCompileUnit] = [] + + for unit in compile_units: + if unit.key in seen: + continue + + seen.add(unit.key) + unique_compile_units.append(unit) + + return unique_compile_units + + +# Execute compile units under inference mode. +def _compile_cutedsl_warmup_units( + compile_units: Iterable[CuTeDSLCompileUnit], +) -> int: + compiled = 0 + with torch.inference_mode(): + for unit in compile_units: + unit.compile() + compiled += 1 + torch.accelerator.synchronize() + return compiled + + +# Run all CuTeDSL warmup before serving. +@instrument(span_name="CuTeDSL warmup") +def cutedsl_warmup() -> None: + """Run CuTeDSL compile providers before serving.""" + if not current_platform.is_cuda(): + logger.info("Skipping CuTeDSL warmup on non-CUDA platform.") + return + + compile_units = _collect_unique_compile_units(_iter_cutedsl_warmup_compile_units()) + if not compile_units: + logger.info("Skipping CuTeDSL warmup because no compile units were requested.") + return + + unit_names = list(dict.fromkeys(unit.name for unit in compile_units)) + logger.info( + "Warming up CuTeDSL compile_units=%d names=%s.", + len(compile_units), + unit_names, + ) + + start_time = time.perf_counter() + compiled_count = _compile_cutedsl_warmup_units(compile_units) + logger.info( + "CuTeDSL warmup compiled %d units in %.2f s.", + compiled_count, + time.perf_counter() - start_time, + ) diff --git a/vllm/model_executor/warmup/fa4_cutedsl_config.py b/vllm/model_executor/warmup/fa4_cutedsl_config.py new file mode 100644 index 0000000000..916ffd8a67 --- /dev/null +++ b/vllm/model_executor/warmup/fa4_cutedsl_config.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""FA4 MLA prefill CuTeDSL compile warmup config.""" + +from __future__ import annotations + +from collections.abc import Hashable, Iterator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +import torch + +if TYPE_CHECKING: + from vllm.v1.attention.backends.fa_utils import ( + FlashAttentionCuTeDSLCompileSpec, + ) + +FA4ArchitectureFamily = Literal["sm90", "sm100f", "sm120"] + +FA4_STANDARD_DTYPES = (torch.bfloat16, torch.float16) + +# Current vLLM MLA prefill expands K/V to num_heads before FA4, so this plan +# covers qhead_per_kvhead=1. +# Batch is not a current FA4 MLA-prefill key field. Use b1 for compile-only +# specs because it is the conservative case for Split-KV shape heuristics. +# TODO(roberto): FA4 also has direct-GQA and qv/top-k absorbed-MLA paths, but vLLM +# does not use them in this backend yet; they need a separate +# num_kv_heads/qv/top-k-aware warmup plan if wired in later. +FA4_MLA_PREFILL_COMPILE_BATCH_SIZE = 1 +FA4_MLA_PREFILL_Q_TILE = 128 +FA4_MLA_PREFILL_K_TILE = 128 +FA4_MLA_PREFILL_LONG_K_BLOCKS = 32 +FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS = 64 +FA4_MLA_PREFILL_CAUSAL_OPTIONS = (False, True) +FA4_MLA_PREFILL_LSE_OPTIONS = (False, True) + + +@dataclass(frozen=True) +class FA4MLAPrefillCompileContext: + dtype: torch.dtype + num_heads: int + qk_head_dim: int + v_head_dim: int + kv_nope_head_dim: int + requires_v_padding: bool + scale: float + num_splits: int + fa_version: int + + # Return the V head dim FA4 sees. + @property + def effective_v_head_dim(self) -> int: + if self.requires_v_padding: + return self.qk_head_dim + return self.v_head_dim + + +@dataclass(frozen=True) +class FA4MLAPrefillCompileRequest: + """One compile-only FA4 MLA prefill request.""" + + key: Hashable + compile_spec: FlashAttentionCuTeDSLCompileSpec + + # Compile this request. + def compile(self) -> None: + self.compile_spec.compile() + + +# Yield deduped compile requests. +def iter_fa4_mla_prefill_compile_requests( + ctx: FA4MLAPrefillCompileContext, +) -> Iterator[FA4MLAPrefillCompileRequest]: + """Yield compile requests for this fixed MLA backend. + + FA4 dedupes duplicate atomic kernel selections in its own JIT cache. + """ + seen: set[Hashable] = set() + for compile_spec in iter_fa4_mla_prefill_compile_specs(ctx): + key = compile_spec.request_key() + if key in seen: + continue + seen.add(key) + yield FA4MLAPrefillCompileRequest( + key=key, + compile_spec=compile_spec, + ) + + +# Build compile specs for this setup. +def iter_fa4_mla_prefill_compile_specs( + ctx: FA4MLAPrefillCompileContext, +) -> Iterator[FlashAttentionCuTeDSLCompileSpec]: + """Yield compile-only FA4 MLA prefill requests for this fixed setup.""" + + arch_family = _fa4_architecture_family_from_compute_capability( + *torch.cuda.get_device_capability() + ) + if not _supports_fa4_mla_prefill(ctx, arch_family): + return + + from vllm.v1.attention.backends.fa_utils import ( + FlashAttentionCuTeDSLCompileSpec, + ) + + batch_size = FA4_MLA_PREFILL_COMPILE_BATCH_SIZE + v_stride = None + if not ctx.requires_v_padding: + v_stride = ( + ctx.num_heads * ctx.kv_nope_head_dim, + ctx.kv_nope_head_dim, + 1, + ) + + for _, max_seqlen_q, max_seqlen_k in _shape_probes_for_context(ctx, arch_family): + total_q_tokens = batch_size * max_seqlen_q + total_kv_tokens = batch_size * max_seqlen_k + for causal in FA4_MLA_PREFILL_CAUSAL_OPTIONS: + for return_lse in FA4_MLA_PREFILL_LSE_OPTIONS: + yield FlashAttentionCuTeDSLCompileSpec( + q_shape=(total_q_tokens, ctx.num_heads, ctx.qk_head_dim), + k_shape=(total_kv_tokens, ctx.num_heads, ctx.qk_head_dim), + v_shape=( + total_kv_tokens, + ctx.num_heads, + ctx.effective_v_head_dim, + ), + v_stride=v_stride, + q_dtype=ctx.dtype, + cu_seqlens_q_shape=(batch_size + 1,), + cu_seqlens_k_shape=(batch_size + 1,), + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + softmax_scale=ctx.scale, + causal=causal, + return_softmax_lse=return_lse, + num_splits=ctx.num_splits, + fa_version=ctx.fa_version, + ) + + +# Pick one q/k point per current FA4 MLA-prefill shape regime. +def _shape_probes_for_context( + ctx: FA4MLAPrefillCompileContext, + arch_family: FA4ArchitectureFamily, +) -> tuple[tuple[str, int, int], ...]: + q_stage1_q = 1 + q_stage2_q = FA4_MLA_PREFILL_Q_TILE + 1 + # FA4 never auto-splits when ceil(max_seqlen_k / tile_n) <= 4. + no_split_k = 4 * FA4_MLA_PREFILL_K_TILE + long_k = FA4_MLA_PREFILL_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE + # Diff-head-dim Blackwell Split-KV switches tile_n at 64 K blocks. + very_long_k = FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE + + base_probes = ( + ("q_stage1", q_stage1_q, FA4_MLA_PREFILL_K_TILE), + ("q_stage2", q_stage2_q, no_split_k), + ) + # SM120 currently rejects Split-KV in FA4; num_splits=1 also has no split + # shape regimes on any architecture. + if ctx.num_splits == 1 or arch_family == "sm120": + return base_probes + + long_k_probes = ( + ("q_stage1_long_k", q_stage1_q, long_k), + ("q_stage2_long_k", q_stage2_q, long_k), + ) + + # SM90 does not have the SM100 q_stage or diff-head-dim tile_n=64 branch. + # Same-dim SM100-family MLA also does not need the very-long-K probe. + if arch_family == "sm90" or ctx.qk_head_dim == ctx.effective_v_head_dim: + return (*base_probes, *long_k_probes) + + very_long_k_probes = ( + ("q_stage1_very_long_k", q_stage1_q, very_long_k), + ("q_stage2_very_long_k", q_stage2_q, very_long_k), + ) + return (*base_probes, *long_k_probes, *very_long_k_probes) + + +# Check whether this setup can use FA4 MLA prefill. +def _supports_fa4_mla_prefill( + ctx: FA4MLAPrefillCompileContext, + arch_family: FA4ArchitectureFamily, +) -> bool: + return ( + ctx.dtype in FA4_STANDARD_DTYPES + and ctx.num_heads > 0 + and (arch_family != "sm120" or ctx.num_splits == 1) + ) + + +# Map CUDA capability to the FA4 arch family used by warmup checks. +def _fa4_architecture_family_from_compute_capability( + major: int, + minor: int, +) -> FA4ArchitectureFamily: + if (major, minor) == (9, 0): + return "sm90" + if major == 10: + return "sm100f" + if (major, minor) == (12, 0): + return "sm120" + raise ValueError(f"FA4 warmup does not know CUDA capability {major}.{minor}") diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index f1d7788a98..e31a14db66 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -12,6 +12,7 @@ import torch import vllm.envs as envs from vllm.logger import init_logger +from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import ( deepseek_v4_mhc_warmup, @@ -125,6 +126,9 @@ def kernel_warmup(worker: "Worker"): create_mixed_batch=True, ) + if worker.vllm_config.kernel_config.enable_cutedsl_warmup: + cutedsl_warmup() + def flashinfer_autotune(runner: "GPUModelRunner") -> None: """ diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py index 474523780f..6c0debab99 100644 --- a/vllm/v1/attention/backends/fa_utils.py +++ b/vllm/v1/attention/backends/fa_utils.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass from typing import Any +import torch + import vllm.envs as envs from vllm.logger import init_logger from vllm.platforms import current_platform @@ -18,6 +21,7 @@ _ROCM_FLASH_ATTN_AVAILABLE = False if current_platform.is_cuda(): from vllm._custom_ops import reshape_and_cache_flash from vllm.vllm_flash_attn import ( # type: ignore[attr-defined] + compile_flash_attn_varlen_func_from_specs, flash_attn_varlen_func, get_scheduler_metadata, ) @@ -28,11 +32,14 @@ elif current_platform.is_xpu(): reshape_and_cache_flash = ops.reshape_and_cache_flash flash_attn_varlen_func = xpu_ops.flash_attn_varlen_func # type: ignore[assignment] + compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment] get_scheduler_metadata = xpu_ops.get_scheduler_metadata # type: ignore[assignment] elif current_platform.is_rocm(): try: from flash_attn import flash_attn_varlen_func # type: ignore[no-redef] + compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment] + # Mark that upstream flash-attn is available on ROCm _ROCM_FLASH_ATTN_AVAILABLE = True except ImportError: @@ -43,6 +50,8 @@ elif current_platform.is_rocm(): "to be installed. Please install flash-attn first." ) + compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment] + # ROCm doesn't use scheduler metadata (FA3 feature), provide stub def get_scheduler_metadata(*args: Any, **kwargs: Any) -> None: # type: ignore[misc] return None @@ -53,6 +62,73 @@ elif current_platform.is_rocm(): reshape_and_cache_flash = ops.reshape_and_cache_flash +@dataclass(frozen=True) +class FlashAttentionCuTeDSLCompileSpec: + """High-level FA4 compile-only request used by vLLM warmup. + + This is not the CuTeDSL cache key. FA4 owns the selector that maps these + serving inputs to the actual compile-static fields: tile sizes, q_stage, + Split-KV, scheduler choice, layout-presence booleans, dtype/head dims, + arch, and related fields. + """ + + q_shape: tuple[int, ...] + k_shape: tuple[int, ...] + v_shape: tuple[int, ...] + q_dtype: torch.dtype + max_seqlen_q: int + max_seqlen_k: int + softmax_scale: float + causal: bool + fa_version: int + v_stride: tuple[int, ...] | None = None + cu_seqlens_q_shape: tuple[int, ...] | None = None + cu_seqlens_k_shape: tuple[int, ...] | None = None + window_size: tuple[int, int] | None = None + return_softmax_lse: bool = False + num_splits: int = 0 + + def compile(self) -> None: + assert compile_flash_attn_varlen_func_from_specs is not None + window_size = list(self.window_size) if self.window_size is not None else None + compile_flash_attn_varlen_func_from_specs( + q_shape=self.q_shape, + k_shape=self.k_shape, + v_shape=self.v_shape, + q_dtype=self.q_dtype, + v_stride=self.v_stride, + cu_seqlens_q_shape=self.cu_seqlens_q_shape, + cu_seqlens_k_shape=self.cu_seqlens_k_shape, + max_seqlen_q=self.max_seqlen_q, + max_seqlen_k=self.max_seqlen_k, + softmax_scale=self.softmax_scale, + causal=self.causal, + window_size=window_size, + return_softmax_lse=self.return_softmax_lse, + fa_version=self.fa_version, + num_splits=self.num_splits, + ) + + def request_key(self) -> tuple[object, ...]: + return ( + self.q_shape, + self.k_shape, + self.v_shape, + self.q_dtype, + self.max_seqlen_q, + self.max_seqlen_k, + self.softmax_scale, + self.causal, + self.fa_version, + self.v_stride, + self.cu_seqlens_q_shape, + self.cu_seqlens_k_shape, + self.window_size, + self.return_softmax_lse, + self.num_splits, + ) + + def get_flash_attn_version( requires_alibi: bool = False, head_size: int | None = None, diff --git a/vllm/v1/attention/backends/mla/prefill/flash_attn.py b/vllm/v1/attention/backends/mla/prefill/flash_attn.py index 24763378e6..470197876e 100644 --- a/vllm/v1/attention/backends/mla/prefill/flash_attn.py +++ b/vllm/v1/attention/backends/mla/prefill/flash_attn.py @@ -11,8 +11,17 @@ import vllm.envs as envs from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, ) +from vllm.model_executor.warmup.cutedsl_warmup import ( + CuTeDSLCompileUnit, + register_cutedsl_warmup_provider, +) +from vllm.model_executor.warmup.fa4_cutedsl_config import ( + FA4MLAPrefillCompileContext, + iter_fa4_mla_prefill_compile_requests, +) from vllm.platforms import current_platform from vllm.v1.attention.backends.fa_utils import ( + compile_flash_attn_varlen_func_from_specs, get_flash_attn_version, is_flash_attn_varlen_func_available, ) @@ -90,6 +99,46 @@ class FlashAttnPrefillBackend(MLAPrefillBackend): # Track whether we're using vllm's FA or upstream (for ROCm) self._is_vllm_fa = current_platform.is_cuda() or current_platform.is_xpu() + if self.vllm_flash_attn_version == 4: + register_cutedsl_warmup_provider(self) + + def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]: + if self.vllm_flash_attn_version != 4: + return () + if compile_flash_attn_varlen_func_from_specs is None: + raise RuntimeError( + "FA4 compile-only API is unavailable; CuTeDSL warmup does not " + "fall back to synthetic forward passes." + ) + + dtype = self.vllm_config.model_config.dtype + if dtype not in self.supported_dtypes: + dtype = torch.bfloat16 + + qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim + ctx = FA4MLAPrefillCompileContext( + dtype=dtype, + num_heads=self.num_heads, + qk_head_dim=qk_head_dim, + v_head_dim=self.v_head_dim, + kv_nope_head_dim=self.qk_nope_head_dim + self.v_head_dim, + requires_v_padding=self.requires_v_padding, + scale=self.scale, + num_splits=1 if envs.VLLM_BATCH_INVARIANT else 0, + fa_version=self.vllm_flash_attn_version, + ) + compile_requests = tuple(iter_fa4_mla_prefill_compile_requests(ctx)) + if not compile_requests: + return () + + return tuple( + CuTeDSLCompileUnit( + name="fa4_mla_prefill", + key=request.key, + compile=request.compile, + ) + for request in compile_requests + ) def supports_quant_output(self, quant_key: "QuantKey") -> bool: device_capability = current_platform.get_device_capability() diff --git a/vllm/vllm_flash_attn/__init__.py b/vllm/vllm_flash_attn/__init__.py index 7dea1f659b..6ea5b87361 100644 --- a/vllm/vllm_flash_attn/__init__.py +++ b/vllm/vllm_flash_attn/__init__.py @@ -23,6 +23,7 @@ if os.path.islink(_cute_dir) and "flash_attn" not in sys.modules: from vllm.vllm_flash_attn.flash_attn_interface import ( # noqa: E402 FA2_AVAILABLE, FA3_AVAILABLE, + compile_flash_attn_varlen_func_from_specs, fa_version_unsupported_reason, flash_attn_varlen_func, get_scheduler_metadata, @@ -36,6 +37,7 @@ if not (FA2_AVAILABLE or FA3_AVAILABLE): ) __all__ = [ + "compile_flash_attn_varlen_func_from_specs", "fa_version_unsupported_reason", "flash_attn_varlen_func", "get_scheduler_metadata", diff --git a/vllm/vllm_flash_attn/flash_attn_interface.py b/vllm/vllm_flash_attn/flash_attn_interface.py index f015003367..bef811f6e6 100644 --- a/vllm/vllm_flash_attn/flash_attn_interface.py +++ b/vllm/vllm_flash_attn/flash_attn_interface.py @@ -417,6 +417,66 @@ def flash_attn_varlen_func( return (out, softmax_lse) if return_softmax_lse else out +def compile_flash_attn_varlen_func_from_specs( + *, + q_shape: tuple[int, ...], + k_shape: tuple[int, ...], + v_shape: tuple[int, ...], + q_dtype: torch.dtype, + v_stride: tuple[int, ...] | None = None, + cu_seqlens_q_shape: tuple[int, ...] | None = None, + cu_seqlens_k_shape: tuple[int, ...] | None = None, + max_seqlen_q: int | None = None, + max_seqlen_k: int | None = None, + dropout_p: float = 0.0, + softmax_scale=None, + causal=False, + window_size: list[int] | None = None, + deterministic=False, + return_softmax_lse=False, + num_splits: int = 0, + fa_version: int = DEFAULT_FA_VERSION, +) -> None: + if fa_version != 4: + raise ValueError( + f"Compile-only FlashAttention is only supported for FA4, got FA{fa_version}" + ) + if dropout_p != 0.0: + raise NotImplementedError("FA4 compile-only wrapper does not support dropout") + del deterministic + + from vllm.vllm_flash_attn.cute.interface import ( + compile_flash_attn_varlen_func_from_specs as _fa4_compile_flash_attn_varlen_func_from_specs, + ) + + real_window_size: tuple[int, int] + if window_size is None: + real_window_size = (-1, -1) + else: + assert len(window_size) == 2 + real_window_size = (window_size[0], window_size[1]) + + if softmax_scale is None: + softmax_scale = q_shape[-1] ** (-0.5) + + return _fa4_compile_flash_attn_varlen_func_from_specs( + q_shape=q_shape, + k_shape=k_shape, + v_shape=v_shape, + q_dtype=q_dtype, + v_stride=v_stride, + cu_seqlens_q_shape=cu_seqlens_q_shape, + cu_seqlens_k_shape=cu_seqlens_k_shape, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + softmax_scale=softmax_scale, + causal=causal, + window_size=real_window_size, + num_splits=num_splits, + return_lse=return_softmax_lse, + ) + + def sparse_attn_func( q, k,