[3/N][warmup][DSv4] Migrate FA4 MLA and shared CuTeDSL kernels (#53565)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Roberto L. Castro
2026-09-07 09:36:35 -04:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1713b9866a
commit 42801b3a6b
8 changed files with 52 additions and 67 deletions
@@ -1,53 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up FA4 CuTeDSL kernels."""
from __future__ import annotations
from typing import TYPE_CHECKING
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
if TYPE_CHECKING:
from vllm.v1.worker.gpu_worker import Worker
def _warm_fa4_mla_prefill(worker: Worker) -> None:
runner = worker.model_runner
if runner.is_pooling_model:
return
vllm_config = runner.vllm_config
if not vllm_config.model_config.use_mla:
return
try:
backend_cls = get_mla_prefill_backend(vllm_config)
except ValueError:
# fall back to top-k MQA prefill path.
return
if backend_cls.get_name() != "FLASH_ATTN":
return
from vllm.v1.attention.backends.mla.prefill import flash_attn
flash_attn.FA4_MLA_PREFILL_KERNEL.warmup(vllm_config)
def _warm_inkling_fa4_rel_attention(worker: Worker) -> None:
from vllm.models.inkling.configs import InklingMMConfig, InklingModelConfig
from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
INKLING_FA4_REL_ATTENTION_KERNEL,
)
vllm_config = worker.vllm_config
hf_config = vllm_config.model_config.hf_config
if not isinstance(hf_config, (InklingMMConfig, InklingModelConfig)):
return
INKLING_FA4_REL_ATTENTION_KERNEL.warmup(vllm_config)
def fa4_cutedsl_warmup(worker: Worker) -> None:
_warm_fa4_mla_prefill(worker)
_warm_inkling_fa4_rel_attention(worker)
@@ -20,9 +20,6 @@ from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
deepseek_v4_mhc_warmup,
)
from vllm.model_executor.warmup.fa4_cutedsl_warmup import (
fa4_cutedsl_warmup,
)
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
write_flashinfer_autotune_cache,
@@ -169,7 +166,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
# Run next so input-prep kernels JIT against pristine runner state.
if worker.vllm_config.kernel_config.enable_jit_warmup:
kimi_k3_triton_warmup(worker)
fa4_cutedsl_warmup(worker)
spec_decode_rejection_warmup(worker)
qwen4_exp_qsa_triton_warmup(worker)