[1/N][warmup][DSv4] Migrate generic MLA metadata and indexing kernels (#50175)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-08-31 23:26:05 -04:00
committed by GitHub
co-authored by Codex
parent 188716ace7
commit e16b5e518d
17 changed files with 1261 additions and 617 deletions
@@ -604,6 +604,33 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.use_sparse = use_sparse
if vllm_config.kernel_config.enable_jit_warmup:
backend_name = self.attn_backend.get_name()
if backend_name in (
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
"FLASHINFER_MLA_SPARSE_SM120",
"DEEPSEEK_V32_INDEXER",
):
from vllm.v1.attention.backends.mla.compressor_utils import (
_COMPRESSED_SLOT_MAPPING_KERNEL,
)
from vllm.v1.attention.backends.mla.indexer import (
_BUILD_PREFILL_CHUNK_METADATA_KERNEL,
_PREPARE_UNIFORM_DECODE_KERNEL,
)
_COMPRESSED_SLOT_MAPPING_KERNEL.register_warmup()
_PREPARE_UNIFORM_DECODE_KERNEL.register_warmup()
_BUILD_PREFILL_CHUNK_METADATA_KERNEL.register_warmup()
if backend_name != "DEEPSEEK_V32_INDEXER":
from vllm.v1.attention.backends.mla.sparse_swa import (
_COMPUTE_PREFILL_METADATA_KERNEL,
)
_COMPUTE_PREFILL_METADATA_KERNEL.register_warmup()
self.dcp_manager: MLADCPManager | None = None
if self.impl.dcp_world_size > 1:
query_dtype = (
@@ -652,6 +679,26 @@ class MLAAttention(nn.Module, AttentionLayerBase):
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
# [B, H=1, N, C] -> [B, N, C]
self.kv_cache = kv_cache.squeeze(1)
if (
self._vllm_config.kernel_config.enable_jit_warmup
and self.attn_backend.get_name()
in (
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
"FLASHINFER_MLA_SPARSE_SM120",
"DEEPSEEK_V32_INDEXER",
)
):
from vllm.v1.attention.backends.mla.sparse_utils import (
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
)
row_width = self.kv_cache.shape[-1]
assert self.kv_cache.stride(0) % row_width == 0
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL.register_warmup(
self._vllm_config,
block_stride_rows=self.kv_cache.stride(0) // row_width,
)
@property
def chunked_prefill_workspace_size(self) -> int:
+5 -1
View File
@@ -737,7 +737,11 @@ class JitWarmupRegistry:
for kernel, registrations in self._registrations.items():
compile_keys: dict[Any, None] = {}
for args, kwargs in registrations:
if not args and not kwargs:
if (
not args
and not kwargs
and inspect.signature(kernel.get_warmup_keys).parameters
):
args = (self.vllm_config,)
for compile_key in kernel.get_warmup_keys(*args, **kwargs):
compile_keys[compile_key] = None
@@ -2,15 +2,21 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ast
import inspect
from collections.abc import Callable
from abc import abstractmethod
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from typing import Any
from functools import cached_property, wraps
from typing import Any, ClassVar, Generic, TypeVar, cast
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
get_ast_full_name,
get_function_source_node,
)
CompileKeyT = TypeVar("CompileKeyT")
LaunchSpec = tuple[tuple[int, ...], dict[str, Any]]
def triton_scalar_specialization_rep(value: int) -> int:
"""Return an integer with the same default Triton JIT specialization.
@@ -50,10 +56,16 @@ def triton_scalar_specialization_rep(value: int) -> int:
@dataclass(frozen=True)
class TritonWarmupTensor:
# Compile-only tensor descriptor for Triton pointer specialization.
"""Compile-only tensor metadata used by Triton warmup.
``strides=None`` represents compact row-major storage. Pass explicit strides
whenever the runtime tensor can be padded, transposed, or otherwise strided.
"""
dtype: Any
aligned: bool = True
shape: tuple[int, ...] = (1,)
strides: tuple[int, ...] | None = None
def data_ptr(self) -> int:
return 0 if self.aligned else 1
@@ -61,13 +73,81 @@ class TritonWarmupTensor:
def ptr_range(self) -> int:
return 0
def stride(self) -> tuple[int, ...]:
strides: list[int] = []
stride = 1
for size in reversed(self.shape):
strides.append(stride)
stride *= size
return tuple(reversed(strides))
def stride(self, dim: int | None = None) -> int | tuple[int, ...]:
if self.strides is None:
strides: list[int] = []
stride = 1
for size in reversed(self.shape):
strides.append(stride)
stride *= size
result = tuple(reversed(strides))
else:
result = self.strides
return result if dim is None else result[dim]
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""Triton owner whose runtime launch specification is reused for warmup."""
kernel: ClassVar[Any]
_warming = False
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
"""Return runtime-shaped inputs that reproduce one compile key."""
raise NotImplementedError
def compile(self, compile_key: CompileKeyT) -> None:
inputs = self.warmup_inputs(compile_key)
self._warming = True
try:
cast(Callable[..., None], self)(**inputs)
finally:
self._warming = False
@cached_property
def _kernel_param_names(self) -> frozenset[str]:
return frozenset(self.kernel.arg_names)
def launch(
self,
grid: tuple[int, ...],
inputs: Mapping[str, Any],
/,
**kwargs: Any,
) -> Any:
for name, value in inputs.items():
target = name if name in self._kernel_param_names else f"{name}_ptr"
if target in self._kernel_param_names and target not in kwargs:
kwargs[target] = value
if self._warming:
warmup = getattr(self.kernel, "warmup", None)
assert warmup is not None
return warmup(grid=(1,), **kwargs)
return self.kernel[grid](**kwargs)
def kernel_launcher(
call_fn: Callable[..., LaunchSpec],
) -> Callable[..., None]:
"""Launch a Triton kernel from a declarative ``__call__`` specification."""
signature = inspect.signature(call_fn)
@wraps(call_fn)
def wrapper(
self: VllmTritonJitKernel[Any],
*args: Any,
**kwargs: Any,
) -> None:
grid, launch_kwargs = call_fn(self, *args, **kwargs)
bound = signature.bind(self, *args, **kwargs)
bound.apply_defaults()
inputs = {
name: value for name, value in bound.arguments.items() if name != "self"
}
self.launch(grid, inputs, **launch_kwargs)
return wrapper
@dataclass(frozen=True)
@@ -35,9 +35,6 @@ from vllm.model_executor.warmup.kimi_k3_triton_warmup import (
kimi_k3_triton_warmup,
)
from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
from vllm.model_executor.warmup.sparse_mla_triton_warmup import (
sparse_mla_triton_warmup,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import has_flashinfer
@@ -154,7 +151,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
if worker.vllm_config.kernel_config.enable_jit_warmup:
kimi_k3_triton_warmup(worker)
fa4_cutedsl_warmup(worker)
sparse_mla_triton_warmup(worker)
if current_platform.has_device_capability(90):
_warmup_ll_bf16_router_gemm(worker.get_model())
@@ -1,111 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up sparse-MLA Triton metadata kernels."""
from typing import TYPE_CHECKING
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
_DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
{
"FLASHMLA_SPARSE_DSV4",
"FLASHINFER_MLA_SPARSE_DSV4",
"ROCM_FLASHMLA_SPARSE_DSV4",
"DEEPSEEK_SPARSE_SWA",
}
)
_GENERIC_SPARSE_MLA_BACKENDS = frozenset(
{
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
"FLASHINFER_MLA_SPARSE_SM120",
}
)
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
def _attention_backend_name(backend: object) -> str | None:
get_name = getattr(backend, "get_name", None)
if get_name is None:
return None
try:
return get_name()
except NotImplementedError:
return None
def _has_attention_backend(
runner: "GPUModelRunner",
backend_names: frozenset[str],
) -> bool:
for groups in getattr(runner, "attn_groups", []) or ():
for group in groups:
name = _attention_backend_name(getattr(group, "backend", None))
if name in backend_names:
return True
return False
def _compile_sparse_swa_prefill_metadata_kernel(
vllm_config: "VllmConfig",
) -> None:
from vllm.v1.attention.backends.mla.sparse_swa import (
_COMPUTE_PREFILL_METADATA_KERNEL,
)
_COMPUTE_PREFILL_METADATA_KERNEL.warmup(vllm_config)
def _compile_prefill_chunk_metadata_kernel(
vllm_config: "VllmConfig",
) -> None:
from vllm.v1.attention.backends.mla.indexer import (
_BUILD_PREFILL_CHUNK_METADATA_KERNEL,
)
_BUILD_PREFILL_CHUNK_METADATA_KERNEL.warmup(vllm_config)
def _compile_combine_topk_swa_indices_kernel(
vllm_config: "VllmConfig",
) -> None:
from vllm.models.deepseek_v4.common.ops.cache_utils import (
_COMBINE_TOPK_SWA_INDICES_KERNEL,
)
_COMBINE_TOPK_SWA_INDICES_KERNEL.warmup(vllm_config)
def sparse_mla_triton_warmup(worker: "Worker") -> None:
runner = worker.model_runner
if runner.is_pooling_model:
return
max_tokens = worker.scheduler_config.max_num_batched_tokens
max_num_prefills = min(worker.scheduler_config.max_num_seqs, max_tokens)
if max_tokens <= 0 or max_num_prefills <= 0:
return
vllm_config = runner.vllm_config
try:
if _has_attention_backend(runner, _DEEPSEEK_V4_SPARSE_MLA_BACKENDS):
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
_compile_prefill_chunk_metadata_kernel(vllm_config)
_compile_combine_topk_swa_indices_kernel(vllm_config)
elif _has_attention_backend(runner, _GENERIC_SPARSE_MLA_BACKENDS):
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
_compile_prefill_chunk_metadata_kernel(vllm_config)
elif _has_attention_backend(runner, _INDEXER_PREFILL_CHUNK_METADATA_BACKENDS):
_compile_prefill_chunk_metadata_kernel(vllm_config)
except Exception:
logger.warning("Skipping sparse MLA Triton warmup.", exc_info=True)