[1/N][warmup][DSv4] Migrate generic MLA metadata and indexing kernels (#50175)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
parent
188716ace7
commit
e16b5e518d
@@ -604,6 +604,33 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
self.use_sparse = use_sparse
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
backend_name = self.attn_backend.get_name()
|
||||
if backend_name in (
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE_SM120",
|
||||
"DEEPSEEK_V32_INDEXER",
|
||||
):
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
_COMPRESSED_SLOT_MAPPING_KERNEL,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
_BUILD_PREFILL_CHUNK_METADATA_KERNEL,
|
||||
_PREPARE_UNIFORM_DECODE_KERNEL,
|
||||
)
|
||||
|
||||
_COMPRESSED_SLOT_MAPPING_KERNEL.register_warmup()
|
||||
_PREPARE_UNIFORM_DECODE_KERNEL.register_warmup()
|
||||
_BUILD_PREFILL_CHUNK_METADATA_KERNEL.register_warmup()
|
||||
|
||||
if backend_name != "DEEPSEEK_V32_INDEXER":
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
_COMPUTE_PREFILL_METADATA_KERNEL,
|
||||
)
|
||||
|
||||
_COMPUTE_PREFILL_METADATA_KERNEL.register_warmup()
|
||||
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.impl.dcp_world_size > 1:
|
||||
query_dtype = (
|
||||
@@ -652,6 +679,26 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
# [B, H=1, N, C] -> [B, N, C]
|
||||
self.kv_cache = kv_cache.squeeze(1)
|
||||
if (
|
||||
self._vllm_config.kernel_config.enable_jit_warmup
|
||||
and self.attn_backend.get_name()
|
||||
in (
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE_SM120",
|
||||
"DEEPSEEK_V32_INDEXER",
|
||||
)
|
||||
):
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
|
||||
)
|
||||
|
||||
row_width = self.kv_cache.shape[-1]
|
||||
assert self.kv_cache.stride(0) % row_width == 0
|
||||
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL.register_warmup(
|
||||
self._vllm_config,
|
||||
block_stride_rows=self.kv_cache.stride(0) // row_width,
|
||||
)
|
||||
|
||||
@property
|
||||
def chunked_prefill_workspace_size(self) -> int:
|
||||
|
||||
@@ -737,7 +737,11 @@ class JitWarmupRegistry:
|
||||
for kernel, registrations in self._registrations.items():
|
||||
compile_keys: dict[Any, None] = {}
|
||||
for args, kwargs in registrations:
|
||||
if not args and not kwargs:
|
||||
if (
|
||||
not args
|
||||
and not kwargs
|
||||
and inspect.signature(kernel.get_warmup_keys).parameters
|
||||
):
|
||||
args = (self.vllm_config,)
|
||||
for compile_key in kernel.get_warmup_keys(*args, **kwargs):
|
||||
compile_keys[compile_key] = None
|
||||
|
||||
@@ -2,15 +2,21 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import ast
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from functools import cached_property, wraps
|
||||
from typing import Any, ClassVar, Generic, TypeVar, cast
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
VllmJitKernel,
|
||||
get_ast_full_name,
|
||||
get_function_source_node,
|
||||
)
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
LaunchSpec = tuple[tuple[int, ...], dict[str, Any]]
|
||||
|
||||
|
||||
def triton_scalar_specialization_rep(value: int) -> int:
|
||||
"""Return an integer with the same default Triton JIT specialization.
|
||||
@@ -50,10 +56,16 @@ def triton_scalar_specialization_rep(value: int) -> int:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonWarmupTensor:
|
||||
# Compile-only tensor descriptor for Triton pointer specialization.
|
||||
"""Compile-only tensor metadata used by Triton warmup.
|
||||
|
||||
``strides=None`` represents compact row-major storage. Pass explicit strides
|
||||
whenever the runtime tensor can be padded, transposed, or otherwise strided.
|
||||
"""
|
||||
|
||||
dtype: Any
|
||||
aligned: bool = True
|
||||
shape: tuple[int, ...] = (1,)
|
||||
strides: tuple[int, ...] | None = None
|
||||
|
||||
def data_ptr(self) -> int:
|
||||
return 0 if self.aligned else 1
|
||||
@@ -61,13 +73,81 @@ class TritonWarmupTensor:
|
||||
def ptr_range(self) -> int:
|
||||
return 0
|
||||
|
||||
def stride(self) -> tuple[int, ...]:
|
||||
strides: list[int] = []
|
||||
stride = 1
|
||||
for size in reversed(self.shape):
|
||||
strides.append(stride)
|
||||
stride *= size
|
||||
return tuple(reversed(strides))
|
||||
def stride(self, dim: int | None = None) -> int | tuple[int, ...]:
|
||||
if self.strides is None:
|
||||
strides: list[int] = []
|
||||
stride = 1
|
||||
for size in reversed(self.shape):
|
||||
strides.append(stride)
|
||||
stride *= size
|
||||
result = tuple(reversed(strides))
|
||||
else:
|
||||
result = self.strides
|
||||
return result if dim is None else result[dim]
|
||||
|
||||
|
||||
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""Triton owner whose runtime launch specification is reused for warmup."""
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
_warming = False
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
|
||||
"""Return runtime-shaped inputs that reproduce one compile key."""
|
||||
raise NotImplementedError
|
||||
|
||||
def compile(self, compile_key: CompileKeyT) -> None:
|
||||
inputs = self.warmup_inputs(compile_key)
|
||||
self._warming = True
|
||||
try:
|
||||
cast(Callable[..., None], self)(**inputs)
|
||||
finally:
|
||||
self._warming = False
|
||||
|
||||
@cached_property
|
||||
def _kernel_param_names(self) -> frozenset[str]:
|
||||
return frozenset(self.kernel.arg_names)
|
||||
|
||||
def launch(
|
||||
self,
|
||||
grid: tuple[int, ...],
|
||||
inputs: Mapping[str, Any],
|
||||
/,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
for name, value in inputs.items():
|
||||
target = name if name in self._kernel_param_names else f"{name}_ptr"
|
||||
if target in self._kernel_param_names and target not in kwargs:
|
||||
kwargs[target] = value
|
||||
if self._warming:
|
||||
warmup = getattr(self.kernel, "warmup", None)
|
||||
assert warmup is not None
|
||||
return warmup(grid=(1,), **kwargs)
|
||||
return self.kernel[grid](**kwargs)
|
||||
|
||||
|
||||
def kernel_launcher(
|
||||
call_fn: Callable[..., LaunchSpec],
|
||||
) -> Callable[..., None]:
|
||||
"""Launch a Triton kernel from a declarative ``__call__`` specification."""
|
||||
signature = inspect.signature(call_fn)
|
||||
|
||||
@wraps(call_fn)
|
||||
def wrapper(
|
||||
self: VllmTritonJitKernel[Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
grid, launch_kwargs = call_fn(self, *args, **kwargs)
|
||||
bound = signature.bind(self, *args, **kwargs)
|
||||
bound.apply_defaults()
|
||||
inputs = {
|
||||
name: value for name, value in bound.arguments.items() if name != "self"
|
||||
}
|
||||
self.launch(grid, inputs, **launch_kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -35,9 +35,6 @@ from vllm.model_executor.warmup.kimi_k3_triton_warmup import (
|
||||
kimi_k3_triton_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
|
||||
from vllm.model_executor.warmup.sparse_mla_triton_warmup import (
|
||||
sparse_mla_triton_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
@@ -154,7 +151,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
fa4_cutedsl_warmup(worker)
|
||||
sparse_mla_triton_warmup(worker)
|
||||
|
||||
if current_platform.has_device_capability(90):
|
||||
_warmup_ll_bf16_router_gemm(worker.get_model())
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up sparse-MLA Triton metadata kernels."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
|
||||
{
|
||||
"FLASHMLA_SPARSE_DSV4",
|
||||
"FLASHINFER_MLA_SPARSE_DSV4",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4",
|
||||
"DEEPSEEK_SPARSE_SWA",
|
||||
}
|
||||
)
|
||||
_GENERIC_SPARSE_MLA_BACKENDS = frozenset(
|
||||
{
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE_SM120",
|
||||
}
|
||||
)
|
||||
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
|
||||
|
||||
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
|
||||
|
||||
|
||||
def _attention_backend_name(backend: object) -> str | None:
|
||||
get_name = getattr(backend, "get_name", None)
|
||||
if get_name is None:
|
||||
return None
|
||||
try:
|
||||
return get_name()
|
||||
except NotImplementedError:
|
||||
return None
|
||||
|
||||
|
||||
def _has_attention_backend(
|
||||
runner: "GPUModelRunner",
|
||||
backend_names: frozenset[str],
|
||||
) -> bool:
|
||||
for groups in getattr(runner, "attn_groups", []) or ():
|
||||
for group in groups:
|
||||
name = _attention_backend_name(getattr(group, "backend", None))
|
||||
if name in backend_names:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _compile_sparse_swa_prefill_metadata_kernel(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
_COMPUTE_PREFILL_METADATA_KERNEL,
|
||||
)
|
||||
|
||||
_COMPUTE_PREFILL_METADATA_KERNEL.warmup(vllm_config)
|
||||
|
||||
|
||||
def _compile_prefill_chunk_metadata_kernel(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
_BUILD_PREFILL_CHUNK_METADATA_KERNEL,
|
||||
)
|
||||
|
||||
_BUILD_PREFILL_CHUNK_METADATA_KERNEL.warmup(vllm_config)
|
||||
|
||||
|
||||
def _compile_combine_topk_swa_indices_kernel(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
_COMBINE_TOPK_SWA_INDICES_KERNEL,
|
||||
)
|
||||
|
||||
_COMBINE_TOPK_SWA_INDICES_KERNEL.warmup(vllm_config)
|
||||
|
||||
|
||||
def sparse_mla_triton_warmup(worker: "Worker") -> None:
|
||||
runner = worker.model_runner
|
||||
if runner.is_pooling_model:
|
||||
return
|
||||
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
max_num_prefills = min(worker.scheduler_config.max_num_seqs, max_tokens)
|
||||
if max_tokens <= 0 or max_num_prefills <= 0:
|
||||
return
|
||||
|
||||
vllm_config = runner.vllm_config
|
||||
try:
|
||||
if _has_attention_backend(runner, _DEEPSEEK_V4_SPARSE_MLA_BACKENDS):
|
||||
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
|
||||
_compile_prefill_chunk_metadata_kernel(vllm_config)
|
||||
_compile_combine_topk_swa_indices_kernel(vllm_config)
|
||||
elif _has_attention_backend(runner, _GENERIC_SPARSE_MLA_BACKENDS):
|
||||
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
|
||||
_compile_prefill_chunk_metadata_kernel(vllm_config)
|
||||
elif _has_attention_backend(runner, _INDEXER_PREFILL_CHUNK_METADATA_BACKENDS):
|
||||
_compile_prefill_chunk_metadata_kernel(vllm_config)
|
||||
|
||||
except Exception:
|
||||
logger.warning("Skipping sparse MLA Triton warmup.", exc_info=True)
|
||||
Reference in New Issue
Block a user