From 030c9523bdb6a6292545768c863fd747c195b06b Mon Sep 17 00:00:00 2001 From: "Roberto L. Castro" <38211239+LopezCastroRoberto@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:40:34 +0200 Subject: [PATCH] [Perf][1/N] Expand Triton kernel warmup coverage, DSv4 (#46634) Signed-off-by: LopezCastroRoberto Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Lucas Wilkinson --- vllm/model_executor/warmup/kernel_warmup.py | 13 + .../warmup/sparse_mla_triton_warmup.py | 330 ++++++++++++++++++ .../warmup/v1_block_table_warmup.py | 43 +++ 3 files changed, 386 insertions(+) create mode 100644 vllm/model_executor/warmup/sparse_mla_triton_warmup.py create mode 100644 vllm/model_executor/warmup/v1_block_table_warmup.py diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index d9b71b2a0c..f1d7788a98 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -25,6 +25,12 @@ from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import ( flashinfer_sparse_mla_decode_autotune_warmup, ) from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup +from vllm.model_executor.warmup.sparse_mla_triton_warmup import ( + sparse_mla_triton_warmup_if_needed, +) +from vllm.model_executor.warmup.v1_block_table_warmup import ( + warm_v1_block_table_kernels, +) from vllm.platforms import current_platform from vllm.utils.deep_gemm import is_deep_gemm_supported from vllm.utils.flashinfer import has_flashinfer @@ -41,6 +47,12 @@ def kernel_warmup(worker: "Worker"): minimax_m3_msa_warmup, ) + # Pooling models do not use the generation slot-mapping path. + if not worker.use_v2_model_runner and not worker.model_runner.is_pooling_model: + warm_v1_block_table_kernels( + getattr(worker.model_runner, "device", torch.device("cuda")), + worker.scheduler_config.max_num_batched_tokens, + ) qwen_triton_warmup(worker.model_runner, worker.vllm_config.model_config) # DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder @@ -55,6 +67,7 @@ def kernel_warmup(worker: "Worker"): ) # Run next so input-prep kernels JIT against pristine runner state. + sparse_mla_triton_warmup_if_needed(worker) flashinfer_sparse_mla_decode_autotune_warmup(worker) deepseek_v4_sparse_mla_attention_warmup(worker) diff --git a/vllm/model_executor/warmup/sparse_mla_triton_warmup.py b/vllm/model_executor/warmup/sparse_mla_triton_warmup.py new file mode 100644 index 0000000000..e932849fac --- /dev/null +++ b/vllm/model_executor/warmup/sparse_mla_triton_warmup.py @@ -0,0 +1,330 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Warm up sparse-MLA Triton metadata kernels.""" + +from typing import TYPE_CHECKING + +import torch + +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.v1.worker.gpu_model_runner import GPUModelRunner + from vllm.v1.worker.gpu_worker import Worker + +logger = init_logger(__name__) + +_DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset( + { + "FLASHMLA_SPARSE_DSV4", + "FLASHINFER_MLA_SPARSE_DSV4", + "ROCM_FLASHMLA_SPARSE_DSV4", + "DEEPSEEK_SPARSE_SWA", + } +) +_GENERIC_SPARSE_MLA_BACKENDS = frozenset( + { + "FLASHMLA_SPARSE", + "FLASHINFER_MLA_SPARSE", + "FLASHINFER_MLA_SPARSE_SM120", + } +) + +_SPARSE_PREFILL_METADATA_NUM_PREFILLS = (1, 2, 4, 8) +_SPARSE_PREFILL_METADATA_NUM_DECODES = (0, 1, 2) +_DSV4_PREFILL_CHUNK_METADATA_COMPRESS_RATIOS = (4, 128) +_PREFILL_CHUNK_METADATA_SEQ_LEN_MULTIPLIERS = (2, 3) +_PREFILL_CHUNK_METADATA_QUERY_SLICE_OFFSETS = ( + # query_slice_start offset, query_slice_stop offset + (0, 0), + (0, -1), + (1, 0), + (1, -1), +) +_COMBINE_TOPK_SWA_INPUT_VARIANTS = ( + # offset_topk, offset_query_and_seq, offset_gather + (False, False, False), + (False, True, False), + (True, True, True), +) +_DSV4_COMBINE_TOPK_SWA_WARMUP_CASES = ( + # compress_ratio, topk, topk_width, N + (1, 0, 512, 512), + (4, 512, 512, 512 * 4), + # DSv4-Pro C4A traffic uses top-k 1024 with N=1024. + (4, 1024, 1024, 1024), + (128, 8192, 8192, 8192 * 128), + # Real C128A traffic also specializes N=1 in one call path. + (128, 8192, 8192, 1), +) + + +def _clamp_warmup_tokens(num_tokens: int, max_tokens: int) -> int: + return max(0, min(num_tokens, max_tokens)) + + +def _next_power_of_2(x: int) -> int: + return 1 << (x - 1).bit_length() + + +def _hf_config_int(runner: "GPUModelRunner", name: str, default: int) -> int: + model_config = getattr(runner.vllm_config, "model_config", None) + hf_config = getattr(model_config, "hf_config", None) + return int(getattr(hf_config, name, default) or default) + + +def _attention_backend_name(backend: object) -> str | None: + get_name = getattr(backend, "get_name", None) + if get_name is None: + return None + try: + return get_name() + except NotImplementedError: + return None + + +def _has_attention_backend( + runner: "GPUModelRunner", + backend_names: frozenset[str], +) -> bool: + for groups in getattr(runner, "attn_groups", []) or (): + for group in groups: + name = _attention_backend_name(getattr(group, "backend", None)) + if name in backend_names: + return True + return False + + +def _warm_sparse_swa_prefill_metadata_kernel( + device: torch.device, + window_size: int, + prefill_tokens: int, +) -> None: + from vllm.v1.attention.backends.mla.sparse_swa import ( + _compute_prefill_metadata_kernel, + ) + + for num_prefills in _SPARSE_PREFILL_METADATA_NUM_PREFILLS: + for num_decodes in _SPARSE_PREFILL_METADATA_NUM_DECODES: + query_lens = [1] * num_decodes + query_lens += [prefill_tokens] * num_prefills + query_start_locs = [0] + for query_len in query_lens: + query_start_locs.append(query_start_locs[-1] + query_len) + query_start_loc = torch.tensor( + query_start_locs, + dtype=torch.int32, + device=device, + ) + seq_lens = torch.tensor( + [1] * num_decodes + [window_size + q for q in query_lens[num_decodes:]], + dtype=torch.int32, + device=device, + ) + prefill_gather_lens = torch.empty( + num_prefills, dtype=torch.int32, device=device + ) + _compute_prefill_metadata_kernel[(1,)]( + prefill_gather_lens, + seq_lens, + query_start_loc, + num_prefills, + num_decodes, + window_size, + BLOCK_SIZE=_next_power_of_2(num_prefills), + ) + + +def _warm_prefill_chunk_metadata_kernel( + device: torch.device, + compress_ratio: int, + query_len: int, +) -> None: + from vllm.v1.attention.backends.mla.indexer import build_prefill_chunk_metadata + + num_reqs = 2 + query_start_loc_cpu = torch.arange( + 0, (num_reqs + 1) * query_len, query_len, dtype=torch.int32 + ) + query_start_loc = query_start_loc_cpu.to(device=device) + + uncompressed_seq_lens_cpu = torch.tensor( + [ + compress_ratio * multiplier + query_len + for multiplier in _PREFILL_CHUNK_METADATA_SEQ_LEN_MULTIPLIERS + ], + dtype=torch.int32, + ) + compressed_seq_lens_cpu = uncompressed_seq_lens_cpu // compress_ratio + uncompressed_seq_lens = uncompressed_seq_lens_cpu.to(device=device) + compressed_seq_lens = compressed_seq_lens_cpu.to(device=device) + block_table = torch.zeros( + (num_reqs, int(compressed_seq_lens_cpu.max().item())), + dtype=torch.int32, + device=device, + ) + + offset_uncompressed_seq_lens = torch.empty( + num_reqs + 1, dtype=torch.int32, device=device + )[1:] + offset_uncompressed_seq_lens.copy_(uncompressed_seq_lens) + query_slices = tuple( + slice(start, num_reqs * query_len + stop) + for start, stop in _PREFILL_CHUNK_METADATA_QUERY_SLICE_OFFSETS + ) + for warmup_uncompressed_seq_lens in ( + uncompressed_seq_lens, + offset_uncompressed_seq_lens, + ): + for query_slice in query_slices: + build_prefill_chunk_metadata( + 0, + num_reqs, + query_start_loc, + query_start_loc_cpu, + warmup_uncompressed_seq_lens, + compressed_seq_lens, + compressed_seq_lens_cpu, + block_table, + compress_ratio, + query_slice=query_slice, + ) + + +def _warm_combine_topk_swa_indices_kernel( + device: torch.device, + num_tokens: int, + window_size: int, + compress_ratio: int, + topk: int, + topk_width: int, + n: int, +) -> None: + from vllm.models.deepseek_v4.common.ops.cache_utils import combine_topk_swa_indices + + if num_tokens <= 0: + return + + def _make_topk_indices(*, offset: bool) -> torch.Tensor: + if offset: + topk_storage = torch.full( + (num_tokens * topk_width + 1,), + -1, + dtype=torch.int32, + device=device, + ) + topk_indices = topk_storage[1:].reshape(num_tokens, topk_width) + else: + topk_indices = torch.full( + (num_tokens, topk_width), -1, dtype=torch.int32, device=device + ) + if topk > 0: + topk_indices.copy_( + torch.arange(num_tokens * topk_width, dtype=torch.int32, device=device) + .reshape(num_tokens, topk_width) + .remainder(topk_width) + ) + return topk_indices + + query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device) + seq_lens = torch.tensor( + [window_size + num_tokens], dtype=torch.int32, device=device + ) + gather_lens = torch.tensor( + [min(window_size + num_tokens, window_size + num_tokens - 1)], + dtype=torch.int32, + device=device, + ) + offset_query_start_loc = torch.empty(3, dtype=torch.int32, device=device)[1:] + offset_query_start_loc.copy_(query_start_loc) + offset_seq_lens = torch.empty(2, dtype=torch.int32, device=device)[1:] + offset_seq_lens.copy_(seq_lens) + offset_gather_lens = torch.empty(2, dtype=torch.int32, device=device)[1:] + offset_gather_lens.copy_(gather_lens) + + for ( + offset_topk, + offset_query_and_seq, + offset_gather, + ) in _COMBINE_TOPK_SWA_INPUT_VARIANTS: + warmup_topk_indices = _make_topk_indices(offset=offset_topk) + warmup_query_start_loc = ( + offset_query_start_loc if offset_query_and_seq else query_start_loc + ) + warmup_seq_lens = offset_seq_lens if offset_query_and_seq else seq_lens + warmup_gather_lens = offset_gather_lens if offset_gather else gather_lens + n_values = (n,) if n == 1 else (n, n + 1) + for m in (window_size + num_tokens, topk_width): + for n_value in n_values: + combine_topk_swa_indices( + warmup_topk_indices, + warmup_query_start_loc, + warmup_seq_lens, + warmup_gather_lens, + window_size, + compress_ratio, + topk, + M=m, + N=n_value, + ) + + +@torch.inference_mode() +def sparse_mla_triton_warmup( + runner: "GPUModelRunner", + num_tokens: int, + *, + compress_ratios: tuple[int, ...], + combine_topk_swa_cases: tuple[tuple[int, int, int, int], ...] = (), +) -> None: + device = getattr(runner, "device", torch.device("cuda")) + window_size = _hf_config_int(runner, "sliding_window", 128) + + _warm_sparse_swa_prefill_metadata_kernel(device, window_size, num_tokens) + for compress_ratio in compress_ratios: + _warm_prefill_chunk_metadata_kernel(device, compress_ratio, num_tokens) + for compress_ratio, topk, topk_width, n in combine_topk_swa_cases: + _warm_combine_topk_swa_indices_kernel( + device, + num_tokens, + window_size, + compress_ratio, + topk, + topk_width, + n, + ) + + +def deepseek_v4_sparse_triton_warmup( + runner: "GPUModelRunner", + num_tokens: int, +) -> None: + sparse_mla_triton_warmup( + runner, + num_tokens, + compress_ratios=_DSV4_PREFILL_CHUNK_METADATA_COMPRESS_RATIOS, + combine_topk_swa_cases=_DSV4_COMBINE_TOPK_SWA_WARMUP_CASES, + ) + + +def sparse_mla_triton_warmup_if_needed(worker: "Worker") -> None: + runner = worker.model_runner + if runner.is_pooling_model: + return + + max_tokens = worker.scheduler_config.max_num_batched_tokens + num_tokens = _clamp_warmup_tokens(8, max_tokens) + if num_tokens <= 0: + return + + try: + if _has_attention_backend(runner, _DEEPSEEK_V4_SPARSE_MLA_BACKENDS): + deepseek_v4_sparse_triton_warmup(runner, num_tokens) + elif _has_attention_backend(runner, _GENERIC_SPARSE_MLA_BACKENDS): + sparse_mla_triton_warmup( + runner, + num_tokens, + compress_ratios=(1,), + ) + except Exception: + logger.warning("Skipping sparse MLA Triton warmup.", exc_info=True) diff --git a/vllm/model_executor/warmup/v1_block_table_warmup.py b/vllm/model_executor/warmup/v1_block_table_warmup.py new file mode 100644 index 0000000000..8d2328432e --- /dev/null +++ b/vllm/model_executor/warmup/v1_block_table_warmup.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Warm up v1 block-table Triton kernels.""" + +import torch + +_SLOT_MAPPING_WARMUP_TOKENS = 8 +_SLOT_MAPPING_WARMUP_BLOCK_SIZES = (3, 16) +_SLOT_MAPPING_WARMUP_CP_KV_CACHE_INTERLEAVE_SIZE = 1 + + +def warm_v1_block_table_kernels( + device: torch.device, + max_tokens: int, +) -> None: + from vllm.v1.worker.block_table import BlockTable + + num_tokens = max(0, min(_SLOT_MAPPING_WARMUP_TOKENS, max_tokens)) + if num_tokens <= 0: + return + + query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device) + positions = torch.arange(num_tokens, dtype=torch.int64, device=device) + for block_size in _SLOT_MAPPING_WARMUP_BLOCK_SIZES: + max_num_blocks_per_req = max( + 1, (max(num_tokens, max_tokens) + block_size - 1) // block_size + ) + max_num_blocks_per_req = ((max_num_blocks_per_req + 15) // 16) * 16 + block_table = BlockTable( + block_size=block_size, + max_num_reqs=1, + max_num_blocks_per_req=max_num_blocks_per_req, + max_num_batched_tokens=max(num_tokens, max_tokens), + pin_memory=False, + device=device, + kernel_block_size=block_size, + cp_kv_cache_interleave_size=( + _SLOT_MAPPING_WARMUP_CP_KV_CACHE_INTERLEAVE_SIZE + ), + ) + block_table.add_row(list(range(max_num_blocks_per_req)), 0) + block_table.commit_block_table(1) + block_table.compute_slot_mapping(1, query_start_loc, positions)