[Core] Warm up runner-owned Triton kernels before the first request (#49903)

This commit is contained in:
Nick Hill
2026-07-28 07:58:02 -07:00
committed by GitHub
parent 1e81853afc
commit 0d0504b54c
7 changed files with 183 additions and 189 deletions
+10 -6
View File
@@ -98,12 +98,16 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
minimax_m3_msa_warmup,
)
# Pooling models do not use the generation slot-mapping path.
if not worker.use_v2_model_runner and not worker.model_runner.is_pooling_model:
warm_v1_block_table_kernels(
getattr(worker.model_runner, "device", torch.device("cuda")),
worker.scheduler_config.max_num_batched_tokens,
)
if not worker.use_v2_model_runner:
# Pooling models do not use the generation slot-mapping path.
if not worker.model_runner.is_pooling_model:
warm_v1_block_table_kernels(worker.model_runner)
# The KV-block zeroing kernel is driven by the scheduler's
# `new_block_ids_to_zero`, so no dummy run ever reaches it.
zeroer = getattr(worker.model_runner, "_kv_block_zeroer", None)
if zeroer is not None:
zeroer.warmup(worker.model_runner.kv_cache_config.num_blocks)
qwen_triton_warmup(worker.model_runner, worker.vllm_config.model_config)
# DSv4 mHC TileLang kernels (hc_pre/hc_post/hc_head_op) run every decoder
@@ -24,24 +24,10 @@ _QWEN_MODEL_TYPES = frozenset(
}
)
_ZERO_KV_N_BLOCKS = (1, 2)
_SLOT_MAPPING_KV_BLOCK_SIZE = 16
_SLOT_MAPPING_CP_KV_CACHE_INTERLEAVE_SIZE = 1
_SLOT_MAPPING_BLOCK_TABLE_STRIDES = (1, 3)
# Covers L=1 constexpr, non-divisible runtime L, and divisible runtime L.
_FLA_POST_CONV_WARMUP_LENGTHS = (1, 2, 16)
@dataclass(frozen=True)
class _ZeroKvWarmupConfig:
seg_page_sizes: torch.Tensor
max_chunks: int
block_size: int
n_segs: int
@dataclass(frozen=True)
class _QwenGDNWarmupConfig:
h: int
@@ -148,96 +134,6 @@ def _qwen_gdn_warmup_config(
return None
def _get_kv_block_zeroer(runner: object) -> object | None:
zeroer = getattr(runner, "kv_block_zeroer", None)
if zeroer is None:
zeroer = getattr(runner, "_kv_block_zeroer", None)
return zeroer
def _zero_kv_warmup_config(runner: object) -> _ZeroKvWarmupConfig | None:
zeroer = _get_kv_block_zeroer(runner)
meta = getattr(zeroer, "_meta", None)
if meta is None:
return None
_, seg_page_sizes, max_chunks, block_size, n_segs = meta
return _ZeroKvWarmupConfig(
seg_page_sizes=seg_page_sizes,
max_chunks=int(max_chunks),
block_size=int(block_size),
n_segs=int(n_segs),
)
def _warm_zero_kv_blocks_with_runner_zeroer(runner: object) -> bool:
zeroer = _get_kv_block_zeroer(runner)
zero_block_ids = getattr(zeroer, "zero_block_ids", None)
if not callable(zero_block_ids):
return False
for n_blocks in _ZERO_KV_N_BLOCKS:
zero_block_ids(list(range(n_blocks)))
return True
def _warm_zero_kv_blocks_kernel(
device: torch.device, config: _ZeroKvWarmupConfig
) -> None:
from vllm.v1.worker.utils import _zero_kv_blocks_kernel
max_n_blocks = max(_ZERO_KV_N_BLOCKS)
max_page_size = int(config.seg_page_sizes.max().item())
scratch = torch.empty(
max_n_blocks * max_page_size,
dtype=torch.int32,
device=device,
)
seg_addrs = torch.tensor(
[scratch.data_ptr()] * config.n_segs,
dtype=torch.uint64,
device=device,
)
for n_blocks in _ZERO_KV_N_BLOCKS:
block_ids = torch.arange(n_blocks, dtype=torch.int64, device=device)
grid = (n_blocks * config.n_segs * config.max_chunks,)
_zero_kv_blocks_kernel[grid](
seg_addrs,
config.seg_page_sizes,
block_ids,
n_blocks,
N_SEGS=config.n_segs,
MAX_CHUNKS=config.max_chunks,
BLOCK_SIZE=config.block_size,
)
def _warm_compute_slot_mapping_kernel(device: torch.device) -> None:
from vllm.v1.worker.block_table import BlockTable
# num_tokens/max_num_tokens are do_not_specialize; keep the launch tiny.
num_tokens = 1
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
for block_table_stride in _SLOT_MAPPING_BLOCK_TABLE_STRIDES:
# Use BlockTable so the JIT key matches the production slot-mapping call.
block_table = BlockTable(
block_size=_SLOT_MAPPING_KV_BLOCK_SIZE,
max_num_reqs=1,
max_num_blocks_per_req=block_table_stride,
max_num_batched_tokens=num_tokens,
pin_memory=False,
device=device,
kernel_block_size=_SLOT_MAPPING_KV_BLOCK_SIZE,
cp_kv_cache_interleave_size=_SLOT_MAPPING_CP_KV_CACHE_INTERLEAVE_SIZE,
)
block_table.add_row(list(range(block_table_stride)), 0)
block_table.commit_block_table(num_reqs=1)
block_table.compute_slot_mapping(1, query_start_loc, positions)
def _warm_causal_conv1d_fwd_kernel(
device: torch.device, config: _QwenGDNWarmupConfig
) -> None:
@@ -373,16 +269,6 @@ def qwen_triton_warmup(
device = getattr(runner, "device", torch.device("cuda"))
logger.info("Warming up Qwen Triton kernels for model_type=%s.", model_type)
zero_config = _zero_kv_warmup_config(runner)
warmed_zeroer = _warm_zero_kv_blocks_with_runner_zeroer(runner)
if zero_config is not None:
_warm_zero_kv_blocks_kernel(device, zero_config)
elif not warmed_zeroer:
logger.info("Skipping Qwen zero-kv warmup: no KVBlockZeroer metadata.")
_warm_compute_slot_mapping_kernel(device)
_synchronize_device(device)
compilation_config = getattr(runner, "compilation_config", None)
static_forward_context = getattr(compilation_config, "static_forward_context", None)
gdn_config = _qwen_gdn_warmup_config(static_forward_context)
@@ -2,42 +2,28 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up v1 block-table Triton kernels."""
from typing import TYPE_CHECKING
import torch
if TYPE_CHECKING:
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
_SLOT_MAPPING_WARMUP_TOKENS = 8
_SLOT_MAPPING_WARMUP_BLOCK_SIZES = (3, 16)
_SLOT_MAPPING_WARMUP_CP_KV_CACHE_INTERLEAVE_SIZE = 1
def warm_v1_block_table_kernels(
device: torch.device,
max_tokens: int,
) -> None:
from vllm.v1.worker.block_table import BlockTable
def warm_v1_block_table_kernels(runner: "GPUModelRunner") -> None:
"""JIT-compile ``_compute_slot_mapping_kernel`` for the real block tables."""
num_tokens = max(0, min(_SLOT_MAPPING_WARMUP_TOKENS, max_tokens))
device = runner.device
block_table = runner.input_batch.block_table
num_tokens = min(
_SLOT_MAPPING_WARMUP_TOKENS,
runner.scheduler_config.max_num_batched_tokens,
)
if num_tokens <= 0:
return
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
for block_size in _SLOT_MAPPING_WARMUP_BLOCK_SIZES:
max_num_blocks_per_req = max(
1, (max(num_tokens, max_tokens) + block_size - 1) // block_size
)
max_num_blocks_per_req = ((max_num_blocks_per_req + 15) // 16) * 16
block_table = BlockTable(
block_size=block_size,
max_num_reqs=1,
max_num_blocks_per_req=max_num_blocks_per_req,
max_num_batched_tokens=max(num_tokens, max_tokens),
pin_memory=False,
device=device,
kernel_block_size=block_size,
cp_kv_cache_interleave_size=(
_SLOT_MAPPING_WARMUP_CP_KV_CACHE_INTERLEAVE_SIZE
),
)
block_table.add_row(list(range(max_num_blocks_per_req)), 0)
block_table.commit_block_table(1)
block_table.compute_slot_mapping(1, query_start_loc, positions)
block_table.compute_slot_mapping(1, query_start_loc, positions)