[Perf] Extend Qwen Triton warmup to avoid first-request latency spikes (#54797)
Signed-off-by: Juqi Li <2223621784@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -34,10 +34,12 @@ from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
from vllm.model_executor.warmup.kimi_k3_triton_warmup import (
|
||||
kimi_k3_triton_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.mamba_triton_warmup import mamba_triton_warmup
|
||||
from vllm.model_executor.warmup.qwen4_exp_qsa_warmup import (
|
||||
qwen4_exp_qsa_triton_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
|
||||
from vllm.model_executor.warmup.qwen_vl_triton_warmup import qwen_vl_triton_warmup
|
||||
from vllm.model_executor.warmup.replayssm_warmup import (
|
||||
replayssm_autotune_warmup,
|
||||
)
|
||||
@@ -149,6 +151,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
)
|
||||
|
||||
qwen_triton_warmup(worker.model_runner, worker.vllm_config.model_config)
|
||||
qwen_vl_triton_warmup(worker.model_runner)
|
||||
mamba_triton_warmup(worker.model_runner)
|
||||
|
||||
compilation_config = worker.vllm_config.compilation_config
|
||||
cudagraph_capture_sizes = list(compilation_config.cudagraph_capture_sizes or [])
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm Mamba-style Triton kernels shared across GDN / Mamba / KDA models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _has_mamba_style_cache(runner: "GPUModelRunner") -> bool:
|
||||
from vllm.v1.kv_cache_interface import MambaSpec, UniformTypeKVCacheSpecs
|
||||
|
||||
for group in runner.kv_cache_config.kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
spec = spec.first_spec
|
||||
if isinstance(spec, MambaSpec):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _warm_batch_memcpy_kernel(device: torch.device) -> None:
|
||||
"""Warm the Mamba prefix-cache state copy specialization."""
|
||||
from vllm.v1.worker.mamba_utils import batch_memcpy
|
||||
|
||||
src = torch.empty(1024, dtype=torch.uint8, device=device)
|
||||
dst = torch.empty_like(src)
|
||||
batch_memcpy(
|
||||
torch.tensor([src.data_ptr()], dtype=torch.uint64, device=device),
|
||||
torch.tensor([dst.data_ptr()], dtype=torch.uint64, device=device),
|
||||
# Keep this int32: Triton's compile key includes pointer element dtypes,
|
||||
# and the production prefix-cache path passes an int32 sizes tensor.
|
||||
torch.tensor([src.numel()], dtype=torch.int32, device=device),
|
||||
)
|
||||
logger.info("Warmed Mamba batch_memcpy_kernel.")
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def mamba_triton_warmup(runner: "GPUModelRunner") -> None:
|
||||
"""Warm prefix-cache memcpy for every Mamba-style KV cache group."""
|
||||
device = runner.device
|
||||
if device.type != "cuda":
|
||||
return
|
||||
if not _has_mamba_style_cache(runner):
|
||||
return
|
||||
_warm_batch_memcpy_kernel(device)
|
||||
@@ -283,33 +283,27 @@ def qwen_triton_warmup(
|
||||
runner: "GPUModelRunner",
|
||||
model_config: "ModelConfig",
|
||||
) -> None:
|
||||
"""Warm Qwen Triton kernels reported by the JIT monitor."""
|
||||
if runner.is_pooling_model:
|
||||
return
|
||||
|
||||
hf_text_config = getattr(model_config, "hf_text_config", None)
|
||||
hf_config = getattr(model_config, "hf_config", None)
|
||||
model_type = None
|
||||
for config in (hf_text_config, hf_config):
|
||||
model_type = getattr(config, "model_type", None)
|
||||
if model_type is not None:
|
||||
model_type = str(model_type)
|
||||
break
|
||||
"""Warm Qwen GDN Triton kernels reported by the JIT monitor."""
|
||||
model_type = getattr(model_config.hf_text_config, "model_type", "") or getattr(
|
||||
model_config.hf_config, "model_type", ""
|
||||
)
|
||||
if model_type not in _QWEN_MODEL_TYPES:
|
||||
return
|
||||
|
||||
device = getattr(runner, "device", torch.device("cuda"))
|
||||
logger.info("Warming up Qwen Triton kernels for model_type=%s.", model_type)
|
||||
device = runner.device
|
||||
logger.info("Warming up Qwen GDN Triton kernels for model_type=%s.", model_type)
|
||||
|
||||
compilation_config = getattr(runner, "compilation_config", None)
|
||||
static_forward_context = getattr(compilation_config, "static_forward_context", None)
|
||||
gdn_config = _qwen_gdn_warmup_config(static_forward_context)
|
||||
gdn_config = _qwen_gdn_warmup_config(
|
||||
runner.compilation_config.static_forward_context
|
||||
)
|
||||
if gdn_config is None:
|
||||
return
|
||||
|
||||
max_num_tokens = max(1, int(getattr(runner, "max_num_tokens", 1)))
|
||||
max_num_tokens = max(1, int(runner.max_num_tokens))
|
||||
_warm_gated_rms_norm_kernel(device, gdn_config, max_num_tokens, model_config.dtype)
|
||||
_warm_causal_conv1d_fwd_kernel(device, gdn_config)
|
||||
_warm_fused_post_conv_kernel(device, gdn_config)
|
||||
_warm_fused_sigmoid_gating_delta_rule_update_kernel(device, gdn_config)
|
||||
# Pooling only runs full prefills; the decode update kernel is unused.
|
||||
if not runner.is_pooling_model:
|
||||
_warm_fused_sigmoid_gating_delta_rule_update_kernel(device, gdn_config)
|
||||
_synchronize_device(device)
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm Qwen3-VL ViT kernels and M-RoPE for SupportsMRoPE models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Covers Triton's ==1 / %16==0 / other integer buckets.
|
||||
_MROPE_TOKEN_COUNTS = (1, 2, 16)
|
||||
|
||||
|
||||
def _warm_vision(model: torch.nn.Module, device: torch.device) -> None:
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3_VisionTransformer
|
||||
|
||||
for visual in model.modules():
|
||||
if not isinstance(visual, Qwen3_VisionTransformer):
|
||||
continue
|
||||
# CPU offloading keeps tower weights off the accelerator until the
|
||||
# module forward hook copies them back, so warming here would launch
|
||||
# Triton on host pointers.
|
||||
if visual.pos_embed.weight.device.type != device.type:
|
||||
continue
|
||||
attention = visual.blocks[0].attn
|
||||
num_heads = int(attention.num_attention_heads_per_partition)
|
||||
head_size = int(attention.hidden_size_per_attention_head)
|
||||
merge_size = int(visual.spatial_merge_size)
|
||||
divisible = round_up(16, merge_size)
|
||||
while divisible % 16:
|
||||
divisible += merge_size
|
||||
spatial_sizes = [divisible]
|
||||
if merge_size % 16:
|
||||
spatial_sizes.append(merge_size)
|
||||
if merge_size == 1:
|
||||
spatial_sizes.append(2)
|
||||
grids = [(1, h, w) for h in spatial_sizes for w in spatial_sizes]
|
||||
|
||||
for grid in grids:
|
||||
grid_thw = [list(grid)]
|
||||
visual.fast_pos_embed_interpolate(grid_thw)
|
||||
cos, sin = visual.rot_pos_emb(grid_thw)
|
||||
qk = torch.empty(
|
||||
(2, grid[0] * grid[1] * grid[2], num_heads, head_size),
|
||||
dtype=cos.dtype,
|
||||
device=cos.device,
|
||||
)
|
||||
attention.apply_rotary_emb(qk, cos, sin)
|
||||
|
||||
logger.info(
|
||||
"Warmed position embedding and vision rotary kernels on grids=%s.",
|
||||
grids,
|
||||
)
|
||||
|
||||
|
||||
def _warm_mrope(runner: "GPUModelRunner", model: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
from vllm.model_executor.models.interfaces import supports_mrope
|
||||
|
||||
is_mrope_model = supports_mrope(model)
|
||||
if not is_mrope_model:
|
||||
return
|
||||
|
||||
rope = next(
|
||||
(module for module in model.modules() if isinstance(module, MRotaryEmbedding)),
|
||||
None,
|
||||
)
|
||||
if rope is None:
|
||||
return
|
||||
|
||||
model_config = runner.model_config
|
||||
parallel_config = runner.parallel_config
|
||||
num_query_heads = int(model_config.get_num_attention_heads(parallel_config))
|
||||
num_kv_heads = int(model_config.get_num_kv_heads(parallel_config))
|
||||
for num_tokens in _MROPE_TOKEN_COUNTS:
|
||||
positions = torch.arange(
|
||||
num_tokens, dtype=torch.long, device=runner.device
|
||||
).expand(3, -1)
|
||||
query = torch.empty(
|
||||
(num_tokens, num_query_heads * rope.head_size),
|
||||
dtype=runner.dtype,
|
||||
device=runner.device,
|
||||
)
|
||||
key_tensor = torch.empty(
|
||||
(num_tokens, num_kv_heads * rope.head_size),
|
||||
dtype=runner.dtype,
|
||||
device=runner.device,
|
||||
)
|
||||
rope(positions, query, key_tensor)
|
||||
|
||||
logger.info("Warmed M-RoPE Triton kernels.")
|
||||
|
||||
|
||||
def _synchronize_device(device: torch.device) -> None:
|
||||
if device.type == "cuda":
|
||||
torch.accelerator.synchronize(device)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def qwen_vl_triton_warmup(runner: "GPUModelRunner") -> None:
|
||||
"""Warm Qwen3-VL ViT kernels and M-RoPE for ``SupportsMRoPE`` models."""
|
||||
model = runner.get_model()
|
||||
_warm_vision(model, runner.device)
|
||||
_warm_mrope(runner, model)
|
||||
_synchronize_device(runner.device)
|
||||
Reference in New Issue
Block a user