[Perf] Extend Qwen Triton warmup to avoid first-request latency spikes (#54797)
Signed-off-by: Juqi Li <2223621784@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.mamba_triton_warmup import _warm_batch_memcpy_kernel
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA is required")
|
||||
def test_mamba_batch_memcpy_kernel_compiles_on_gpu() -> None:
|
||||
_warm_batch_memcpy_kernel(torch.device("cuda"))
|
||||
torch.accelerator.synchronize("cuda")
|
||||
@@ -0,0 +1,55 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.qwen_triton_warmup import (
|
||||
_FLA_POST_CONV_WARMUP_LENGTHS,
|
||||
_QwenGDNWarmupConfig,
|
||||
_warm_causal_conv1d_fwd_kernel,
|
||||
_warm_fused_post_conv_kernel,
|
||||
_warm_gated_rms_norm_kernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _cuda_gdn_config() -> _QwenGDNWarmupConfig:
|
||||
h, hv, k, v = 2, 2, 16, 16
|
||||
conv_kernel_size = 4
|
||||
conv_dim = 2 * h * k + hv * v
|
||||
device = torch.device("cuda")
|
||||
conv_state = torch.empty(
|
||||
(8, conv_dim, conv_kernel_size - 1),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
return _QwenGDNWarmupConfig(
|
||||
h=h,
|
||||
hv=hv,
|
||||
k=k,
|
||||
v=v,
|
||||
conv_kernel_size=conv_kernel_size,
|
||||
conv_state=conv_state,
|
||||
conv_dtype=conv_state.dtype,
|
||||
norm_weight_dtype=torch.bfloat16,
|
||||
norm_before_gate=True,
|
||||
norm_activation="silu",
|
||||
a_log=torch.zeros(hv, dtype=torch.float32, device=device),
|
||||
dt_bias=torch.zeros(hv, dtype=torch.float32, device=device),
|
||||
state_stride_token=hv * v * k,
|
||||
state_dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA is required")
|
||||
def test_qwen_gdn_prefill_warmup_kernels_compile_on_gpu() -> None:
|
||||
config = _cuda_gdn_config()
|
||||
device = torch.device("cuda")
|
||||
_warm_gated_rms_norm_kernel(
|
||||
device, config, max_num_tokens=16, x_dtype=config.conv_dtype
|
||||
)
|
||||
_warm_causal_conv1d_fwd_kernel(device, config)
|
||||
_warm_fused_post_conv_kernel(device, config)
|
||||
assert _FLA_POST_CONV_WARMUP_LENGTHS == (1, 2, 16)
|
||||
torch.accelerator.synchronize(device)
|
||||
@@ -0,0 +1,117 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.qwen_vl_triton_warmup import (
|
||||
_warm_mrope,
|
||||
_warm_vision,
|
||||
)
|
||||
|
||||
|
||||
def test_vision_warmup_calls_only_position_and_rotary_paths() -> None:
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3_VisionTransformer
|
||||
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
class FakeAttention:
|
||||
num_attention_heads_per_partition = 4
|
||||
hidden_size_per_attention_head = 8
|
||||
|
||||
def apply_rotary_emb(self, qk, cos, sin):
|
||||
calls.append(("rotary", qk.shape))
|
||||
|
||||
class FakeVisual(Qwen3_VisionTransformer):
|
||||
spatial_merge_size = 2
|
||||
|
||||
def __init__(self) -> None:
|
||||
torch.nn.Module.__init__(self)
|
||||
self.blocks = [SimpleNamespace(attn=FakeAttention())]
|
||||
self.pos_embed = torch.nn.Embedding(4, 4, device="cpu")
|
||||
|
||||
def fast_pos_embed_interpolate(self, grid_thw):
|
||||
calls.append(("position", grid_thw[0]))
|
||||
|
||||
def rot_pos_emb(self, grid_thw):
|
||||
_, h, w = grid_thw[0]
|
||||
shape = (h * w, 4)
|
||||
return torch.empty(shape), torch.empty(shape)
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
raise AssertionError("vision warmup must not run the full tower")
|
||||
|
||||
model = torch.nn.Module()
|
||||
model.visual = FakeVisual()
|
||||
_warm_vision(model, torch.device("cpu"))
|
||||
|
||||
grids = [(1, 16, 16), (1, 16, 2), (1, 2, 16), (1, 2, 2)]
|
||||
assert [value for name, value in calls if name == "position"] == [
|
||||
list(grid) for grid in grids
|
||||
]
|
||||
assert [value for name, value in calls if name == "rotary"] == [
|
||||
torch.Size((2, h * w, 4, 8)) for _, h, w in grids
|
||||
]
|
||||
|
||||
# CPU offloading leaves tower weights off the accelerator, so the Triton
|
||||
# paths must not be launched on host pointers.
|
||||
calls.clear()
|
||||
_warm_vision(model, torch.device("cuda"))
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_mrope_warmup_launches_triton_shapes() -> None:
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
|
||||
launched: list[tuple[torch.Size, torch.Size]] = []
|
||||
|
||||
class FakeRope(MRotaryEmbedding):
|
||||
def __init__(self) -> None:
|
||||
torch.nn.Module.__init__(self)
|
||||
self.head_size = 8
|
||||
self.rotary_dim = 8
|
||||
self.mrope_section = [2, 3, 3]
|
||||
self.mrope_interleaved = False
|
||||
self.is_neox_style = True
|
||||
|
||||
def forward(self, positions, query, key):
|
||||
launched.append((positions.shape, query.shape))
|
||||
return query, key
|
||||
|
||||
class FakeMropeModel(torch.nn.Module):
|
||||
supports_mrope = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.rotary_emb = FakeRope()
|
||||
|
||||
def get_mrope_input_positions(self, input_tokens, mm_features):
|
||||
raise AssertionError("warmup must not build live M-RoPE positions")
|
||||
|
||||
runner = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda parallel_config: 4,
|
||||
get_num_kv_heads=lambda parallel_config: 2,
|
||||
),
|
||||
parallel_config=object(),
|
||||
dtype=torch.bfloat16,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
_warm_mrope(runner, FakeMropeModel())
|
||||
assert [shape for shape, _ in launched] == [
|
||||
torch.Size((3, 1)),
|
||||
torch.Size((3, 2)),
|
||||
torch.Size((3, 16)),
|
||||
]
|
||||
|
||||
|
||||
def test_mrope_warmup_skips_models_without_supports_mrope() -> None:
|
||||
def fail(*_args, **_kwargs):
|
||||
raise AssertionError("M-RoPE warmup must not run without SupportsMRoPE")
|
||||
|
||||
runner = SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_attention_heads=fail),
|
||||
get_model=fail,
|
||||
)
|
||||
_warm_mrope(runner, torch.nn.Module())
|
||||
Reference in New Issue
Block a user