[Perf] Parallelize registered CUDA Triton kernel warmup at startup (#58582)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Misha Goin
2026-09-24 19:52:00 -04:00
committed by GitHub
co-authored by Codex
parent d5051abaf1
commit 71891bdb94
3 changed files with 139 additions and 0 deletions
@@ -1,14 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from dataclasses import dataclass
from typing import Any
import pytest
import torch
from vllm import envs
from vllm.model_executor.warmup import jit_warmup_triton_helper
from vllm.model_executor.warmup.jit_warmup import (
JitWarmupRegistry,
WarmupChoices,
WarmupIntRange,
kernel_launcher,
@@ -116,6 +119,105 @@ def test_triton_launcher_supports_compile_and_runtime_adapters() -> None:
assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})]
@pytest.mark.skipif(
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_failed_parallel_warmup_does_not_leak_into_runtime(monkeypatch) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
owner = _TestTritonKernel()
owner.kernel = _FakeTritonKernel()
def fail():
raise RuntimeError("warmup failed")
def warmup(**kwargs):
active_mode.get().submit(kwargs["second"], fail, lambda result: None)
monkeypatch.setattr(owner.kernel, "warmup", warmup)
with pytest.raises(RuntimeError, match="warmup failed"):
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
assert active_mode.get() is None
owner("runtime", 3, None)
assert len(owner.kernel.runtime_calls) == 1
def test_autotuning_warmup_stays_serial(monkeypatch) -> None:
owner = _TestTritonKernel()
owner.kernel = _FakeTritonKernel()
owner._run_autotune = True
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 4)
def unexpected_async(*args):
pytest.fail("Autotuning must not use asynchronous compilation")
monkeypatch.setattr(triton, "AsyncCompileMode", unexpected_async, raising=False)
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
assert len(owner.kernel.runtime_calls) == 2
@triton.jit
def _warmup_store_kernel(out, VALUE: tl.constexpr):
tl.store(out, VALUE)
@pytest.mark.skipif(
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
monkeypatch,
) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
def warmup_inputs():
return dict(out=TritonWarmupTensor(torch.int32), value=WarmupChoices(1, 2))
@triton_kernel_dispatcher_with_warmup(
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
)
def store(out, value):
return (1,), dict(VALUE=value)
store.get_warmup_keys() # Initialize Triton's binder before wrapping its compiler.
caller = threading.get_ident()
barrier = threading.Barrier(2, timeout=10)
compiled_on = []
compile_kernel = _warmup_store_kernel.compile
def record_compile(*args, **kwargs):
if threading.get_ident() != caller:
barrier.wait()
result = compile_kernel(*args, **kwargs)
compiled_on.append(threading.get_ident())
return result
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
registry = JitWarmupRegistry(None)
with registry.activate():
store.register_warmup()
registry.warmup()
assert len(compiled_on) == 2
assert all(thread != caller for thread in compiled_on)
assert active_mode.get() is None
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
for value in (1, 2):
store(out, value)
assert out.item() == value
assert len(compiled_on) == 2
# A specialization first seen during inference uses the normal blocking JIT.
store(out, 3)
assert out.item() == 3
assert compiled_on[2:] == [caller]
def test_triton_launcher_supports_cpu_function_wrappers() -> None:
calls: list[tuple[Any, ...]] = []
+7
View File
@@ -123,6 +123,7 @@ if TYPE_CHECKING:
VLLM_USE_TRITON_AWQ: bool = False
VLLM_FASTSAFETENSORS_QUEUE_SIZE: int = 0
VLLM_TRITON_FORCE_FIRST_CONFIG: bool = False
VLLM_TRITON_JIT_WARMUP_NUM_THREADS: int = 4
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
VLLM_SKIP_P2P_CHECK: bool = False
VLLM_DISABLED_KERNELS: list[str] = []
@@ -1181,6 +1182,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
os.environ.get("VLLM_TRITON_FORCE_FIRST_CONFIG", "0").strip().lower()
in ("1", "true")
),
# Maximum compiler threads per worker for registered CUDA Triton JIT warmup.
# Set to 1 for serial warmup. Does not affect runtime JIT or autotuning.
"VLLM_TRITON_JIT_WARMUP_NUM_THREADS": lambda: int(
os.getenv("VLLM_TRITON_JIT_WARMUP_NUM_THREADS", "4")
),
# If set, allow loading or unloading lora adapters in runtime,
"VLLM_ALLOW_RUNTIME_LORA_UPDATING": lambda: (
os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower()
@@ -2272,6 +2278,7 @@ def compile_factors() -> dict[str, object]:
hash everything else. This keeps the cache key aligned across workers."""
ignored_factors: set[str] = {
"MAX_JOBS",
"VLLM_TRITON_JIT_WARMUP_NUM_THREADS",
"VLLM_RPC_BASE_PATH",
"VLLM_USE_MODELSCOPE",
"VLLM_RINGBUFFER_WARNING_INTERVAL",
@@ -4,11 +4,14 @@ import ast
import inspect
from abc import abstractmethod
from collections.abc import Callable, Hashable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext, suppress
from contextvars import copy_context
from dataclasses import dataclass, field
from functools import cache, cached_property, update_wrapper
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
from vllm import envs
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
get_ast_full_name,
@@ -227,6 +230,33 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
self._warming = False
self._warming_compile_key = None
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
"""Compile CUDA startup warmup variants in parallel, then wait."""
keys = list(compile_keys)
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
async_compile = getattr(triton, "AsyncCompileMode", None)
# AMD LLVM code generation can abort under concurrent compilation.
if (
not current_platform.is_cuda()
or self._run_autotune
or async_compile is None
or num_threads <= 1
):
return super().compile_many(keys)
def compile_all() -> None:
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
async_compile(executor),
):
# Dispatch stays on this thread: compile() mutates owner state.
# Only Triton's underlying compiler work runs in the pool.
super(VllmTritonJitKernel, self).compile_many(keys)
# Older Triton versions leave async mode active on compilation failure.
# Isolate that state so it cannot affect later runtime JIT compilation.
copy_context().run(compile_all)
@cached_property
def _kernel_arg_names(self) -> tuple[str, ...]:
arg_names = getattr(self.kernel, "arg_names", None)