[Perf] Parallelize registered CUDA Triton kernel warmup at startup (#58582)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Misha Goin
2026-09-24 19:52:00 -04:00
committed by GitHub
co-authored by Codex
parent d5051abaf1
commit 71891bdb94
3 changed files with 139 additions and 0 deletions
@@ -4,11 +4,14 @@ import ast
import inspect
from abc import abstractmethod
from collections.abc import Callable, Hashable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext, suppress
from contextvars import copy_context
from dataclasses import dataclass, field
from functools import cache, cached_property, update_wrapper
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
from vllm import envs
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
get_ast_full_name,
@@ -227,6 +230,33 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
self._warming = False
self._warming_compile_key = None
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
"""Compile CUDA startup warmup variants in parallel, then wait."""
keys = list(compile_keys)
num_threads = min(envs.VLLM_TRITON_JIT_WARMUP_NUM_THREADS, len(keys))
async_compile = getattr(triton, "AsyncCompileMode", None)
# AMD LLVM code generation can abort under concurrent compilation.
if (
not current_platform.is_cuda()
or self._run_autotune
or async_compile is None
or num_threads <= 1
):
return super().compile_many(keys)
def compile_all() -> None:
with (
ThreadPoolExecutor(max_workers=num_threads) as executor,
async_compile(executor),
):
# Dispatch stays on this thread: compile() mutates owner state.
# Only Triton's underlying compiler work runs in the pool.
super(VllmTritonJitKernel, self).compile_many(keys)
# Older Triton versions leave async mode active on compilation failure.
# Isolate that state so it cannot affect later runtime JIT compilation.
copy_context().run(compile_all)
@cached_property
def _kernel_arg_names(self) -> tuple[str, ...]:
arg_names = getattr(self.kernel, "arg_names", None)