[3/N][Feat][Perf] Add new warmup infrastructure for JITs. Add provider registry and orchestration for JIT warmup (#50174)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-08-17 10:11:14 -07:00
committed by GitHub
co-authored by Codex
parent 7075ddac28
commit 49905ad94d
12 changed files with 1064 additions and 165 deletions
+211 -18
View File
@@ -5,16 +5,20 @@
from __future__ import annotations
import ast
import builtins
import inspect
import itertools
import operator
import textwrap
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Callable, Iterable, Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Any, Generic, TypeVar
__all__ = [
"JitWarmupRegistry",
"VllmJitKernel",
"WarmupIntRange",
"get_ast_full_name",
@@ -136,8 +140,15 @@ class _CompileKeyDispatchTrace:
field_exprs: tuple[tuple[str, ast.AST], ...]
globals: Mapping[str, Any]
input_names: frozenset[str]
# Named parameters are excluded from direct **kwargs forwarding.
named_parameters: frozenset[str] | None
defaults: Mapping[str, Any]
def input_names_for(self, available_names: set[str]) -> frozenset[str]:
if self.named_parameters is None:
return self.input_names
return self.input_names | (available_names - self.named_parameters)
def compile_key(
self,
compile_key_type: type[CompileKeyT],
@@ -146,12 +157,20 @@ class _CompileKeyDispatchTrace:
dispatch_values = _eval_local_exprs(
self.local_exprs, {**self.defaults, **kwargs}, self.globals
)
return compile_key_type(
**{
field: _eval_dispatch_expr(expr, dispatch_values, self.globals)
for field, expr in self.field_exprs
named_parameters = self.named_parameters
# Materialize direct fields before evaluating named AST expressions.
fields: dict[str, Any] = {}
if named_parameters is not None:
fields = {
name: value
for name, value in kwargs.items()
if name not in named_parameters
}
)
for field, expr in self.field_exprs:
if field in fields:
raise TypeError(f"CompileKey field '{field}' is specified twice")
fields[field] = _eval_dispatch_expr(expr, dispatch_values, self.globals)
return compile_key_type(**fields)
@dataclass(frozen=True)
@@ -186,6 +205,10 @@ _CMP_OPS: dict[type[ast.cmpop], Callable[[Any, Any], bool]] = {
ast.LtE: operator.le,
ast.Gt: operator.gt,
ast.GtE: operator.ge,
ast.In: lambda left, right: left in right,
ast.NotIn: lambda left, right: left not in right,
ast.Is: operator.is_,
ast.IsNot: operator.is_not,
}
@@ -200,8 +223,9 @@ def _dispatch_expr_error(node: ast.AST, reason: str) -> ValueError:
return ValueError(
f"{reason}: {_dispatch_expr_source(node)}. "
"Supported dispatch expressions are names, constants, attributes, "
"tuple/list literals, conditional expressions, comparisons, boolean "
"operators, unary not/minus, arithmetic, and calls without **kwargs."
"subscriptions, tuple/list literals, conditional expressions, "
"comparisons, boolean operators, unary not/minus, arithmetic, and "
"calls without **kwargs."
)
@@ -225,6 +249,8 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
return self.values[node.id]
if node.id in self.globals:
return self.globals[node.id]
if hasattr(builtins, node.id):
return getattr(builtins, node.id)
raise _dispatch_expr_error(node, f"Unknown dispatch name '{node.id}'")
def visit_Constant(self, node: ast.Constant) -> Any:
@@ -299,6 +325,9 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
def visit_Attribute(self, node: ast.Attribute) -> Any:
return getattr(self.visit(node.value), node.attr)
def visit_Subscript(self, node: ast.Subscript) -> Any:
return self.visit(node.value)[self.visit(node.slice)]
def get_ast_full_name(node: ast.AST) -> str | None:
if isinstance(node, ast.Name):
@@ -310,15 +339,23 @@ def get_ast_full_name(node: ast.AST) -> str | None:
return None
def get_function_source_node(fn: Callable[..., Any]) -> ast.FunctionDef:
def get_function_source_node(fn: Callable[..., Any]) -> ast.FunctionDef | ast.Lambda:
source_fn = getattr(fn, "fn", fn)
source = textwrap.dedent(inspect.getsource(source_fn))
tree = ast.parse(source)
function_defs = [node for node in tree.body if isinstance(node, ast.FunctionDef)]
if len(function_defs) != 1:
name = getattr(source_fn, "__name__", type(source_fn).__name__)
raise ValueError(f"Expected one function in {name}, found {len(function_defs)}")
return function_defs[0]
if len(function_defs) == 1:
return function_defs[0]
lambdas = [node for node in ast.walk(tree) if isinstance(node, ast.Lambda)]
if len(lambdas) == 1:
return lambdas[0]
name = getattr(source_fn, "__name__", type(source_fn).__name__)
raise ValueError(
f"Expected one function or lambda in {name}, found "
f"{len(function_defs)} functions and {len(lambdas)} lambdas"
)
def _eval_dispatch_expr(
@@ -349,8 +386,11 @@ def _collect_input_names(
def _collect_expression_body(
fn: Callable[..., Any],
function_def: ast.FunctionDef,
function_def: ast.FunctionDef | ast.Lambda,
) -> tuple[list[tuple[str, ast.AST]], ast.AST]:
if isinstance(function_def, ast.Lambda):
return [], function_def.body
local_exprs: list[tuple[str, ast.AST]] = []
for statement in function_def.body:
if (
@@ -421,6 +461,10 @@ def _trace_compile_key_dispatch(
source_fn = getattr(fn, "__func__", fn)
globals_ = source_fn.__globals__
function_def = get_function_source_node(fn)
if isinstance(function_def, ast.Lambda):
raise _dispatch_expr_error(
function_def, "Dispatch must be a function definition"
)
local_exprs, return_expr = _collect_expression_body(fn, function_def)
if not isinstance(return_expr, ast.Call):
@@ -431,13 +475,37 @@ def _trace_compile_key_dispatch(
field_exprs: list[tuple[str, ast.AST]] = []
defaults, candidate_names = _function_trace_inputs(fn)
# Fields captured by dispatch **kwargs are forwarded, not AST-evaluated.
signature = inspect.signature(fn)
variadic_keyword = next(
(
name
for name, parameter in signature.parameters.items()
if parameter.kind is inspect.Parameter.VAR_KEYWORD
),
None,
)
candidate_names.discard(variadic_keyword)
input_names: set[str] = set()
local_names = {name for name, _ in local_exprs}
for _, expr in local_exprs:
input_names.update(_collect_input_names(expr, candidate_names))
named_parameters: frozenset[str] | None = None
for keyword in return_expr.keywords:
# CompileKey may unpack only that **kwargs parameter, once.
if keyword.arg is None:
raise ValueError(f"{fn.__name__} cannot use **kwargs in CompileKey")
if (
named_parameters is not None
or variadic_keyword is None
or not isinstance(keyword.value, ast.Name)
or keyword.value.id != variadic_keyword
):
raise ValueError(
f"{fn.__name__} may unpack only its own **kwargs parameter "
"once in CompileKey"
)
named_parameters = frozenset(candidate_names)
continue
field_exprs.append((keyword.arg, keyword.value))
input_names.update(
_collect_input_names(keyword.value, candidate_names, local_names)
@@ -448,6 +516,7 @@ def _trace_compile_key_dispatch(
tuple(field_exprs),
globals_,
frozenset(input_names),
named_parameters,
defaults,
)
@@ -482,10 +551,32 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
CompileKey: type[CompileKeyT]
def __init__(self) -> None:
self.compile_key_dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
self._dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
self._compiled_cache: dict[Any, Any] = {}
def compile_key(self, kwargs: Mapping[str, Any]) -> CompileKeyT:
return self.compile_key_dispatch_trace.compile_key(self.CompileKey, kwargs)
return self._dispatch_trace.compile_key(self.CompileKey, kwargs)
def _get_or_compile(
self,
compile_key: CompileKeyT,
*,
runtime_context: Mapping[str, Any] | None = None,
) -> Any:
"""Return a cached executor, compiling it on a monitored cache miss."""
if compile_key not in self._compiled_cache:
self.compile(compile_key)
try:
return self._compiled_cache[compile_key]
except KeyError as exc:
details = [f"compile_key={compile_key!r}"]
if runtime_context:
details.append(f"runtime_context={dict(runtime_context)!r}")
raise RuntimeError(
f"{type(self).__name__}.compile(...) did not cache its JIT "
f"executor ({', '.join(details)})"
) from exc
def _trace_dispatch(
self, dispatch: CompileKeyDispatchFn[CompileKeyT]
@@ -506,7 +597,11 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
predicate_trace = (
_trace_warmup_predicate(_when) if _when is not None else None
)
input_names = compile_key_dispatch_trace.input_names
# Unmatched **kwargs fields also belong to the expansion space.
available_names = set(kwargs).union(
*(group.rows[0] for group in input_groups)
)
input_names = compile_key_dispatch_trace.input_names_for(available_names)
if predicate_trace is not None:
input_names = input_names | predicate_trace.input_names
expanded_input_groups = tuple(
@@ -565,7 +660,105 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
"""Compile one warmup key."""
raise NotImplementedError
def register_warmup(self, *args: Any, **kwargs: Any) -> None:
"""Register this kernel with the active runner's warmup registry."""
JitWarmupRegistry.register(self, *args, **kwargs)
def warmup(self, *args: Any, **kwargs: Any) -> None:
"""Compile this kernel's warmup keys."""
for compile_key in self.get_warmup_keys(*args, **kwargs):
self.compile(compile_key)
class JitWarmupRegistry:
"""Collect and compile JIT kernels selected during runner setup."""
_active: ContextVar[JitWarmupRegistry | None] = ContextVar(
"active_jit_warmup_registry",
default=None,
)
def __init__(self, vllm_config: Any) -> None:
self.vllm_config = vllm_config
self._registrations: dict[
VllmJitKernel[Any],
list[tuple[tuple[Any, ...], dict[str, Any]]],
] = {}
@contextmanager
def activate(self) -> Iterator[None]:
"""Collect registrations made in this context."""
token = self._active.set(self)
try:
yield
finally:
self._active.reset(token)
@classmethod
def register(
cls,
kernel: VllmJitKernel[Any],
*args: Any,
**kwargs: Any,
) -> None:
"""Register a kernel with the active registry, if one exists."""
registry = cls._active.get()
if registry is not None:
registry._add(kernel, args, kwargs)
def _add(
self,
kernel: VllmJitKernel[Any],
args: tuple[Any, ...],
kwargs: dict[str, Any],
) -> None:
registrations = self._registrations.setdefault(kernel, [])
if (
not args
and not kwargs
and any(
not registered_args and not registered_kwargs
for registered_args, registered_kwargs in registrations
)
):
return
registrations.append((args, kwargs))
def __len__(self) -> int:
return sum(len(registrations) for registrations in self._registrations.values())
def warmup(self) -> None:
"""Expand registrations and compile each wrapper/key pair once."""
from tqdm import tqdm
from vllm.distributed import is_global_first_rank
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
for kernel, registrations in self._registrations.items():
compile_keys: dict[Any, None] = {}
for args, kwargs in registrations:
if not args and not kwargs:
args = (self.vllm_config,)
for compile_key in kernel.get_warmup_keys(*args, **kwargs):
compile_keys[compile_key] = None
if compile_keys:
kernel_items.append((kernel, compile_keys))
if not kernel_items:
return
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
with tqdm(
kernel_items,
desc=f"JIT kernel warmup ({total_keys} compile keys)",
disable=not is_global_first_rank(),
dynamic_ncols=True,
unit="kernel",
) as progress:
for kernel, compile_keys in progress:
progress.set_postfix_str(
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
refresh=False,
)
for compile_key in compile_keys:
kernel.compile(compile_key)
@@ -12,6 +12,42 @@ from vllm.model_executor.warmup.jit_warmup import (
)
def triton_scalar_specialization_rep(value: int) -> int:
"""Return an integer with the same default Triton JIT specialization.
For an ordinary integer argument, Triton's cache key contains its inferred
type (``i32``, ``i64``, or ``u64``) and one of three value classes:
* ``1`` is specialized as the exact constant ``1``.
* Multiples of 16 receive a ``tt.divisibility = 16`` attribute.
* All other values have no value specialization.
Warmup only needs one concrete value for each cache-key class. This helper
returns ``1`` for the exact-one class and otherwise returns a divisible or
generic representative while preserving the inferred integer type.
This applies only to non-``constexpr`` integer arguments using Triton's
default specialization. Do not use it for arguments listed in
``do_not_specialize`` or ``do_not_specialize_on_alignment``.
"""
if value == 1:
return 1
if -(1 << 31) <= value < (1 << 31):
divisible_rep = 16
generic_rep = 2
elif -(1 << 63) <= value < (1 << 63):
divisible_rep = 1 << 31
generic_rep = (1 << 31) + 1
elif 0 <= value < (1 << 64):
divisible_rep = 1 << 63
generic_rep = (1 << 63) + 1
else:
raise OverflowError(f"Integer {value} is outside Triton's scalar range")
return divisible_rep if value % 16 == 0 else generic_rep
@dataclass(frozen=True)
class TritonWarmupTensor:
# Compile-only tensor descriptor for Triton pointer specialization.
@@ -167,6 +203,8 @@ def trace_triton_kernel_specialization_args(
kernel: Callable[..., Any],
) -> tuple[str, ...]:
function_def = get_function_source_node(kernel)
if not isinstance(function_def, ast.FunctionDef):
raise ValueError("Expected Triton kernel to be defined as a function")
source_fn = getattr(kernel, "fn", kernel)
arg_names = tuple(inspect.signature(source_fn).parameters)
constexpr_args = _triton_constexpr_arg_names(kernel, function_def, arg_names)
+17 -6
View File
@@ -6,6 +6,7 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
happen during model execution.
"""
import time
from typing import TYPE_CHECKING
import torch
@@ -36,9 +37,6 @@ from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
from vllm.model_executor.warmup.sparse_mla_triton_warmup import (
sparse_mla_triton_warmup,
)
from vllm.model_executor.warmup.v1_block_table_warmup import (
warm_v1_block_table_kernels,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import has_flashinfer
@@ -103,15 +101,28 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
)
if not worker.use_v2_model_runner:
# Pooling models do not use the generation slot-mapping path.
if not worker.model_runner.is_pooling_model:
warm_v1_block_table_kernels(worker.model_runner)
# The KV-block zeroing kernel is driven by the scheduler's
# `new_block_ids_to_zero`, so no dummy run ever reaches it.
zeroer = getattr(worker.model_runner, "_kv_block_zeroer", None)
if zeroer is not None:
zeroer.warmup(worker.model_runner.kv_cache_config.num_blocks)
if worker.vllm_config.kernel_config.enable_jit_warmup:
logger.info("JIT kernel warmup starting.")
jit_warmup_start = time.perf_counter()
try:
worker.model_runner.jit_warmup_registry.warmup()
except Exception:
logger.exception(
"JIT kernel warmup failed after %.2fs.",
time.perf_counter() - jit_warmup_start,
)
raise
logger.info(
"JIT kernel warmup finished in %.2fs.",
time.perf_counter() - jit_warmup_start,
)
qwen_triton_warmup(worker.model_runner, worker.vllm_config.model_config)
compilation_config = worker.vllm_config.compilation_config
@@ -1,29 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up v1 block-table Triton kernels."""
from typing import TYPE_CHECKING
import torch
if TYPE_CHECKING:
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
_SLOT_MAPPING_WARMUP_TOKENS = 8
def warm_v1_block_table_kernels(runner: "GPUModelRunner") -> None:
"""JIT-compile ``_compute_slot_mapping_kernel`` for the real block tables."""
device = runner.device
block_table = runner.input_batch.block_table
num_tokens = min(
_SLOT_MAPPING_WARMUP_TOKENS,
runner.scheduler_config.max_num_batched_tokens,
)
if num_tokens <= 0:
return
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
block_table.compute_slot_mapping(1, query_start_loc, positions)