[3/N][Feat][Perf] Add new warmup infrastructure for JITs. Add provider registry and orchestration for JIT warmup (#50174)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
parent
7075ddac28
commit
49905ad94d
@@ -5,16 +5,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import builtins
|
||||
import inspect
|
||||
import itertools
|
||||
import operator
|
||||
import textwrap
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Generic, TypeVar
|
||||
|
||||
__all__ = [
|
||||
"JitWarmupRegistry",
|
||||
"VllmJitKernel",
|
||||
"WarmupIntRange",
|
||||
"get_ast_full_name",
|
||||
@@ -136,8 +140,15 @@ class _CompileKeyDispatchTrace:
|
||||
field_exprs: tuple[tuple[str, ast.AST], ...]
|
||||
globals: Mapping[str, Any]
|
||||
input_names: frozenset[str]
|
||||
# Named parameters are excluded from direct **kwargs forwarding.
|
||||
named_parameters: frozenset[str] | None
|
||||
defaults: Mapping[str, Any]
|
||||
|
||||
def input_names_for(self, available_names: set[str]) -> frozenset[str]:
|
||||
if self.named_parameters is None:
|
||||
return self.input_names
|
||||
return self.input_names | (available_names - self.named_parameters)
|
||||
|
||||
def compile_key(
|
||||
self,
|
||||
compile_key_type: type[CompileKeyT],
|
||||
@@ -146,12 +157,20 @@ class _CompileKeyDispatchTrace:
|
||||
dispatch_values = _eval_local_exprs(
|
||||
self.local_exprs, {**self.defaults, **kwargs}, self.globals
|
||||
)
|
||||
return compile_key_type(
|
||||
**{
|
||||
field: _eval_dispatch_expr(expr, dispatch_values, self.globals)
|
||||
for field, expr in self.field_exprs
|
||||
named_parameters = self.named_parameters
|
||||
# Materialize direct fields before evaluating named AST expressions.
|
||||
fields: dict[str, Any] = {}
|
||||
if named_parameters is not None:
|
||||
fields = {
|
||||
name: value
|
||||
for name, value in kwargs.items()
|
||||
if name not in named_parameters
|
||||
}
|
||||
)
|
||||
for field, expr in self.field_exprs:
|
||||
if field in fields:
|
||||
raise TypeError(f"CompileKey field '{field}' is specified twice")
|
||||
fields[field] = _eval_dispatch_expr(expr, dispatch_values, self.globals)
|
||||
return compile_key_type(**fields)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -186,6 +205,10 @@ _CMP_OPS: dict[type[ast.cmpop], Callable[[Any, Any], bool]] = {
|
||||
ast.LtE: operator.le,
|
||||
ast.Gt: operator.gt,
|
||||
ast.GtE: operator.ge,
|
||||
ast.In: lambda left, right: left in right,
|
||||
ast.NotIn: lambda left, right: left not in right,
|
||||
ast.Is: operator.is_,
|
||||
ast.IsNot: operator.is_not,
|
||||
}
|
||||
|
||||
|
||||
@@ -200,8 +223,9 @@ def _dispatch_expr_error(node: ast.AST, reason: str) -> ValueError:
|
||||
return ValueError(
|
||||
f"{reason}: {_dispatch_expr_source(node)}. "
|
||||
"Supported dispatch expressions are names, constants, attributes, "
|
||||
"tuple/list literals, conditional expressions, comparisons, boolean "
|
||||
"operators, unary not/minus, arithmetic, and calls without **kwargs."
|
||||
"subscriptions, tuple/list literals, conditional expressions, "
|
||||
"comparisons, boolean operators, unary not/minus, arithmetic, and "
|
||||
"calls without **kwargs."
|
||||
)
|
||||
|
||||
|
||||
@@ -225,6 +249,8 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
|
||||
return self.values[node.id]
|
||||
if node.id in self.globals:
|
||||
return self.globals[node.id]
|
||||
if hasattr(builtins, node.id):
|
||||
return getattr(builtins, node.id)
|
||||
raise _dispatch_expr_error(node, f"Unknown dispatch name '{node.id}'")
|
||||
|
||||
def visit_Constant(self, node: ast.Constant) -> Any:
|
||||
@@ -299,6 +325,9 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
|
||||
def visit_Attribute(self, node: ast.Attribute) -> Any:
|
||||
return getattr(self.visit(node.value), node.attr)
|
||||
|
||||
def visit_Subscript(self, node: ast.Subscript) -> Any:
|
||||
return self.visit(node.value)[self.visit(node.slice)]
|
||||
|
||||
|
||||
def get_ast_full_name(node: ast.AST) -> str | None:
|
||||
if isinstance(node, ast.Name):
|
||||
@@ -310,15 +339,23 @@ def get_ast_full_name(node: ast.AST) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def get_function_source_node(fn: Callable[..., Any]) -> ast.FunctionDef:
|
||||
def get_function_source_node(fn: Callable[..., Any]) -> ast.FunctionDef | ast.Lambda:
|
||||
source_fn = getattr(fn, "fn", fn)
|
||||
source = textwrap.dedent(inspect.getsource(source_fn))
|
||||
tree = ast.parse(source)
|
||||
function_defs = [node for node in tree.body if isinstance(node, ast.FunctionDef)]
|
||||
if len(function_defs) != 1:
|
||||
name = getattr(source_fn, "__name__", type(source_fn).__name__)
|
||||
raise ValueError(f"Expected one function in {name}, found {len(function_defs)}")
|
||||
return function_defs[0]
|
||||
if len(function_defs) == 1:
|
||||
return function_defs[0]
|
||||
|
||||
lambdas = [node for node in ast.walk(tree) if isinstance(node, ast.Lambda)]
|
||||
if len(lambdas) == 1:
|
||||
return lambdas[0]
|
||||
|
||||
name = getattr(source_fn, "__name__", type(source_fn).__name__)
|
||||
raise ValueError(
|
||||
f"Expected one function or lambda in {name}, found "
|
||||
f"{len(function_defs)} functions and {len(lambdas)} lambdas"
|
||||
)
|
||||
|
||||
|
||||
def _eval_dispatch_expr(
|
||||
@@ -349,8 +386,11 @@ def _collect_input_names(
|
||||
|
||||
def _collect_expression_body(
|
||||
fn: Callable[..., Any],
|
||||
function_def: ast.FunctionDef,
|
||||
function_def: ast.FunctionDef | ast.Lambda,
|
||||
) -> tuple[list[tuple[str, ast.AST]], ast.AST]:
|
||||
if isinstance(function_def, ast.Lambda):
|
||||
return [], function_def.body
|
||||
|
||||
local_exprs: list[tuple[str, ast.AST]] = []
|
||||
for statement in function_def.body:
|
||||
if (
|
||||
@@ -421,6 +461,10 @@ def _trace_compile_key_dispatch(
|
||||
source_fn = getattr(fn, "__func__", fn)
|
||||
globals_ = source_fn.__globals__
|
||||
function_def = get_function_source_node(fn)
|
||||
if isinstance(function_def, ast.Lambda):
|
||||
raise _dispatch_expr_error(
|
||||
function_def, "Dispatch must be a function definition"
|
||||
)
|
||||
|
||||
local_exprs, return_expr = _collect_expression_body(fn, function_def)
|
||||
if not isinstance(return_expr, ast.Call):
|
||||
@@ -431,13 +475,37 @@ def _trace_compile_key_dispatch(
|
||||
|
||||
field_exprs: list[tuple[str, ast.AST]] = []
|
||||
defaults, candidate_names = _function_trace_inputs(fn)
|
||||
# Fields captured by dispatch **kwargs are forwarded, not AST-evaluated.
|
||||
signature = inspect.signature(fn)
|
||||
variadic_keyword = next(
|
||||
(
|
||||
name
|
||||
for name, parameter in signature.parameters.items()
|
||||
if parameter.kind is inspect.Parameter.VAR_KEYWORD
|
||||
),
|
||||
None,
|
||||
)
|
||||
candidate_names.discard(variadic_keyword)
|
||||
input_names: set[str] = set()
|
||||
local_names = {name for name, _ in local_exprs}
|
||||
for _, expr in local_exprs:
|
||||
input_names.update(_collect_input_names(expr, candidate_names))
|
||||
named_parameters: frozenset[str] | None = None
|
||||
for keyword in return_expr.keywords:
|
||||
# CompileKey may unpack only that **kwargs parameter, once.
|
||||
if keyword.arg is None:
|
||||
raise ValueError(f"{fn.__name__} cannot use **kwargs in CompileKey")
|
||||
if (
|
||||
named_parameters is not None
|
||||
or variadic_keyword is None
|
||||
or not isinstance(keyword.value, ast.Name)
|
||||
or keyword.value.id != variadic_keyword
|
||||
):
|
||||
raise ValueError(
|
||||
f"{fn.__name__} may unpack only its own **kwargs parameter "
|
||||
"once in CompileKey"
|
||||
)
|
||||
named_parameters = frozenset(candidate_names)
|
||||
continue
|
||||
field_exprs.append((keyword.arg, keyword.value))
|
||||
input_names.update(
|
||||
_collect_input_names(keyword.value, candidate_names, local_names)
|
||||
@@ -448,6 +516,7 @@ def _trace_compile_key_dispatch(
|
||||
tuple(field_exprs),
|
||||
globals_,
|
||||
frozenset(input_names),
|
||||
named_parameters,
|
||||
defaults,
|
||||
)
|
||||
|
||||
@@ -482,10 +551,32 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
CompileKey: type[CompileKeyT]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.compile_key_dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
|
||||
self._dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
|
||||
self._compiled_cache: dict[Any, Any] = {}
|
||||
|
||||
def compile_key(self, kwargs: Mapping[str, Any]) -> CompileKeyT:
|
||||
return self.compile_key_dispatch_trace.compile_key(self.CompileKey, kwargs)
|
||||
return self._dispatch_trace.compile_key(self.CompileKey, kwargs)
|
||||
|
||||
def _get_or_compile(
|
||||
self,
|
||||
compile_key: CompileKeyT,
|
||||
*,
|
||||
runtime_context: Mapping[str, Any] | None = None,
|
||||
) -> Any:
|
||||
"""Return a cached executor, compiling it on a monitored cache miss."""
|
||||
if compile_key not in self._compiled_cache:
|
||||
self.compile(compile_key)
|
||||
|
||||
try:
|
||||
return self._compiled_cache[compile_key]
|
||||
except KeyError as exc:
|
||||
details = [f"compile_key={compile_key!r}"]
|
||||
if runtime_context:
|
||||
details.append(f"runtime_context={dict(runtime_context)!r}")
|
||||
raise RuntimeError(
|
||||
f"{type(self).__name__}.compile(...) did not cache its JIT "
|
||||
f"executor ({', '.join(details)})"
|
||||
) from exc
|
||||
|
||||
def _trace_dispatch(
|
||||
self, dispatch: CompileKeyDispatchFn[CompileKeyT]
|
||||
@@ -506,7 +597,11 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
predicate_trace = (
|
||||
_trace_warmup_predicate(_when) if _when is not None else None
|
||||
)
|
||||
input_names = compile_key_dispatch_trace.input_names
|
||||
# Unmatched **kwargs fields also belong to the expansion space.
|
||||
available_names = set(kwargs).union(
|
||||
*(group.rows[0] for group in input_groups)
|
||||
)
|
||||
input_names = compile_key_dispatch_trace.input_names_for(available_names)
|
||||
if predicate_trace is not None:
|
||||
input_names = input_names | predicate_trace.input_names
|
||||
expanded_input_groups = tuple(
|
||||
@@ -565,7 +660,105 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
"""Compile one warmup key."""
|
||||
raise NotImplementedError
|
||||
|
||||
def register_warmup(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""Register this kernel with the active runner's warmup registry."""
|
||||
JitWarmupRegistry.register(self, *args, **kwargs)
|
||||
|
||||
def warmup(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""Compile this kernel's warmup keys."""
|
||||
for compile_key in self.get_warmup_keys(*args, **kwargs):
|
||||
self.compile(compile_key)
|
||||
|
||||
|
||||
class JitWarmupRegistry:
|
||||
"""Collect and compile JIT kernels selected during runner setup."""
|
||||
|
||||
_active: ContextVar[JitWarmupRegistry | None] = ContextVar(
|
||||
"active_jit_warmup_registry",
|
||||
default=None,
|
||||
)
|
||||
|
||||
def __init__(self, vllm_config: Any) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self._registrations: dict[
|
||||
VllmJitKernel[Any],
|
||||
list[tuple[tuple[Any, ...], dict[str, Any]]],
|
||||
] = {}
|
||||
|
||||
@contextmanager
|
||||
def activate(self) -> Iterator[None]:
|
||||
"""Collect registrations made in this context."""
|
||||
token = self._active.set(self)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._active.reset(token)
|
||||
|
||||
@classmethod
|
||||
def register(
|
||||
cls,
|
||||
kernel: VllmJitKernel[Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Register a kernel with the active registry, if one exists."""
|
||||
registry = cls._active.get()
|
||||
if registry is not None:
|
||||
registry._add(kernel, args, kwargs)
|
||||
|
||||
def _add(
|
||||
self,
|
||||
kernel: VllmJitKernel[Any],
|
||||
args: tuple[Any, ...],
|
||||
kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
registrations = self._registrations.setdefault(kernel, [])
|
||||
if (
|
||||
not args
|
||||
and not kwargs
|
||||
and any(
|
||||
not registered_args and not registered_kwargs
|
||||
for registered_args, registered_kwargs in registrations
|
||||
)
|
||||
):
|
||||
return
|
||||
registrations.append((args, kwargs))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return sum(len(registrations) for registrations in self._registrations.values())
|
||||
|
||||
def warmup(self) -> None:
|
||||
"""Expand registrations and compile each wrapper/key pair once."""
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.distributed import is_global_first_rank
|
||||
|
||||
kernel_items: list[tuple[VllmJitKernel[Any], dict[Any, None]]] = []
|
||||
for kernel, registrations in self._registrations.items():
|
||||
compile_keys: dict[Any, None] = {}
|
||||
for args, kwargs in registrations:
|
||||
if not args and not kwargs:
|
||||
args = (self.vllm_config,)
|
||||
for compile_key in kernel.get_warmup_keys(*args, **kwargs):
|
||||
compile_keys[compile_key] = None
|
||||
if compile_keys:
|
||||
kernel_items.append((kernel, compile_keys))
|
||||
|
||||
if not kernel_items:
|
||||
return
|
||||
|
||||
total_keys = sum(len(compile_keys) for _, compile_keys in kernel_items)
|
||||
with tqdm(
|
||||
kernel_items,
|
||||
desc=f"JIT kernel warmup ({total_keys} compile keys)",
|
||||
disable=not is_global_first_rank(),
|
||||
dynamic_ncols=True,
|
||||
unit="kernel",
|
||||
) as progress:
|
||||
for kernel, compile_keys in progress:
|
||||
progress.set_postfix_str(
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
)
|
||||
for compile_key in compile_keys:
|
||||
kernel.compile(compile_key)
|
||||
|
||||
@@ -12,6 +12,42 @@ from vllm.model_executor.warmup.jit_warmup import (
|
||||
)
|
||||
|
||||
|
||||
def triton_scalar_specialization_rep(value: int) -> int:
|
||||
"""Return an integer with the same default Triton JIT specialization.
|
||||
|
||||
For an ordinary integer argument, Triton's cache key contains its inferred
|
||||
type (``i32``, ``i64``, or ``u64``) and one of three value classes:
|
||||
|
||||
* ``1`` is specialized as the exact constant ``1``.
|
||||
* Multiples of 16 receive a ``tt.divisibility = 16`` attribute.
|
||||
* All other values have no value specialization.
|
||||
|
||||
Warmup only needs one concrete value for each cache-key class. This helper
|
||||
returns ``1`` for the exact-one class and otherwise returns a divisible or
|
||||
generic representative while preserving the inferred integer type.
|
||||
|
||||
This applies only to non-``constexpr`` integer arguments using Triton's
|
||||
default specialization. Do not use it for arguments listed in
|
||||
``do_not_specialize`` or ``do_not_specialize_on_alignment``.
|
||||
"""
|
||||
if value == 1:
|
||||
return 1
|
||||
|
||||
if -(1 << 31) <= value < (1 << 31):
|
||||
divisible_rep = 16
|
||||
generic_rep = 2
|
||||
elif -(1 << 63) <= value < (1 << 63):
|
||||
divisible_rep = 1 << 31
|
||||
generic_rep = (1 << 31) + 1
|
||||
elif 0 <= value < (1 << 64):
|
||||
divisible_rep = 1 << 63
|
||||
generic_rep = (1 << 63) + 1
|
||||
else:
|
||||
raise OverflowError(f"Integer {value} is outside Triton's scalar range")
|
||||
|
||||
return divisible_rep if value % 16 == 0 else generic_rep
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonWarmupTensor:
|
||||
# Compile-only tensor descriptor for Triton pointer specialization.
|
||||
@@ -167,6 +203,8 @@ def trace_triton_kernel_specialization_args(
|
||||
kernel: Callable[..., Any],
|
||||
) -> tuple[str, ...]:
|
||||
function_def = get_function_source_node(kernel)
|
||||
if not isinstance(function_def, ast.FunctionDef):
|
||||
raise ValueError("Expected Triton kernel to be defined as a function")
|
||||
source_fn = getattr(kernel, "fn", kernel)
|
||||
arg_names = tuple(inspect.signature(source_fn).parameters)
|
||||
constexpr_args = _triton_constexpr_arg_names(kernel, function_def, arg_names)
|
||||
|
||||
@@ -6,6 +6,7 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
|
||||
happen during model execution.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -36,9 +37,6 @@ from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
|
||||
from vllm.model_executor.warmup.sparse_mla_triton_warmup import (
|
||||
sparse_mla_triton_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.v1_block_table_warmup import (
|
||||
warm_v1_block_table_kernels,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
@@ -103,15 +101,28 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
)
|
||||
|
||||
if not worker.use_v2_model_runner:
|
||||
# Pooling models do not use the generation slot-mapping path.
|
||||
if not worker.model_runner.is_pooling_model:
|
||||
warm_v1_block_table_kernels(worker.model_runner)
|
||||
# The KV-block zeroing kernel is driven by the scheduler's
|
||||
# `new_block_ids_to_zero`, so no dummy run ever reaches it.
|
||||
zeroer = getattr(worker.model_runner, "_kv_block_zeroer", None)
|
||||
if zeroer is not None:
|
||||
zeroer.warmup(worker.model_runner.kv_cache_config.num_blocks)
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
logger.info("JIT kernel warmup starting.")
|
||||
jit_warmup_start = time.perf_counter()
|
||||
try:
|
||||
worker.model_runner.jit_warmup_registry.warmup()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"JIT kernel warmup failed after %.2fs.",
|
||||
time.perf_counter() - jit_warmup_start,
|
||||
)
|
||||
raise
|
||||
logger.info(
|
||||
"JIT kernel warmup finished in %.2fs.",
|
||||
time.perf_counter() - jit_warmup_start,
|
||||
)
|
||||
|
||||
qwen_triton_warmup(worker.model_runner, worker.vllm_config.model_config)
|
||||
|
||||
compilation_config = worker.vllm_config.compilation_config
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up v1 block-table Triton kernels."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
_SLOT_MAPPING_WARMUP_TOKENS = 8
|
||||
|
||||
|
||||
def warm_v1_block_table_kernels(runner: "GPUModelRunner") -> None:
|
||||
"""JIT-compile ``_compute_slot_mapping_kernel`` for the real block tables."""
|
||||
|
||||
device = runner.device
|
||||
block_table = runner.input_batch.block_table
|
||||
num_tokens = min(
|
||||
_SLOT_MAPPING_WARMUP_TOKENS,
|
||||
runner.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
if num_tokens <= 0:
|
||||
return
|
||||
|
||||
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
block_table.compute_slot_mapping(1, query_start_loc, positions)
|
||||
Reference in New Issue
Block a user