[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
@@ -1,8 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -33,11 +41,39 @@ def _scatter_states_kernel(
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
def _scatter_states_warmup_inputs(
*,
state_shape: tuple[int, ...],
state_dtype: torch.dtype,
indices_dtype: torch.dtype,
max_num_tokens: int,
) -> dict[str, object]:
num_tokens: Any = WarmupIntRange(1, max_num_tokens + 1)
return dict(
state=TritonWarmupTensor(
state_dtype,
shape=(1,) + state_shape,
),
src=TritonWarmupTensor(
state_dtype,
shape=(num_tokens,) + state_shape,
),
indices=TritonWarmupTensor(
indices_dtype,
shape=(num_tokens,),
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=_scatter_states_kernel,
warmup_inputs=_scatter_states_warmup_inputs,
)
def scatter_states(
state: torch.Tensor,
src: torch.Tensor,
indices: torch.Tensor,
) -> None:
) -> DispatchSpec:
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
@@ -58,13 +94,10 @@ def scatter_states(
assert src[0].is_contiguous()
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
_scatter_states_kernel[grid](
state,
src,
indices,
state.stride(0),
src.stride(0),
indices.stride(0),
return grid, dict(
stride_state_batch=state.stride(0),
stride_src_batch=src.stride(0),
stride_indices=indices.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
+312 -33
View File
@@ -21,6 +21,7 @@ from typing import Any, Generic, TypeVar, cast
__all__ = [
"JitWarmupRegistry",
"VllmJitKernel",
"WarmupChoices",
"WarmupIntRange",
"get_ast_full_name",
"get_function_source_node",
@@ -62,6 +63,21 @@ class WarmupIntRange:
advance: Callable[[int], int] | None = None
@dataclass(frozen=True)
class WarmupChoices:
"""Expand an explicit finite set of values in a traced warmup method."""
values: tuple[Any, ...]
def __init__(self, *values: Any) -> None:
object.__setattr__(self, "values", values)
def _when(condition: bool) -> None:
"""Filter symbolic cases in an AST-traced warmup-cases method."""
raise RuntimeError("_when() is only valid in an AST-traced warmup method")
WarmupValues = Any
CompileKeyDispatchFn = Callable[..., CompileKeyT]
WarmupPredicateFn = Callable[..., bool]
@@ -87,6 +103,8 @@ class _WarmupInputRows:
def _expand_warmup_values(values: WarmupValues) -> tuple[Any, ...]:
if isinstance(values, WarmupChoices):
return values.values
if isinstance(values, WarmupIntRange):
if values.advance is None:
return tuple(range(values.start, values.stop, values.step))
@@ -278,6 +296,27 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
def visit_Constant(self, node: ast.Constant) -> Any:
return node.value
def visit_Lambda(self, node: ast.Lambda) -> Callable[..., Any]:
arguments = node.args
if (
arguments.posonlyargs
or arguments.vararg is not None
or arguments.kwonlyargs
or arguments.kwarg is not None
or arguments.defaults
):
raise _dispatch_expr_error(node, "Traced lambdas require positional args")
names = tuple(argument.arg for argument in arguments.args)
def evaluate(*values: Any) -> Any:
if len(values) != len(names):
raise TypeError(f"Expected {len(names)} lambda arguments")
return type(self)(
dict(self.values) | dict(zip(names, values)), self.globals
).eval(node.body)
return evaluate
def visit_IfExp(self, node: ast.IfExp) -> Any:
return self.visit(node.body if self.visit(node.test) else node.orelse)
@@ -333,7 +372,12 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
return op(self.visit(node.left), self.visit(node.right))
def visit_Call(self, node: ast.Call) -> Any:
args = [self.visit(arg) for arg in node.args]
args: list[Any] = []
for arg in node.args:
if isinstance(arg, ast.Starred):
args.extend(self.visit(arg.value))
else:
args.append(self.visit(arg))
fn = self.visit(node.func)
call_kwargs: dict[str, Any] = {}
for keyword in node.keywords:
@@ -388,6 +432,45 @@ def _eval_dispatch_expr(
return _DispatchExprEvaluator(kwargs, globals_).eval(node)
def _validate_dispatch_expr(node: ast.AST) -> None:
"""Enforce the dispatch AST subset before compiling traced code."""
allowed_nodes = (
ast.Name,
ast.Constant,
ast.Lambda,
ast.arguments,
ast.arg,
ast.IfExp,
ast.Tuple,
ast.List,
ast.BoolOp,
ast.And,
ast.Or,
ast.Compare,
*tuple(_CMP_OPS),
ast.UnaryOp,
ast.Not,
ast.USub,
ast.BinOp,
*tuple(_BIN_OPS),
ast.Call,
ast.keyword,
ast.Starred,
ast.Attribute,
ast.Subscript,
ast.Load,
)
for child in ast.walk(node):
if not isinstance(child, allowed_nodes):
raise _dispatch_expr_error(child, "Unsupported dispatch expression")
if isinstance(child, ast.Call) and any(
keyword.arg is None for keyword in child.keywords
):
raise _dispatch_expr_error(
child, "Dispatch helper calls cannot use **kwargs"
)
def _collect_input_names(
node: ast.AST,
candidate_names: set[str],
@@ -406,6 +489,23 @@ def _collect_input_names(
}
def _named_assignment(
statement: ast.stmt,
subject: str,
) -> tuple[str, ast.AST] | None:
match statement:
case ast.Assign(targets=[ast.Name(id=name)], value=value):
return name, value
case ast.AnnAssign(target=ast.Name(id=name), value=value) if value is not None:
return name, value
case ast.Assign() | ast.AnnAssign():
raise _dispatch_expr_error(
statement, f"{subject} assignments require one name"
)
case _:
return None
def _collect_expression_body(
fn: Callable[..., Any],
function_def: ast.FunctionDef | ast.Lambda,
@@ -422,37 +522,18 @@ def _collect_expression_body(
):
continue
if isinstance(statement, ast.Assign):
if len(statement.targets) != 1 or not isinstance(
statement.targets[0], ast.Name
):
raise _dispatch_expr_error(
statement,
"Traced warmup helper assignments must target one local name",
)
local_exprs.append((statement.targets[0].id, statement.value))
if (
assignment := _named_assignment(statement, "Traced warmup helper")
) is not None:
local_exprs.append(assignment)
continue
if isinstance(statement, ast.AnnAssign):
if statement.value is None:
raise _dispatch_expr_error(
statement, "Traced warmup helper annotations must assign a value"
)
if not isinstance(statement.target, ast.Name):
raise _dispatch_expr_error(
statement,
"Traced warmup helper assignments must target one local name",
)
local_exprs.append((statement.target.id, statement.value))
continue
if isinstance(statement, ast.Return) and statement.value is not None:
return local_exprs, statement.value
if isinstance(statement, ast.Return):
raise _dispatch_expr_error(
statement, "Traced warmup helper must return an expression"
)
if statement.value is None:
raise _dispatch_expr_error(
statement, "Traced warmup helper must return an expression"
)
return local_exprs, statement.value
raise _dispatch_expr_error(
statement,
@@ -574,10 +655,17 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
bind_launch_inputs = True
def __init__(self) -> None:
self._dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
dispatch = type(self).dispatch
self._dispatch_trace = (
None
if dispatch is VllmJitKernel.dispatch
else _trace_compile_key_dispatch(self.dispatch)
)
self._compiled_cache: dict[Any, Any] = {}
def compile_key(self, kwargs: Mapping[str, Any]) -> CompileKeyT:
if self._dispatch_trace is None:
raise TypeError(f"{type(self).__name__} does not define dispatch()")
return self._dispatch_trace.compile_key(self.CompileKey, kwargs)
def _get_or_compile(
@@ -683,7 +771,184 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
return traced
@abstractmethod
def _expand_warmup_cases(
self,
cases_fn: Callable[..., Any],
*args: Any,
_value_expander: Callable[[WarmupValues], tuple[Any, ...]] = (
_expand_warmup_values
),
**kwargs: Any,
) -> Iterator[Mapping[str, Any]]:
"""Expand symbolic domains declared inside a warmup-cases method."""
function_def = get_function_source_node(cases_fn)
if isinstance(function_def, ast.Lambda):
raise _dispatch_expr_error(
function_def, "Warmup cases must be a function definition"
)
signature = inspect.signature(cases_fn)
bound = signature.bind(*args, **kwargs)
bound.apply_defaults()
static_values = dict(bound.arguments)
bound_self = getattr(cases_fn, "__self__", None)
if bound_self is not None:
static_values["self"] = bound_self
globals_ = getattr(cases_fn, "__func__", cases_fn).__globals__
domains: list[tuple[str, tuple[Any, ...]]] = []
local_exprs: list[tuple[str, ast.AST]] = []
predicates: list[ast.AST] = []
return_expr: ast.AST | None = None
for statement in function_def.body:
if (
isinstance(statement, ast.Expr)
and isinstance(statement.value, ast.Constant)
and isinstance(statement.value.value, str)
):
continue
assignment = _named_assignment(statement, "Warmup case")
if assignment is not None:
name, value_expr = assignment
call_name = (
(get_ast_full_name(value_expr.func) or "").split(".")[-1]
if isinstance(value_expr, ast.Call)
else ""
)
if call_name in {"WarmupIntRange", "WarmupChoices"}:
value = _eval_dispatch_expr(value_expr, static_values, globals_)
domains.append((name, _value_expander(value)))
else:
local_exprs.append((name, value_expr))
continue
if (
isinstance(statement, ast.Expr)
and isinstance(statement.value, ast.Call)
and (get_ast_full_name(statement.value.func) or "").endswith("_when")
):
if len(statement.value.args) != 1 or statement.value.keywords:
raise _dispatch_expr_error(
statement, "_when requires exactly one positional expression"
)
predicates.append(statement.value.args[0])
continue
if isinstance(statement, ast.Return):
if return_expr is not None or statement.value is None:
raise _dispatch_expr_error(
statement, "Warmup cases require one final return expression"
)
return_expr = statement.value
continue
raise _dispatch_expr_error(
statement,
"AST-traced warmup cases support assignments, _when, and return",
)
if return_expr is None or not isinstance(return_expr, ast.Call):
raise ValueError("AST-traced warmup cases must return one case(...) call")
if any(keyword.arg is None for keyword in return_expr.keywords):
raise _dispatch_expr_error(
return_expr, "Warmup case expressions do not support **kwargs"
)
class _InlineDomainRewriter(ast.NodeTransformer):
def visit_Call(self, node: ast.Call) -> ast.AST:
call_name = (get_ast_full_name(node.func) or "").split(".")[-1]
if call_name not in {"WarmupIntRange", "WarmupChoices"}:
return self.generic_visit(node)
name = f"__warmup_domain_{len(domains)}"
domain = _eval_dispatch_expr(node, static_values, globals_)
domains.append((name, _value_expander(domain)))
return ast.copy_location(ast.Name(id=name, ctx=ast.Load()), node)
return_expr = cast(ast.Call, _InlineDomainRewriter().visit(return_expr))
domain_names = tuple(name for name, _ in domains)
dynamic_names = set(domain_names)
dynamic_local_exprs: list[tuple[str, ast.AST]] = []
for name, expr in local_exprs:
referenced_names = {
node.id for node in ast.walk(expr) if isinstance(node, ast.Name)
}
if referenced_names & dynamic_names:
dynamic_local_exprs.append((name, expr))
dynamic_names.add(name)
else:
static_values[name] = _eval_dispatch_expr(expr, static_values, globals_)
for _, expr in dynamic_local_exprs:
_validate_dispatch_expr(expr)
for predicate in predicates:
_validate_dispatch_expr(predicate)
_validate_dispatch_expr(return_expr)
case_body: list[ast.stmt] = [
ast.Assign(
targets=[ast.Name(id=name, ctx=ast.Store())],
value=cast(ast.expr, expr),
)
for name, expr in dynamic_local_exprs
]
if predicates:
predicate = (
predicates[0]
if len(predicates) == 1
else ast.BoolOp(
op=ast.And(),
values=[cast(ast.expr, value) for value in predicates],
)
)
case_body.append(
ast.If(
test=ast.UnaryOp(op=ast.Not(), operand=cast(ast.expr, predicate)),
body=[ast.Return(value=ast.Constant(value=None))],
orelse=[],
)
)
case_body.append(ast.Return(value=cast(ast.expr, return_expr)))
# FunctionDef fields vary across the supported Python versions, so no
# single constructor overload matches every mypy target.
case_function = ast.FunctionDef( # type: ignore[call-overload]
name="__vllm_warmup_case",
args=ast.arguments(
posonlyargs=[],
args=[ast.arg(arg=name) for name in domain_names],
kwonlyargs=[],
kw_defaults=[],
defaults=[],
),
body=case_body,
decorator_list=[],
returns=None,
type_comment=None,
)
case_globals = dict(globals_)
case_globals.update(static_values)
exec(
compile(
ast.fix_missing_locations(
ast.Module(body=[case_function], type_ignores=[])
),
"<jit-warmup>",
"exec",
),
case_globals,
)
evaluate_case = cast(Callable[..., Any], case_globals[case_function.name])
domain_values = tuple(values for _, values in domains)
for values in itertools.product(*domain_values):
case = evaluate_case(*values)
if case is None:
continue
if not isinstance(case, Mapping):
raise TypeError("AST-traced warmup cases must return a mapping")
if any(
isinstance(value, WarmupChoices | WarmupIntRange)
for value in case.values()
):
raise TypeError("Warmup domains must be direct expressions")
yield case
def dispatch(self, **kwargs: Any) -> CompileKeyT:
"""Build one compile key from one concrete dispatch point."""
raise NotImplementedError
@@ -751,6 +1016,19 @@ class JitWarmupRegistry:
list[tuple[tuple[Any, ...], dict[str, Any]]],
] = {}
@classmethod
def capture(cls, init_fn: Callable[..., None]) -> Callable[..., None]:
"""Collect warmup registrations made while the decorated callable runs."""
@wraps(init_fn)
def wrapped(instance: Any, vllm_config: Any, *args: Any, **kwargs: Any) -> None:
registry = cls(vllm_config)
instance.jit_warmup_registry = registry
with registry.activate():
init_fn(instance, vllm_config, *args, **kwargs)
return wrapped
@contextmanager
def activate(self) -> Iterator[None]:
"""Collect registrations made in this context."""
@@ -808,9 +1086,10 @@ class JitWarmupRegistry:
if (
not args
and not kwargs
and inspect.signature(kernel.get_warmup_keys).parameters
and "vllm_config"
in inspect.signature(kernel.get_warmup_keys).parameters
):
args = (self.vllm_config,)
kwargs = {"vllm_config": self.vllm_config}
for compile_key in kernel.get_warmup_keys(*args, **kwargs):
compile_keys[compile_key] = None
if compile_keys:
@@ -3,25 +3,130 @@
import ast
import inspect
from abc import abstractmethod
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import cached_property
from typing import Any, ClassVar, Generic, TypeVar, cast
from collections.abc import Callable, Hashable, Iterable, Mapping
from contextlib import nullcontext, suppress
from dataclasses import dataclass, field
from functools import cache, cached_property, update_wrapper
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
get_ast_full_name,
get_function_source_node,
)
from vllm.platforms import current_platform
from vllm.triton_utils import triton
CompileKeyT = TypeVar("CompileKeyT")
# ``(grid, launch_kwargs)`` or ``(grid, launch_kwargs, outputs)`` for
# self-allocating kernels. ``grid=None`` skips the launch (e.g. empty batch)
# but still returns the declared outputs.
P = ParamSpec("P")
# ``kernel_launcher`` also supports a ``None`` grid when no launch is needed.
LaunchSpec = (
tuple[tuple[int, ...] | None, dict[str, Any]]
| tuple[tuple[int, ...] | None, dict[str, Any], Any]
)
# Dispatchers select a concrete launch configuration for a Triton kernel.
DispatchSpec = (
tuple[tuple[int, ...], dict[str, Any]] | tuple[tuple[int, ...], dict[str, Any], Any]
)
@dataclass(frozen=True)
class _LaunchBindingPlan:
input_targets: tuple[tuple[int, str], ...]
stride_targets: tuple[tuple[str, str, int], ...]
@cache
def _launch_binding_plan(
arg_names: tuple[str, ...],
input_names: tuple[str, ...],
launch_names: tuple[str, ...],
) -> _LaunchBindingPlan:
arg_name_set = set(arg_names)
launch_name_set = set(launch_names)
def kernel_arg(input_name: str) -> str | None:
pointer_name = f"{input_name}_ptr"
candidates = (
input_name,
pointer_name,
input_name.upper(),
pointer_name.upper(),
)
return next(
(candidate for candidate in candidates if candidate in arg_name_set),
None,
)
input_targets = tuple(
(index, target)
for index, name in enumerate(input_names)
if (target := kernel_arg(name)) is not None and target not in launch_name_set
)
bound_names = launch_name_set | {target for _, target in input_targets}
stride_targets = []
for target in arg_names:
if target in bound_names:
continue
name, separator, dim = target.rpartition("_stride")
if not separator:
name, separator, dim = target.rpartition("_STRIDE")
dim = dim.removeprefix("_")
if separator and (not dim or dim.isdigit()):
pointer = f"{name}_PTR" if separator == "_STRIDE" else f"{name}_ptr"
source = name if name in bound_names else pointer
if source in bound_names:
stride_targets.append((target, source, int(dim or 0)))
return _LaunchBindingPlan(input_targets, tuple(stride_targets))
def triton_warmup_inputs(
kernel: Any,
*args: Any,
grid: tuple[int, ...],
pointer_dtypes: (
Mapping[Any, Iterable[str]] | Iterable[tuple[Any, Iterable[str]]] | None
) = None,
**kwargs: Any,
) -> dict[str, Any]:
"""Build launcher inputs from Triton's native positional argument order."""
arg_names = tuple(kernel.arg_names)
if len(args) > len(arg_names):
raise ValueError(
f"Received {len(args)} positional inputs for {len(arg_names)} "
"Triton kernel arguments"
)
inputs = dict(zip(arg_names, args))
duplicate_names = inputs.keys() & kwargs.keys()
if duplicate_names:
raise ValueError(
f"Triton inputs passed twice: {', '.join(sorted(duplicate_names))}"
)
inputs.update(kwargs)
if pointer_dtypes is not None:
function_def = get_function_source_node(kernel)
if not isinstance(function_def, ast.FunctionDef):
raise ValueError("Expected Triton kernel to be defined as a function")
pointer_names = _pointer_arg_names(function_def, arg_names)
groups = (
pointer_dtypes.items()
if isinstance(pointer_dtypes, Mapping)
else pointer_dtypes
)
for dtype, names in groups:
for name in names:
if name not in arg_names:
raise ValueError(f"Unknown Triton pointer argument: {name}")
if name not in pointer_names:
raise ValueError(f"Triton argument is not a pointer: {name}")
if name in inputs:
raise ValueError(f"Triton input passed twice: {name}")
inputs[name] = TritonWarmupTensor(dtype)
missing_pointers = pointer_names - inputs.keys()
if missing_pointers:
names = ", ".join(sorted(missing_pointers))
raise ValueError(f"Missing Triton pointer inputs: {names}")
return {"grid": grid, **inputs}
def triton_scalar_specialization_rep(value: int) -> int:
@@ -72,6 +177,7 @@ class TritonWarmupTensor:
aligned: bool = True
shape: tuple[int, ...] = (1,)
strides: tuple[int, ...] | None = None
init: Any = 0
def data_ptr(self) -> int:
return 0 if self.aligned else 1
@@ -79,6 +185,12 @@ class TritonWarmupTensor:
def ptr_range(self) -> int:
return 0
@property
def device(self) -> Any:
import torch
return torch.device(current_platform.device_type)
def stride(self, dim: int | None = None) -> int | tuple[int, ...]:
if self.strides is None:
strides: list[int] = []
@@ -95,9 +207,10 @@ class TritonWarmupTensor:
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""Triton owner whose runtime launch specification is reused for warmup."""
kernel: ClassVar[Any]
kernel: Any
_warming = False
_warming_compile_key: CompileKeyT | None = None
_run_autotune = False
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
@@ -126,6 +239,21 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
f"Cannot inspect kernel parameters for {type(self.kernel).__name__}"
)
def _prepare_launch_kwargs(
self,
input_names: tuple[str, ...],
input_values: tuple[Any, ...],
launch_kwargs: dict[str, Any],
) -> dict[str, Any]:
plan = _launch_binding_plan(
self._kernel_arg_names, input_names, tuple(launch_kwargs)
)
for index, target in plan.input_targets:
launch_kwargs[target] = input_values[index]
for target, source, dim in plan.stride_targets:
launch_kwargs[target] = launch_kwargs[source].stride(dim)
return launch_kwargs
def launch(
self,
launch_spec: LaunchSpec,
@@ -136,15 +264,20 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
else:
grid, launch_kwargs = launch_spec
outputs = None
kwargs = dict(launch_kwargs)
kwargs = self._prepare_launch_kwargs(
tuple(inputs), tuple(inputs.values()), dict(launch_kwargs)
)
kernel = kwargs.pop("kernel", None) or self.kernel
runtime_launcher = kwargs.pop("_runtime_launcher", None)
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
for name, value in inputs.items():
target = name if name in self._kernel_arg_names else f"{name}_ptr"
if target in self._kernel_arg_names and target not in kwargs:
kwargs[target] = value
if self._warming:
if self._run_autotune:
assert grid is not None
kernel[grid](**kwargs)
return outputs
kwargs = {
name: _triton_metadata_arg(value) for name, value in kwargs.items()
}
if (
self._warming_compile_key is not None
and "launch_pdl" in kwargs
@@ -168,6 +301,367 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
return outputs
@dataclass(frozen=True)
class TritonJitKey:
"""Process-local identity of one Triton JIT specialization."""
jit_function_id: int
jit_function_key: Hashable
device: Hashable
cache_key: Hashable
@dataclass(frozen=True)
class TritonCompileKey:
"""Triton-derived compile key with one non-comparing replay input."""
jit_keys: frozenset[TritonJitKey]
inputs: tuple[tuple[str, Any], ...] = field(compare=False, hash=False, repr=False)
def _triton_key_deriver(
kernel: Any,
) -> Callable[[Mapping[str, Any]], set[TritonJitKey]]:
"""Prepare Triton's key derivation once for one kernel and device."""
knobs = triton.knobs
Autotuner = triton.runtime.autotuner.Autotuner
Heuristics = triton.runtime.autotuner.Heuristics
driver = triton.runtime.driver
JITFunction = triton.runtime.jit.JITFunction
compute_cache_key = triton.runtime.jit.compute_cache_key
if isinstance(kernel, Heuristics):
derive_inner = _triton_key_deriver(kernel.fn)
def derive_heuristic(kwargs: Mapping[str, Any]) -> set[TritonJitKey]:
heuristic_kwargs = dict(kwargs)
for name, heuristic in kernel.values.items():
heuristic_kwargs[name] = heuristic(heuristic_kwargs)
return derive_inner(heuristic_kwargs)
return derive_heuristic
if isinstance(kernel, Autotuner):
derive_inner = _triton_key_deriver(kernel.fn)
def derive_autotuned(kwargs: Mapping[str, Any]) -> set[TritonJitKey]:
previous_nargs = getattr(kernel, "nargs", None)
kernel.nargs = {}
try:
configs = kernel.prune_configs(dict(kwargs))
finally:
kernel.nargs = previous_nargs
keys: set[TritonJitKey] = set()
for config in configs:
conflicts = kwargs.keys() & config.kwargs.keys()
if conflicts:
names = ", ".join(sorted(conflicts))
raise ValueError(f"Conflicting autotune parameters: {names}")
keys.update(derive_inner(dict(kwargs) | config.all_kwargs()))
return keys
return derive_autotuned
if not isinstance(kernel, JITFunction):
raise TypeError(f"Unsupported Triton kernel wrapper: {type(kernel).__name__}")
device = cast(Hashable, driver.active.get_current_device())
_, kernel_key_cache, _, _, binder = kernel.device_caches[device]
jit_function_id = id(kernel)
jit_function_key = cast(Hashable, kernel.cache_key)
debug = kernel.debug or knobs.runtime.debug
instrumentation_mode = knobs.compilation.instrumentation_mode
fpsan_homomorphic_casts = getattr(
knobs.compilation, "fpsan_homomorphic_casts", None
)
inspection_hook = knobs.runtime.add_stages_inspection_hook
def derive_jit(kwargs: Mapping[str, Any]) -> set[TritonJitKey]:
binder_kwargs = dict(kwargs)
binder_kwargs["debug"] = binder_kwargs.get("debug", debug) or debug
binder_kwargs["instrumentation_mode"] = instrumentation_mode
if fpsan_homomorphic_casts is not None:
binder_kwargs["fpsan_homomorphic_casts"] = fpsan_homomorphic_casts
_, specialization, options = binder(**binder_kwargs)
if inspection_hook is not None:
_, inspection_hash = inspection_hook()
specialization.append(f'("custom_pipeline", {inspection_hash})')
cache_key = cast(
Hashable,
compute_cache_key(kernel_key_cache, specialization, options),
)
return {
TritonJitKey(
jit_function_id,
jit_function_key,
device,
cache_key,
)
}
return derive_jit
WarmupCases = Mapping[str, Any] | Iterable[Mapping[str, Any]]
class _AutomaticTritonJitKernel(VllmTritonJitKernel[TritonCompileKey]):
def warmup_inputs(self, compile_key: TritonCompileKey) -> dict[str, Any]:
return dict(compile_key.inputs)
def _provider_cases(
self,
provider: Callable[..., WarmupCases],
*args: Any,
**kwargs: Any,
) -> Iterable[Mapping[str, Any]]:
cases_node = get_function_source_node(provider)
if any(
isinstance(node, ast.Call)
and (get_ast_full_name(node.func) or "").split(".")[-1]
in {"WarmupIntRange", "WarmupChoices", "_when"}
for node in ast.walk(cases_node)
):
return self._expand_warmup_cases(provider, *args, **kwargs)
cases = provider(*args, **kwargs)
return (cases,) if isinstance(cases, Mapping) else cases
def _is_autotuned(kernel: Any) -> bool:
Autotuner = triton.runtime.autotuner.Autotuner
while kernel is not None:
if isinstance(kernel, Autotuner):
return True
kernel = getattr(kernel, "fn", None)
return False
def _triton_metadata_arg(value: Any) -> Any:
from torch._subclasses.fake_tensor import FakeTensor
if not isinstance(value, FakeTensor):
return value
return TritonWarmupTensor(
value.dtype,
aligned=getattr(value, "_vllm_warmup_aligned", True),
shape=tuple(value.shape),
strides=tuple(value.stride()),
)
def _materialize_warmup_case(
case: Mapping[str, Any],
*,
real: bool,
mode: Any,
cache: dict[TritonWarmupTensor, Any],
) -> dict[str, Any]:
import torch
def materialize(value: Any) -> Any:
if not isinstance(value, TritonWarmupTensor):
return value
if not real:
try:
return cache[value]
except KeyError:
pass
except TypeError:
# Unhashable initializers are uncommon and safe to materialize directly.
pass
strides = cast(tuple[int, ...], value.stride())
with mode:
tensor = torch.empty_strided(
value.shape,
strides,
dtype=value.dtype,
device=value.device,
)
if real:
if callable(value.init):
value.init(tensor)
else:
tensor.fill_(value.init)
else:
tensor._vllm_warmup_aligned = value.aligned
with suppress(TypeError):
cache[value] = tensor
return tensor
return {name: materialize(value) for name, value in case.items()}
class _DecoratedTritonJitKernel(_AutomaticTritonJitKernel):
def __init__(
self,
kernel: Any,
warmup_inputs: Callable[..., WarmupCases],
dispatch: Callable[..., DispatchSpec] | None,
) -> None:
self.kernel = kernel
self._run_autotune = _is_autotuned(kernel)
self._warmup_inputs_fn = warmup_inputs
self._dispatch_fn = dispatch
self._dispatch_arg_names: tuple[str, ...] = ()
self._dispatch_defaults: dict[str, Any] = {}
if dispatch is not None:
parameters = tuple(inspect.signature(dispatch).parameters.values())
if any(
parameter.kind
in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
for parameter in parameters
):
raise TypeError("Triton dispatchers require explicit parameters")
self._dispatch_arg_names = tuple(parameter.name for parameter in parameters)
self._dispatch_defaults = {
parameter.name: parameter.default
for parameter in parameters
if parameter.default is not inspect.Parameter.empty
}
super().__init__()
def __call__(self, *args: Any, **kwargs: Any) -> Any:
if self._dispatch_fn is None:
grid = kwargs.pop("grid") if not args else args[0]
kernel_args = () if not args else args[1:]
inputs = triton_warmup_inputs(
self.kernel, *kernel_args, grid=grid, **kwargs
)
inputs.pop("grid")
return self.launch((grid, inputs), {})
spec = self._dispatch_fn(*args, **kwargs)
input_values = args
if len(args) != len(self._dispatch_arg_names):
input_values += tuple(
kwargs[name] if name in kwargs else self._dispatch_defaults[name]
for name in self._dispatch_arg_names[len(args) :]
)
return self.launch(
spec,
dict(zip(self._dispatch_arg_names, input_values)),
)
def __getitem__(self, grid: tuple[int, ...]) -> Callable[..., Any]:
return cast(Callable[..., Any], self.kernel[grid])
def __getattr__(self, name: str) -> Any:
return getattr(self.kernel, name)
def get_warmup_keys(
self, *, vllm_config: Any | None = None, **kwargs: Any
) -> list[TritonCompileKey]:
keys: list[TritonCompileKey] = []
seen_jit_keys: set[frozenset[TritonJitKey]] = set()
if "vllm_config" in inspect.signature(self._warmup_inputs_fn).parameters:
kwargs.setdefault("vllm_config", vllm_config)
cases = self._provider_cases(self._warmup_inputs_fn, **kwargs)
derive_keys = _triton_key_deriver(self.kernel)
materialize = self._run_autotune
mode: Any = nullcontext() if self._run_autotune else None
materialized: dict[TritonWarmupTensor, Any] = {}
for case in cases:
inputs = (
_materialize_warmup_case(
case,
real=self._run_autotune,
mode=mode,
cache=materialized,
)
if materialize
else case
)
if self._dispatch_fn is None:
prepared = dict(inputs)
prepared.pop("grid")
else:
dispatch_inputs = {
name: inputs[name]
for name in self._dispatch_arg_names
if name in inputs
}
try:
_, launch_kwargs = self._dispatch_fn(**dispatch_inputs)[:2]
except (AssertionError, AttributeError, TypeError):
if materialize:
raise
from torch._subclasses.fake_tensor import FakeTensorMode
materialize = True
mode = FakeTensorMode()
inputs = _materialize_warmup_case(
case,
real=False,
mode=mode,
cache=materialized,
)
dispatch_inputs = {
name: inputs[name]
for name in self._dispatch_arg_names
if name in inputs
}
_, launch_kwargs = self._dispatch_fn(**dispatch_inputs)[:2]
prepared = self._prepare_launch_kwargs(
tuple(inputs),
tuple(inputs.values()),
dict(launch_kwargs),
)
if materialize:
prepared = {
name: _triton_metadata_arg(value)
for name, value in prepared.items()
}
jit_keys = frozenset(derive_keys(prepared))
if jit_keys and jit_keys not in seen_jit_keys:
seen_jit_keys.add(jit_keys)
keys.append(TritonCompileKey(jit_keys, tuple(inputs.items())))
return keys
class TritonKernelDispatcher(Protocol[P]):
"""Callable Triton launcher with compile-time warmup support."""
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> Any: ...
def register_warmup(self, *args: Any, **kwargs: Any) -> None: ...
@overload
def triton_kernel_dispatcher_with_warmup(
*,
warmup_inputs: Callable[..., WarmupCases],
kernel: None = None,
) -> Callable[[Any], Any]: ...
@overload
def triton_kernel_dispatcher_with_warmup(
*,
warmup_inputs: Callable[..., WarmupCases],
kernel: Any,
) -> Callable[[Callable[P, DispatchSpec]], TritonKernelDispatcher[P]]: ...
def triton_kernel_dispatcher_with_warmup(
*,
warmup_inputs: Callable[..., WarmupCases],
kernel: Any | None = None,
) -> Callable[[Any], Any]:
"""Decorate a dispatch function or a native Triton kernel for warmup."""
def decorate(dispatch: Any) -> Any:
owner = _DecoratedTritonJitKernel(
dispatch if kernel is None else kernel,
warmup_inputs,
None if kernel is None else dispatch,
)
update_wrapper(owner, dispatch, updated=())
return owner
return decorate
@dataclass(frozen=True)
class TritonPointerInputVariant:
# Named pointer-alignment variant for compile-only Triton warmup.
+4 -5
View File
@@ -41,9 +41,6 @@ from vllm.model_executor.warmup.qwen_vl_triton_warmup import qwen_vl_triton_warm
from vllm.model_executor.warmup.replayssm_warmup import (
replayssm_autotune_warmup,
)
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
spec_decode_rejection_warmup,
)
from vllm.model_executor.warmup.watermark_sample_warmup import (
watermark_sample_warmup,
)
@@ -178,7 +175,10 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
logger.info("JIT kernel warmup starting.")
jit_warmup_start = time.perf_counter()
try:
worker.model_runner.jit_warmup_registry.warmup()
registry = (
worker.model_runner.jit_warmup_registry # type: ignore[attr-defined]
)
registry.warmup()
except Exception:
logger.exception(
"JIT kernel warmup failed after %.2fs.",
@@ -209,7 +209,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
# Run next so input-prep kernels JIT against pristine runner state.
if enable_jit_warmup:
kimi_k3_triton_warmup(worker)
spec_decode_rejection_warmup(worker)
watermark_sample_warmup(worker)
qwen4_exp_qsa_triton_warmup(worker)
@@ -1,139 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up speculative-decoding rejection kernels."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import torch
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.v1.worker.gpu_worker import Worker
logger = init_logger(__name__)
@torch.inference_mode()
def spec_decode_rejection_warmup(worker: Worker) -> None:
spec_config = worker.vllm_config.speculative_config
if spec_config is None:
return
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
rejection_sample,
)
model_config = worker.vllm_config.model_config
vocab_size = model_config.get_vocab_size()
num_spec = spec_config.num_speculative_tokens
if num_spec <= 0 or vocab_size <= 0:
return
rejection_method = getattr(spec_config, "rejection_sample_method", None)
use_block_verification = rejection_method == "block"
use_synthetic = rejection_method == "synthetic"
# HAS_DRAFT_LOGITS is a constexpr, so the warmup must match the runtime.
use_draft_logits = (
getattr(spec_config, "draft_sample_method", "greedy") == "probabilistic"
)
device = worker.device
num_reqs = 1
tokens_per_req = num_spec + 1
num_logits = num_reqs * tokens_per_req
watermark_config = getattr(worker.vllm_config, "watermark_config", None)
watermark_kwargs: dict[str, Any] = {}
if watermark_config is not None:
from vllm.v1.watermarking.spec_decode import speculative_target_watermark_key
try:
watermark_key = speculative_target_watermark_key(watermark_config)
except Exception:
logger.warning(
"Skipping spec-decode rejection sampler warmup: could not derive "
"the watermark recovery key.",
exc_info=True,
)
return
# Tensor dtypes and integer widths affect Triton specialization.
watermark_kwargs = {
"contexts": torch.zeros(
(num_logits, watermark_config.context_width),
dtype=torch.int32,
device=device,
),
"watermarking": torch.zeros(num_reqs, dtype=torch.bool, device=device),
"watermark_key": watermark_key,
}
# Sampling-parameter processing may promote either distribution to fp32.
model_dtype = model_config.dtype
warmup_dtype_pairs: set[tuple[torch.dtype, torch.dtype | None]]
if use_draft_logits:
warmup_dtype_pairs = {
(model_dtype, model_dtype),
(torch.float32, torch.float32),
(torch.float32, model_dtype),
(model_dtype, torch.float32),
}
else:
warmup_dtype_pairs = {(model_dtype, None), (torch.float32, None)}
logger.info(
"Warming up spec-decode rejection sampler kernels "
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s, watermark=%s).",
vocab_size,
num_spec,
[(str(t), str(d)) for t, d in warmup_dtype_pairs],
use_block_verification,
watermark_config is not None,
)
for tgt_dtype, draft_dtype in warmup_dtype_pairs:
target_logits = torch.zeros(
(num_logits, vocab_size), dtype=tgt_dtype, device=device
)
draft_logits = (
torch.zeros(
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
)
if draft_dtype is not None
else None
)
synthetic_rates = (
torch.full((num_spec,), 0.5, dtype=torch.float32, device=device)
if use_synthetic
else None
)
try:
rejection_sample(
target_logits=target_logits,
draft_logits=draft_logits,
draft_sampled=torch.zeros(num_logits, dtype=torch.int32, device=device),
cu_num_logits=torch.tensor(
[0, num_logits], dtype=torch.int32, device=device
),
pos=torch.zeros(num_logits, dtype=torch.int64, device=device),
idx_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
expanded_idx_mapping=torch.zeros(
num_logits, dtype=torch.int64, device=device
),
expanded_local_pos=torch.arange(
num_logits, dtype=torch.int32, device=device
),
temperature=torch.zeros(num_reqs, dtype=torch.float32, device=device),
seed=torch.full((num_reqs,), 42, dtype=torch.int64, device=device),
num_speculative_steps=num_spec,
synthetic_conditional_rates=synthetic_rates,
use_fp64=False,
use_block_verification=use_block_verification,
**watermark_kwargs,
)
except Exception:
logger.warning(
"Skipping spec-decode rejection sampler warmup.", exc_info=True
)
return