[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
OpenAI Codex
parent
e19a3e172e
commit
aed894c190
@@ -1,8 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
DispatchSpec,
|
||||
TritonWarmupTensor,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@@ -33,11 +41,39 @@ def _scatter_states_kernel(
|
||||
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def _scatter_states_warmup_inputs(
|
||||
*,
|
||||
state_shape: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
indices_dtype: torch.dtype,
|
||||
max_num_tokens: int,
|
||||
) -> dict[str, object]:
|
||||
num_tokens: Any = WarmupIntRange(1, max_num_tokens + 1)
|
||||
return dict(
|
||||
state=TritonWarmupTensor(
|
||||
state_dtype,
|
||||
shape=(1,) + state_shape,
|
||||
),
|
||||
src=TritonWarmupTensor(
|
||||
state_dtype,
|
||||
shape=(num_tokens,) + state_shape,
|
||||
),
|
||||
indices=TritonWarmupTensor(
|
||||
indices_dtype,
|
||||
shape=(num_tokens,),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_scatter_states_kernel,
|
||||
warmup_inputs=_scatter_states_warmup_inputs,
|
||||
)
|
||||
def scatter_states(
|
||||
state: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> None:
|
||||
) -> DispatchSpec:
|
||||
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
|
||||
|
||||
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
|
||||
@@ -58,13 +94,10 @@ def scatter_states(
|
||||
assert src[0].is_contiguous()
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_scatter_states_kernel[grid](
|
||||
state,
|
||||
src,
|
||||
indices,
|
||||
state.stride(0),
|
||||
src.stride(0),
|
||||
indices.stride(0),
|
||||
return grid, dict(
|
||||
stride_state_batch=state.stride(0),
|
||||
stride_src_batch=src.stride(0),
|
||||
stride_indices=indices.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
|
||||
@@ -21,6 +21,7 @@ from typing import Any, Generic, TypeVar, cast
|
||||
__all__ = [
|
||||
"JitWarmupRegistry",
|
||||
"VllmJitKernel",
|
||||
"WarmupChoices",
|
||||
"WarmupIntRange",
|
||||
"get_ast_full_name",
|
||||
"get_function_source_node",
|
||||
@@ -62,6 +63,21 @@ class WarmupIntRange:
|
||||
advance: Callable[[int], int] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WarmupChoices:
|
||||
"""Expand an explicit finite set of values in a traced warmup method."""
|
||||
|
||||
values: tuple[Any, ...]
|
||||
|
||||
def __init__(self, *values: Any) -> None:
|
||||
object.__setattr__(self, "values", values)
|
||||
|
||||
|
||||
def _when(condition: bool) -> None:
|
||||
"""Filter symbolic cases in an AST-traced warmup-cases method."""
|
||||
raise RuntimeError("_when() is only valid in an AST-traced warmup method")
|
||||
|
||||
|
||||
WarmupValues = Any
|
||||
CompileKeyDispatchFn = Callable[..., CompileKeyT]
|
||||
WarmupPredicateFn = Callable[..., bool]
|
||||
@@ -87,6 +103,8 @@ class _WarmupInputRows:
|
||||
|
||||
|
||||
def _expand_warmup_values(values: WarmupValues) -> tuple[Any, ...]:
|
||||
if isinstance(values, WarmupChoices):
|
||||
return values.values
|
||||
if isinstance(values, WarmupIntRange):
|
||||
if values.advance is None:
|
||||
return tuple(range(values.start, values.stop, values.step))
|
||||
@@ -278,6 +296,27 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
|
||||
def visit_Constant(self, node: ast.Constant) -> Any:
|
||||
return node.value
|
||||
|
||||
def visit_Lambda(self, node: ast.Lambda) -> Callable[..., Any]:
|
||||
arguments = node.args
|
||||
if (
|
||||
arguments.posonlyargs
|
||||
or arguments.vararg is not None
|
||||
or arguments.kwonlyargs
|
||||
or arguments.kwarg is not None
|
||||
or arguments.defaults
|
||||
):
|
||||
raise _dispatch_expr_error(node, "Traced lambdas require positional args")
|
||||
names = tuple(argument.arg for argument in arguments.args)
|
||||
|
||||
def evaluate(*values: Any) -> Any:
|
||||
if len(values) != len(names):
|
||||
raise TypeError(f"Expected {len(names)} lambda arguments")
|
||||
return type(self)(
|
||||
dict(self.values) | dict(zip(names, values)), self.globals
|
||||
).eval(node.body)
|
||||
|
||||
return evaluate
|
||||
|
||||
def visit_IfExp(self, node: ast.IfExp) -> Any:
|
||||
return self.visit(node.body if self.visit(node.test) else node.orelse)
|
||||
|
||||
@@ -333,7 +372,12 @@ class _DispatchExprEvaluator(ast.NodeVisitor):
|
||||
return op(self.visit(node.left), self.visit(node.right))
|
||||
|
||||
def visit_Call(self, node: ast.Call) -> Any:
|
||||
args = [self.visit(arg) for arg in node.args]
|
||||
args: list[Any] = []
|
||||
for arg in node.args:
|
||||
if isinstance(arg, ast.Starred):
|
||||
args.extend(self.visit(arg.value))
|
||||
else:
|
||||
args.append(self.visit(arg))
|
||||
fn = self.visit(node.func)
|
||||
call_kwargs: dict[str, Any] = {}
|
||||
for keyword in node.keywords:
|
||||
@@ -388,6 +432,45 @@ def _eval_dispatch_expr(
|
||||
return _DispatchExprEvaluator(kwargs, globals_).eval(node)
|
||||
|
||||
|
||||
def _validate_dispatch_expr(node: ast.AST) -> None:
|
||||
"""Enforce the dispatch AST subset before compiling traced code."""
|
||||
allowed_nodes = (
|
||||
ast.Name,
|
||||
ast.Constant,
|
||||
ast.Lambda,
|
||||
ast.arguments,
|
||||
ast.arg,
|
||||
ast.IfExp,
|
||||
ast.Tuple,
|
||||
ast.List,
|
||||
ast.BoolOp,
|
||||
ast.And,
|
||||
ast.Or,
|
||||
ast.Compare,
|
||||
*tuple(_CMP_OPS),
|
||||
ast.UnaryOp,
|
||||
ast.Not,
|
||||
ast.USub,
|
||||
ast.BinOp,
|
||||
*tuple(_BIN_OPS),
|
||||
ast.Call,
|
||||
ast.keyword,
|
||||
ast.Starred,
|
||||
ast.Attribute,
|
||||
ast.Subscript,
|
||||
ast.Load,
|
||||
)
|
||||
for child in ast.walk(node):
|
||||
if not isinstance(child, allowed_nodes):
|
||||
raise _dispatch_expr_error(child, "Unsupported dispatch expression")
|
||||
if isinstance(child, ast.Call) and any(
|
||||
keyword.arg is None for keyword in child.keywords
|
||||
):
|
||||
raise _dispatch_expr_error(
|
||||
child, "Dispatch helper calls cannot use **kwargs"
|
||||
)
|
||||
|
||||
|
||||
def _collect_input_names(
|
||||
node: ast.AST,
|
||||
candidate_names: set[str],
|
||||
@@ -406,6 +489,23 @@ def _collect_input_names(
|
||||
}
|
||||
|
||||
|
||||
def _named_assignment(
|
||||
statement: ast.stmt,
|
||||
subject: str,
|
||||
) -> tuple[str, ast.AST] | None:
|
||||
match statement:
|
||||
case ast.Assign(targets=[ast.Name(id=name)], value=value):
|
||||
return name, value
|
||||
case ast.AnnAssign(target=ast.Name(id=name), value=value) if value is not None:
|
||||
return name, value
|
||||
case ast.Assign() | ast.AnnAssign():
|
||||
raise _dispatch_expr_error(
|
||||
statement, f"{subject} assignments require one name"
|
||||
)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _collect_expression_body(
|
||||
fn: Callable[..., Any],
|
||||
function_def: ast.FunctionDef | ast.Lambda,
|
||||
@@ -422,37 +522,18 @@ def _collect_expression_body(
|
||||
):
|
||||
continue
|
||||
|
||||
if isinstance(statement, ast.Assign):
|
||||
if len(statement.targets) != 1 or not isinstance(
|
||||
statement.targets[0], ast.Name
|
||||
):
|
||||
raise _dispatch_expr_error(
|
||||
statement,
|
||||
"Traced warmup helper assignments must target one local name",
|
||||
)
|
||||
local_exprs.append((statement.targets[0].id, statement.value))
|
||||
if (
|
||||
assignment := _named_assignment(statement, "Traced warmup helper")
|
||||
) is not None:
|
||||
local_exprs.append(assignment)
|
||||
continue
|
||||
|
||||
if isinstance(statement, ast.AnnAssign):
|
||||
if statement.value is None:
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Traced warmup helper annotations must assign a value"
|
||||
)
|
||||
if not isinstance(statement.target, ast.Name):
|
||||
raise _dispatch_expr_error(
|
||||
statement,
|
||||
"Traced warmup helper assignments must target one local name",
|
||||
)
|
||||
local_exprs.append((statement.target.id, statement.value))
|
||||
continue
|
||||
|
||||
if isinstance(statement, ast.Return) and statement.value is not None:
|
||||
return local_exprs, statement.value
|
||||
|
||||
if isinstance(statement, ast.Return):
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Traced warmup helper must return an expression"
|
||||
)
|
||||
if statement.value is None:
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Traced warmup helper must return an expression"
|
||||
)
|
||||
return local_exprs, statement.value
|
||||
|
||||
raise _dispatch_expr_error(
|
||||
statement,
|
||||
@@ -574,10 +655,17 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
bind_launch_inputs = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
|
||||
dispatch = type(self).dispatch
|
||||
self._dispatch_trace = (
|
||||
None
|
||||
if dispatch is VllmJitKernel.dispatch
|
||||
else _trace_compile_key_dispatch(self.dispatch)
|
||||
)
|
||||
self._compiled_cache: dict[Any, Any] = {}
|
||||
|
||||
def compile_key(self, kwargs: Mapping[str, Any]) -> CompileKeyT:
|
||||
if self._dispatch_trace is None:
|
||||
raise TypeError(f"{type(self).__name__} does not define dispatch()")
|
||||
return self._dispatch_trace.compile_key(self.CompileKey, kwargs)
|
||||
|
||||
def _get_or_compile(
|
||||
@@ -683,7 +771,184 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
|
||||
return traced
|
||||
|
||||
@abstractmethod
|
||||
def _expand_warmup_cases(
|
||||
self,
|
||||
cases_fn: Callable[..., Any],
|
||||
*args: Any,
|
||||
_value_expander: Callable[[WarmupValues], tuple[Any, ...]] = (
|
||||
_expand_warmup_values
|
||||
),
|
||||
**kwargs: Any,
|
||||
) -> Iterator[Mapping[str, Any]]:
|
||||
"""Expand symbolic domains declared inside a warmup-cases method."""
|
||||
function_def = get_function_source_node(cases_fn)
|
||||
if isinstance(function_def, ast.Lambda):
|
||||
raise _dispatch_expr_error(
|
||||
function_def, "Warmup cases must be a function definition"
|
||||
)
|
||||
signature = inspect.signature(cases_fn)
|
||||
bound = signature.bind(*args, **kwargs)
|
||||
bound.apply_defaults()
|
||||
static_values = dict(bound.arguments)
|
||||
bound_self = getattr(cases_fn, "__self__", None)
|
||||
if bound_self is not None:
|
||||
static_values["self"] = bound_self
|
||||
globals_ = getattr(cases_fn, "__func__", cases_fn).__globals__
|
||||
|
||||
domains: list[tuple[str, tuple[Any, ...]]] = []
|
||||
local_exprs: list[tuple[str, ast.AST]] = []
|
||||
predicates: list[ast.AST] = []
|
||||
return_expr: ast.AST | None = None
|
||||
for statement in function_def.body:
|
||||
if (
|
||||
isinstance(statement, ast.Expr)
|
||||
and isinstance(statement.value, ast.Constant)
|
||||
and isinstance(statement.value.value, str)
|
||||
):
|
||||
continue
|
||||
assignment = _named_assignment(statement, "Warmup case")
|
||||
if assignment is not None:
|
||||
name, value_expr = assignment
|
||||
call_name = (
|
||||
(get_ast_full_name(value_expr.func) or "").split(".")[-1]
|
||||
if isinstance(value_expr, ast.Call)
|
||||
else ""
|
||||
)
|
||||
if call_name in {"WarmupIntRange", "WarmupChoices"}:
|
||||
value = _eval_dispatch_expr(value_expr, static_values, globals_)
|
||||
domains.append((name, _value_expander(value)))
|
||||
else:
|
||||
local_exprs.append((name, value_expr))
|
||||
continue
|
||||
if (
|
||||
isinstance(statement, ast.Expr)
|
||||
and isinstance(statement.value, ast.Call)
|
||||
and (get_ast_full_name(statement.value.func) or "").endswith("_when")
|
||||
):
|
||||
if len(statement.value.args) != 1 or statement.value.keywords:
|
||||
raise _dispatch_expr_error(
|
||||
statement, "_when requires exactly one positional expression"
|
||||
)
|
||||
predicates.append(statement.value.args[0])
|
||||
continue
|
||||
if isinstance(statement, ast.Return):
|
||||
if return_expr is not None or statement.value is None:
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Warmup cases require one final return expression"
|
||||
)
|
||||
return_expr = statement.value
|
||||
continue
|
||||
raise _dispatch_expr_error(
|
||||
statement,
|
||||
"AST-traced warmup cases support assignments, _when, and return",
|
||||
)
|
||||
|
||||
if return_expr is None or not isinstance(return_expr, ast.Call):
|
||||
raise ValueError("AST-traced warmup cases must return one case(...) call")
|
||||
if any(keyword.arg is None for keyword in return_expr.keywords):
|
||||
raise _dispatch_expr_error(
|
||||
return_expr, "Warmup case expressions do not support **kwargs"
|
||||
)
|
||||
|
||||
class _InlineDomainRewriter(ast.NodeTransformer):
|
||||
def visit_Call(self, node: ast.Call) -> ast.AST:
|
||||
call_name = (get_ast_full_name(node.func) or "").split(".")[-1]
|
||||
if call_name not in {"WarmupIntRange", "WarmupChoices"}:
|
||||
return self.generic_visit(node)
|
||||
name = f"__warmup_domain_{len(domains)}"
|
||||
domain = _eval_dispatch_expr(node, static_values, globals_)
|
||||
domains.append((name, _value_expander(domain)))
|
||||
return ast.copy_location(ast.Name(id=name, ctx=ast.Load()), node)
|
||||
|
||||
return_expr = cast(ast.Call, _InlineDomainRewriter().visit(return_expr))
|
||||
|
||||
domain_names = tuple(name for name, _ in domains)
|
||||
dynamic_names = set(domain_names)
|
||||
dynamic_local_exprs: list[tuple[str, ast.AST]] = []
|
||||
for name, expr in local_exprs:
|
||||
referenced_names = {
|
||||
node.id for node in ast.walk(expr) if isinstance(node, ast.Name)
|
||||
}
|
||||
if referenced_names & dynamic_names:
|
||||
dynamic_local_exprs.append((name, expr))
|
||||
dynamic_names.add(name)
|
||||
else:
|
||||
static_values[name] = _eval_dispatch_expr(expr, static_values, globals_)
|
||||
|
||||
for _, expr in dynamic_local_exprs:
|
||||
_validate_dispatch_expr(expr)
|
||||
for predicate in predicates:
|
||||
_validate_dispatch_expr(predicate)
|
||||
_validate_dispatch_expr(return_expr)
|
||||
|
||||
case_body: list[ast.stmt] = [
|
||||
ast.Assign(
|
||||
targets=[ast.Name(id=name, ctx=ast.Store())],
|
||||
value=cast(ast.expr, expr),
|
||||
)
|
||||
for name, expr in dynamic_local_exprs
|
||||
]
|
||||
if predicates:
|
||||
predicate = (
|
||||
predicates[0]
|
||||
if len(predicates) == 1
|
||||
else ast.BoolOp(
|
||||
op=ast.And(),
|
||||
values=[cast(ast.expr, value) for value in predicates],
|
||||
)
|
||||
)
|
||||
case_body.append(
|
||||
ast.If(
|
||||
test=ast.UnaryOp(op=ast.Not(), operand=cast(ast.expr, predicate)),
|
||||
body=[ast.Return(value=ast.Constant(value=None))],
|
||||
orelse=[],
|
||||
)
|
||||
)
|
||||
case_body.append(ast.Return(value=cast(ast.expr, return_expr)))
|
||||
# FunctionDef fields vary across the supported Python versions, so no
|
||||
# single constructor overload matches every mypy target.
|
||||
case_function = ast.FunctionDef( # type: ignore[call-overload]
|
||||
name="__vllm_warmup_case",
|
||||
args=ast.arguments(
|
||||
posonlyargs=[],
|
||||
args=[ast.arg(arg=name) for name in domain_names],
|
||||
kwonlyargs=[],
|
||||
kw_defaults=[],
|
||||
defaults=[],
|
||||
),
|
||||
body=case_body,
|
||||
decorator_list=[],
|
||||
returns=None,
|
||||
type_comment=None,
|
||||
)
|
||||
case_globals = dict(globals_)
|
||||
case_globals.update(static_values)
|
||||
exec(
|
||||
compile(
|
||||
ast.fix_missing_locations(
|
||||
ast.Module(body=[case_function], type_ignores=[])
|
||||
),
|
||||
"<jit-warmup>",
|
||||
"exec",
|
||||
),
|
||||
case_globals,
|
||||
)
|
||||
evaluate_case = cast(Callable[..., Any], case_globals[case_function.name])
|
||||
|
||||
domain_values = tuple(values for _, values in domains)
|
||||
for values in itertools.product(*domain_values):
|
||||
case = evaluate_case(*values)
|
||||
if case is None:
|
||||
continue
|
||||
if not isinstance(case, Mapping):
|
||||
raise TypeError("AST-traced warmup cases must return a mapping")
|
||||
if any(
|
||||
isinstance(value, WarmupChoices | WarmupIntRange)
|
||||
for value in case.values()
|
||||
):
|
||||
raise TypeError("Warmup domains must be direct expressions")
|
||||
yield case
|
||||
|
||||
def dispatch(self, **kwargs: Any) -> CompileKeyT:
|
||||
"""Build one compile key from one concrete dispatch point."""
|
||||
raise NotImplementedError
|
||||
@@ -751,6 +1016,19 @@ class JitWarmupRegistry:
|
||||
list[tuple[tuple[Any, ...], dict[str, Any]]],
|
||||
] = {}
|
||||
|
||||
@classmethod
|
||||
def capture(cls, init_fn: Callable[..., None]) -> Callable[..., None]:
|
||||
"""Collect warmup registrations made while the decorated callable runs."""
|
||||
|
||||
@wraps(init_fn)
|
||||
def wrapped(instance: Any, vllm_config: Any, *args: Any, **kwargs: Any) -> None:
|
||||
registry = cls(vllm_config)
|
||||
instance.jit_warmup_registry = registry
|
||||
with registry.activate():
|
||||
init_fn(instance, vllm_config, *args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
@contextmanager
|
||||
def activate(self) -> Iterator[None]:
|
||||
"""Collect registrations made in this context."""
|
||||
@@ -808,9 +1086,10 @@ class JitWarmupRegistry:
|
||||
if (
|
||||
not args
|
||||
and not kwargs
|
||||
and inspect.signature(kernel.get_warmup_keys).parameters
|
||||
and "vllm_config"
|
||||
in inspect.signature(kernel.get_warmup_keys).parameters
|
||||
):
|
||||
args = (self.vllm_config,)
|
||||
kwargs = {"vllm_config": self.vllm_config}
|
||||
for compile_key in kernel.get_warmup_keys(*args, **kwargs):
|
||||
compile_keys[compile_key] = None
|
||||
if compile_keys:
|
||||
|
||||
@@ -3,25 +3,130 @@
|
||||
import ast
|
||||
import inspect
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from typing import Any, ClassVar, Generic, TypeVar, cast
|
||||
from collections.abc import Callable, Hashable, Iterable, Mapping
|
||||
from contextlib import nullcontext, suppress
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache, cached_property, update_wrapper
|
||||
from typing import Any, Generic, ParamSpec, Protocol, TypeVar, cast, overload
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
VllmJitKernel,
|
||||
get_ast_full_name,
|
||||
get_function_source_node,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
# ``(grid, launch_kwargs)`` or ``(grid, launch_kwargs, outputs)`` for
|
||||
# self-allocating kernels. ``grid=None`` skips the launch (e.g. empty batch)
|
||||
# but still returns the declared outputs.
|
||||
P = ParamSpec("P")
|
||||
# ``kernel_launcher`` also supports a ``None`` grid when no launch is needed.
|
||||
LaunchSpec = (
|
||||
tuple[tuple[int, ...] | None, dict[str, Any]]
|
||||
| tuple[tuple[int, ...] | None, dict[str, Any], Any]
|
||||
)
|
||||
# Dispatchers select a concrete launch configuration for a Triton kernel.
|
||||
DispatchSpec = (
|
||||
tuple[tuple[int, ...], dict[str, Any]] | tuple[tuple[int, ...], dict[str, Any], Any]
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _LaunchBindingPlan:
|
||||
input_targets: tuple[tuple[int, str], ...]
|
||||
stride_targets: tuple[tuple[str, str, int], ...]
|
||||
|
||||
|
||||
@cache
|
||||
def _launch_binding_plan(
|
||||
arg_names: tuple[str, ...],
|
||||
input_names: tuple[str, ...],
|
||||
launch_names: tuple[str, ...],
|
||||
) -> _LaunchBindingPlan:
|
||||
arg_name_set = set(arg_names)
|
||||
launch_name_set = set(launch_names)
|
||||
|
||||
def kernel_arg(input_name: str) -> str | None:
|
||||
pointer_name = f"{input_name}_ptr"
|
||||
candidates = (
|
||||
input_name,
|
||||
pointer_name,
|
||||
input_name.upper(),
|
||||
pointer_name.upper(),
|
||||
)
|
||||
return next(
|
||||
(candidate for candidate in candidates if candidate in arg_name_set),
|
||||
None,
|
||||
)
|
||||
|
||||
input_targets = tuple(
|
||||
(index, target)
|
||||
for index, name in enumerate(input_names)
|
||||
if (target := kernel_arg(name)) is not None and target not in launch_name_set
|
||||
)
|
||||
bound_names = launch_name_set | {target for _, target in input_targets}
|
||||
stride_targets = []
|
||||
for target in arg_names:
|
||||
if target in bound_names:
|
||||
continue
|
||||
name, separator, dim = target.rpartition("_stride")
|
||||
if not separator:
|
||||
name, separator, dim = target.rpartition("_STRIDE")
|
||||
dim = dim.removeprefix("_")
|
||||
if separator and (not dim or dim.isdigit()):
|
||||
pointer = f"{name}_PTR" if separator == "_STRIDE" else f"{name}_ptr"
|
||||
source = name if name in bound_names else pointer
|
||||
if source in bound_names:
|
||||
stride_targets.append((target, source, int(dim or 0)))
|
||||
return _LaunchBindingPlan(input_targets, tuple(stride_targets))
|
||||
|
||||
|
||||
def triton_warmup_inputs(
|
||||
kernel: Any,
|
||||
*args: Any,
|
||||
grid: tuple[int, ...],
|
||||
pointer_dtypes: (
|
||||
Mapping[Any, Iterable[str]] | Iterable[tuple[Any, Iterable[str]]] | None
|
||||
) = None,
|
||||
**kwargs: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""Build launcher inputs from Triton's native positional argument order."""
|
||||
arg_names = tuple(kernel.arg_names)
|
||||
if len(args) > len(arg_names):
|
||||
raise ValueError(
|
||||
f"Received {len(args)} positional inputs for {len(arg_names)} "
|
||||
"Triton kernel arguments"
|
||||
)
|
||||
inputs = dict(zip(arg_names, args))
|
||||
duplicate_names = inputs.keys() & kwargs.keys()
|
||||
if duplicate_names:
|
||||
raise ValueError(
|
||||
f"Triton inputs passed twice: {', '.join(sorted(duplicate_names))}"
|
||||
)
|
||||
inputs.update(kwargs)
|
||||
if pointer_dtypes is not None:
|
||||
function_def = get_function_source_node(kernel)
|
||||
if not isinstance(function_def, ast.FunctionDef):
|
||||
raise ValueError("Expected Triton kernel to be defined as a function")
|
||||
pointer_names = _pointer_arg_names(function_def, arg_names)
|
||||
groups = (
|
||||
pointer_dtypes.items()
|
||||
if isinstance(pointer_dtypes, Mapping)
|
||||
else pointer_dtypes
|
||||
)
|
||||
for dtype, names in groups:
|
||||
for name in names:
|
||||
if name not in arg_names:
|
||||
raise ValueError(f"Unknown Triton pointer argument: {name}")
|
||||
if name not in pointer_names:
|
||||
raise ValueError(f"Triton argument is not a pointer: {name}")
|
||||
if name in inputs:
|
||||
raise ValueError(f"Triton input passed twice: {name}")
|
||||
inputs[name] = TritonWarmupTensor(dtype)
|
||||
missing_pointers = pointer_names - inputs.keys()
|
||||
if missing_pointers:
|
||||
names = ", ".join(sorted(missing_pointers))
|
||||
raise ValueError(f"Missing Triton pointer inputs: {names}")
|
||||
return {"grid": grid, **inputs}
|
||||
|
||||
|
||||
def triton_scalar_specialization_rep(value: int) -> int:
|
||||
@@ -72,6 +177,7 @@ class TritonWarmupTensor:
|
||||
aligned: bool = True
|
||||
shape: tuple[int, ...] = (1,)
|
||||
strides: tuple[int, ...] | None = None
|
||||
init: Any = 0
|
||||
|
||||
def data_ptr(self) -> int:
|
||||
return 0 if self.aligned else 1
|
||||
@@ -79,6 +185,12 @@ class TritonWarmupTensor:
|
||||
def ptr_range(self) -> int:
|
||||
return 0
|
||||
|
||||
@property
|
||||
def device(self) -> Any:
|
||||
import torch
|
||||
|
||||
return torch.device(current_platform.device_type)
|
||||
|
||||
def stride(self, dim: int | None = None) -> int | tuple[int, ...]:
|
||||
if self.strides is None:
|
||||
strides: list[int] = []
|
||||
@@ -95,9 +207,10 @@ class TritonWarmupTensor:
|
||||
class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""Triton owner whose runtime launch specification is reused for warmup."""
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
kernel: Any
|
||||
_warming = False
|
||||
_warming_compile_key: CompileKeyT | None = None
|
||||
_run_autotune = False
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
|
||||
@@ -126,6 +239,21 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
f"Cannot inspect kernel parameters for {type(self.kernel).__name__}"
|
||||
)
|
||||
|
||||
def _prepare_launch_kwargs(
|
||||
self,
|
||||
input_names: tuple[str, ...],
|
||||
input_values: tuple[Any, ...],
|
||||
launch_kwargs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
plan = _launch_binding_plan(
|
||||
self._kernel_arg_names, input_names, tuple(launch_kwargs)
|
||||
)
|
||||
for index, target in plan.input_targets:
|
||||
launch_kwargs[target] = input_values[index]
|
||||
for target, source, dim in plan.stride_targets:
|
||||
launch_kwargs[target] = launch_kwargs[source].stride(dim)
|
||||
return launch_kwargs
|
||||
|
||||
def launch(
|
||||
self,
|
||||
launch_spec: LaunchSpec,
|
||||
@@ -136,15 +264,20 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
else:
|
||||
grid, launch_kwargs = launch_spec
|
||||
outputs = None
|
||||
kwargs = dict(launch_kwargs)
|
||||
kwargs = self._prepare_launch_kwargs(
|
||||
tuple(inputs), tuple(inputs.values()), dict(launch_kwargs)
|
||||
)
|
||||
kernel = kwargs.pop("kernel", None) or self.kernel
|
||||
runtime_launcher = kwargs.pop("_runtime_launcher", None)
|
||||
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
|
||||
for name, value in inputs.items():
|
||||
target = name if name in self._kernel_arg_names else f"{name}_ptr"
|
||||
if target in self._kernel_arg_names and target not in kwargs:
|
||||
kwargs[target] = value
|
||||
if self._warming:
|
||||
if self._run_autotune:
|
||||
assert grid is not None
|
||||
kernel[grid](**kwargs)
|
||||
return outputs
|
||||
kwargs = {
|
||||
name: _triton_metadata_arg(value) for name, value in kwargs.items()
|
||||
}
|
||||
if (
|
||||
self._warming_compile_key is not None
|
||||
and "launch_pdl" in kwargs
|
||||
@@ -168,6 +301,367 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
return outputs
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonJitKey:
|
||||
"""Process-local identity of one Triton JIT specialization."""
|
||||
|
||||
jit_function_id: int
|
||||
jit_function_key: Hashable
|
||||
device: Hashable
|
||||
cache_key: Hashable
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonCompileKey:
|
||||
"""Triton-derived compile key with one non-comparing replay input."""
|
||||
|
||||
jit_keys: frozenset[TritonJitKey]
|
||||
inputs: tuple[tuple[str, Any], ...] = field(compare=False, hash=False, repr=False)
|
||||
|
||||
|
||||
def _triton_key_deriver(
|
||||
kernel: Any,
|
||||
) -> Callable[[Mapping[str, Any]], set[TritonJitKey]]:
|
||||
"""Prepare Triton's key derivation once for one kernel and device."""
|
||||
knobs = triton.knobs
|
||||
Autotuner = triton.runtime.autotuner.Autotuner
|
||||
Heuristics = triton.runtime.autotuner.Heuristics
|
||||
driver = triton.runtime.driver
|
||||
JITFunction = triton.runtime.jit.JITFunction
|
||||
compute_cache_key = triton.runtime.jit.compute_cache_key
|
||||
|
||||
if isinstance(kernel, Heuristics):
|
||||
derive_inner = _triton_key_deriver(kernel.fn)
|
||||
|
||||
def derive_heuristic(kwargs: Mapping[str, Any]) -> set[TritonJitKey]:
|
||||
heuristic_kwargs = dict(kwargs)
|
||||
for name, heuristic in kernel.values.items():
|
||||
heuristic_kwargs[name] = heuristic(heuristic_kwargs)
|
||||
return derive_inner(heuristic_kwargs)
|
||||
|
||||
return derive_heuristic
|
||||
|
||||
if isinstance(kernel, Autotuner):
|
||||
derive_inner = _triton_key_deriver(kernel.fn)
|
||||
|
||||
def derive_autotuned(kwargs: Mapping[str, Any]) -> set[TritonJitKey]:
|
||||
previous_nargs = getattr(kernel, "nargs", None)
|
||||
kernel.nargs = {}
|
||||
try:
|
||||
configs = kernel.prune_configs(dict(kwargs))
|
||||
finally:
|
||||
kernel.nargs = previous_nargs
|
||||
keys: set[TritonJitKey] = set()
|
||||
for config in configs:
|
||||
conflicts = kwargs.keys() & config.kwargs.keys()
|
||||
if conflicts:
|
||||
names = ", ".join(sorted(conflicts))
|
||||
raise ValueError(f"Conflicting autotune parameters: {names}")
|
||||
keys.update(derive_inner(dict(kwargs) | config.all_kwargs()))
|
||||
return keys
|
||||
|
||||
return derive_autotuned
|
||||
|
||||
if not isinstance(kernel, JITFunction):
|
||||
raise TypeError(f"Unsupported Triton kernel wrapper: {type(kernel).__name__}")
|
||||
|
||||
device = cast(Hashable, driver.active.get_current_device())
|
||||
_, kernel_key_cache, _, _, binder = kernel.device_caches[device]
|
||||
jit_function_id = id(kernel)
|
||||
jit_function_key = cast(Hashable, kernel.cache_key)
|
||||
debug = kernel.debug or knobs.runtime.debug
|
||||
instrumentation_mode = knobs.compilation.instrumentation_mode
|
||||
fpsan_homomorphic_casts = getattr(
|
||||
knobs.compilation, "fpsan_homomorphic_casts", None
|
||||
)
|
||||
inspection_hook = knobs.runtime.add_stages_inspection_hook
|
||||
|
||||
def derive_jit(kwargs: Mapping[str, Any]) -> set[TritonJitKey]:
|
||||
binder_kwargs = dict(kwargs)
|
||||
binder_kwargs["debug"] = binder_kwargs.get("debug", debug) or debug
|
||||
binder_kwargs["instrumentation_mode"] = instrumentation_mode
|
||||
if fpsan_homomorphic_casts is not None:
|
||||
binder_kwargs["fpsan_homomorphic_casts"] = fpsan_homomorphic_casts
|
||||
_, specialization, options = binder(**binder_kwargs)
|
||||
if inspection_hook is not None:
|
||||
_, inspection_hash = inspection_hook()
|
||||
specialization.append(f'("custom_pipeline", {inspection_hash})')
|
||||
cache_key = cast(
|
||||
Hashable,
|
||||
compute_cache_key(kernel_key_cache, specialization, options),
|
||||
)
|
||||
return {
|
||||
TritonJitKey(
|
||||
jit_function_id,
|
||||
jit_function_key,
|
||||
device,
|
||||
cache_key,
|
||||
)
|
||||
}
|
||||
|
||||
return derive_jit
|
||||
|
||||
|
||||
WarmupCases = Mapping[str, Any] | Iterable[Mapping[str, Any]]
|
||||
|
||||
|
||||
class _AutomaticTritonJitKernel(VllmTritonJitKernel[TritonCompileKey]):
|
||||
def warmup_inputs(self, compile_key: TritonCompileKey) -> dict[str, Any]:
|
||||
return dict(compile_key.inputs)
|
||||
|
||||
def _provider_cases(
|
||||
self,
|
||||
provider: Callable[..., WarmupCases],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Iterable[Mapping[str, Any]]:
|
||||
cases_node = get_function_source_node(provider)
|
||||
if any(
|
||||
isinstance(node, ast.Call)
|
||||
and (get_ast_full_name(node.func) or "").split(".")[-1]
|
||||
in {"WarmupIntRange", "WarmupChoices", "_when"}
|
||||
for node in ast.walk(cases_node)
|
||||
):
|
||||
return self._expand_warmup_cases(provider, *args, **kwargs)
|
||||
cases = provider(*args, **kwargs)
|
||||
return (cases,) if isinstance(cases, Mapping) else cases
|
||||
|
||||
|
||||
def _is_autotuned(kernel: Any) -> bool:
|
||||
Autotuner = triton.runtime.autotuner.Autotuner
|
||||
|
||||
while kernel is not None:
|
||||
if isinstance(kernel, Autotuner):
|
||||
return True
|
||||
kernel = getattr(kernel, "fn", None)
|
||||
return False
|
||||
|
||||
|
||||
def _triton_metadata_arg(value: Any) -> Any:
|
||||
from torch._subclasses.fake_tensor import FakeTensor
|
||||
|
||||
if not isinstance(value, FakeTensor):
|
||||
return value
|
||||
return TritonWarmupTensor(
|
||||
value.dtype,
|
||||
aligned=getattr(value, "_vllm_warmup_aligned", True),
|
||||
shape=tuple(value.shape),
|
||||
strides=tuple(value.stride()),
|
||||
)
|
||||
|
||||
|
||||
def _materialize_warmup_case(
|
||||
case: Mapping[str, Any],
|
||||
*,
|
||||
real: bool,
|
||||
mode: Any,
|
||||
cache: dict[TritonWarmupTensor, Any],
|
||||
) -> dict[str, Any]:
|
||||
import torch
|
||||
|
||||
def materialize(value: Any) -> Any:
|
||||
if not isinstance(value, TritonWarmupTensor):
|
||||
return value
|
||||
if not real:
|
||||
try:
|
||||
return cache[value]
|
||||
except KeyError:
|
||||
pass
|
||||
except TypeError:
|
||||
# Unhashable initializers are uncommon and safe to materialize directly.
|
||||
pass
|
||||
|
||||
strides = cast(tuple[int, ...], value.stride())
|
||||
with mode:
|
||||
tensor = torch.empty_strided(
|
||||
value.shape,
|
||||
strides,
|
||||
dtype=value.dtype,
|
||||
device=value.device,
|
||||
)
|
||||
if real:
|
||||
if callable(value.init):
|
||||
value.init(tensor)
|
||||
else:
|
||||
tensor.fill_(value.init)
|
||||
else:
|
||||
tensor._vllm_warmup_aligned = value.aligned
|
||||
with suppress(TypeError):
|
||||
cache[value] = tensor
|
||||
return tensor
|
||||
|
||||
return {name: materialize(value) for name, value in case.items()}
|
||||
|
||||
|
||||
class _DecoratedTritonJitKernel(_AutomaticTritonJitKernel):
|
||||
def __init__(
|
||||
self,
|
||||
kernel: Any,
|
||||
warmup_inputs: Callable[..., WarmupCases],
|
||||
dispatch: Callable[..., DispatchSpec] | None,
|
||||
) -> None:
|
||||
self.kernel = kernel
|
||||
self._run_autotune = _is_autotuned(kernel)
|
||||
self._warmup_inputs_fn = warmup_inputs
|
||||
self._dispatch_fn = dispatch
|
||||
self._dispatch_arg_names: tuple[str, ...] = ()
|
||||
self._dispatch_defaults: dict[str, Any] = {}
|
||||
if dispatch is not None:
|
||||
parameters = tuple(inspect.signature(dispatch).parameters.values())
|
||||
if any(
|
||||
parameter.kind
|
||||
in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
|
||||
for parameter in parameters
|
||||
):
|
||||
raise TypeError("Triton dispatchers require explicit parameters")
|
||||
self._dispatch_arg_names = tuple(parameter.name for parameter in parameters)
|
||||
self._dispatch_defaults = {
|
||||
parameter.name: parameter.default
|
||||
for parameter in parameters
|
||||
if parameter.default is not inspect.Parameter.empty
|
||||
}
|
||||
super().__init__()
|
||||
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||
if self._dispatch_fn is None:
|
||||
grid = kwargs.pop("grid") if not args else args[0]
|
||||
kernel_args = () if not args else args[1:]
|
||||
inputs = triton_warmup_inputs(
|
||||
self.kernel, *kernel_args, grid=grid, **kwargs
|
||||
)
|
||||
inputs.pop("grid")
|
||||
return self.launch((grid, inputs), {})
|
||||
spec = self._dispatch_fn(*args, **kwargs)
|
||||
input_values = args
|
||||
if len(args) != len(self._dispatch_arg_names):
|
||||
input_values += tuple(
|
||||
kwargs[name] if name in kwargs else self._dispatch_defaults[name]
|
||||
for name in self._dispatch_arg_names[len(args) :]
|
||||
)
|
||||
return self.launch(
|
||||
spec,
|
||||
dict(zip(self._dispatch_arg_names, input_values)),
|
||||
)
|
||||
|
||||
def __getitem__(self, grid: tuple[int, ...]) -> Callable[..., Any]:
|
||||
return cast(Callable[..., Any], self.kernel[grid])
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self.kernel, name)
|
||||
|
||||
def get_warmup_keys(
|
||||
self, *, vllm_config: Any | None = None, **kwargs: Any
|
||||
) -> list[TritonCompileKey]:
|
||||
keys: list[TritonCompileKey] = []
|
||||
seen_jit_keys: set[frozenset[TritonJitKey]] = set()
|
||||
if "vllm_config" in inspect.signature(self._warmup_inputs_fn).parameters:
|
||||
kwargs.setdefault("vllm_config", vllm_config)
|
||||
cases = self._provider_cases(self._warmup_inputs_fn, **kwargs)
|
||||
derive_keys = _triton_key_deriver(self.kernel)
|
||||
materialize = self._run_autotune
|
||||
mode: Any = nullcontext() if self._run_autotune else None
|
||||
materialized: dict[TritonWarmupTensor, Any] = {}
|
||||
for case in cases:
|
||||
inputs = (
|
||||
_materialize_warmup_case(
|
||||
case,
|
||||
real=self._run_autotune,
|
||||
mode=mode,
|
||||
cache=materialized,
|
||||
)
|
||||
if materialize
|
||||
else case
|
||||
)
|
||||
if self._dispatch_fn is None:
|
||||
prepared = dict(inputs)
|
||||
prepared.pop("grid")
|
||||
else:
|
||||
dispatch_inputs = {
|
||||
name: inputs[name]
|
||||
for name in self._dispatch_arg_names
|
||||
if name in inputs
|
||||
}
|
||||
try:
|
||||
_, launch_kwargs = self._dispatch_fn(**dispatch_inputs)[:2]
|
||||
except (AssertionError, AttributeError, TypeError):
|
||||
if materialize:
|
||||
raise
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
materialize = True
|
||||
mode = FakeTensorMode()
|
||||
inputs = _materialize_warmup_case(
|
||||
case,
|
||||
real=False,
|
||||
mode=mode,
|
||||
cache=materialized,
|
||||
)
|
||||
dispatch_inputs = {
|
||||
name: inputs[name]
|
||||
for name in self._dispatch_arg_names
|
||||
if name in inputs
|
||||
}
|
||||
_, launch_kwargs = self._dispatch_fn(**dispatch_inputs)[:2]
|
||||
prepared = self._prepare_launch_kwargs(
|
||||
tuple(inputs),
|
||||
tuple(inputs.values()),
|
||||
dict(launch_kwargs),
|
||||
)
|
||||
if materialize:
|
||||
prepared = {
|
||||
name: _triton_metadata_arg(value)
|
||||
for name, value in prepared.items()
|
||||
}
|
||||
jit_keys = frozenset(derive_keys(prepared))
|
||||
if jit_keys and jit_keys not in seen_jit_keys:
|
||||
seen_jit_keys.add(jit_keys)
|
||||
keys.append(TritonCompileKey(jit_keys, tuple(inputs.items())))
|
||||
return keys
|
||||
|
||||
|
||||
class TritonKernelDispatcher(Protocol[P]):
|
||||
"""Callable Triton launcher with compile-time warmup support."""
|
||||
|
||||
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> Any: ...
|
||||
|
||||
def register_warmup(self, *args: Any, **kwargs: Any) -> None: ...
|
||||
|
||||
|
||||
@overload
|
||||
def triton_kernel_dispatcher_with_warmup(
|
||||
*,
|
||||
warmup_inputs: Callable[..., WarmupCases],
|
||||
kernel: None = None,
|
||||
) -> Callable[[Any], Any]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def triton_kernel_dispatcher_with_warmup(
|
||||
*,
|
||||
warmup_inputs: Callable[..., WarmupCases],
|
||||
kernel: Any,
|
||||
) -> Callable[[Callable[P, DispatchSpec]], TritonKernelDispatcher[P]]: ...
|
||||
|
||||
|
||||
def triton_kernel_dispatcher_with_warmup(
|
||||
*,
|
||||
warmup_inputs: Callable[..., WarmupCases],
|
||||
kernel: Any | None = None,
|
||||
) -> Callable[[Any], Any]:
|
||||
"""Decorate a dispatch function or a native Triton kernel for warmup."""
|
||||
|
||||
def decorate(dispatch: Any) -> Any:
|
||||
owner = _DecoratedTritonJitKernel(
|
||||
dispatch if kernel is None else kernel,
|
||||
warmup_inputs,
|
||||
None if kernel is None else dispatch,
|
||||
)
|
||||
update_wrapper(owner, dispatch, updated=())
|
||||
return owner
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonPointerInputVariant:
|
||||
# Named pointer-alignment variant for compile-only Triton warmup.
|
||||
|
||||
@@ -41,9 +41,6 @@ from vllm.model_executor.warmup.qwen_vl_triton_warmup import qwen_vl_triton_warm
|
||||
from vllm.model_executor.warmup.replayssm_warmup import (
|
||||
replayssm_autotune_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
|
||||
spec_decode_rejection_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.watermark_sample_warmup import (
|
||||
watermark_sample_warmup,
|
||||
)
|
||||
@@ -178,7 +175,10 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
logger.info("JIT kernel warmup starting.")
|
||||
jit_warmup_start = time.perf_counter()
|
||||
try:
|
||||
worker.model_runner.jit_warmup_registry.warmup()
|
||||
registry = (
|
||||
worker.model_runner.jit_warmup_registry # type: ignore[attr-defined]
|
||||
)
|
||||
registry.warmup()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"JIT kernel warmup failed after %.2fs.",
|
||||
@@ -209,7 +209,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
if enable_jit_warmup:
|
||||
kimi_k3_triton_warmup(worker)
|
||||
spec_decode_rejection_warmup(worker)
|
||||
watermark_sample_warmup(worker)
|
||||
qwen4_exp_qsa_triton_warmup(worker)
|
||||
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up speculative-decoding rejection kernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def spec_decode_rejection_warmup(worker: Worker) -> None:
|
||||
spec_config = worker.vllm_config.speculative_config
|
||||
if spec_config is None:
|
||||
return
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
rejection_sample,
|
||||
)
|
||||
|
||||
model_config = worker.vllm_config.model_config
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
num_spec = spec_config.num_speculative_tokens
|
||||
if num_spec <= 0 or vocab_size <= 0:
|
||||
return
|
||||
|
||||
rejection_method = getattr(spec_config, "rejection_sample_method", None)
|
||||
use_block_verification = rejection_method == "block"
|
||||
use_synthetic = rejection_method == "synthetic"
|
||||
# HAS_DRAFT_LOGITS is a constexpr, so the warmup must match the runtime.
|
||||
use_draft_logits = (
|
||||
getattr(spec_config, "draft_sample_method", "greedy") == "probabilistic"
|
||||
)
|
||||
|
||||
device = worker.device
|
||||
num_reqs = 1
|
||||
tokens_per_req = num_spec + 1
|
||||
num_logits = num_reqs * tokens_per_req
|
||||
|
||||
watermark_config = getattr(worker.vllm_config, "watermark_config", None)
|
||||
watermark_kwargs: dict[str, Any] = {}
|
||||
if watermark_config is not None:
|
||||
from vllm.v1.watermarking.spec_decode import speculative_target_watermark_key
|
||||
|
||||
try:
|
||||
watermark_key = speculative_target_watermark_key(watermark_config)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Skipping spec-decode rejection sampler warmup: could not derive "
|
||||
"the watermark recovery key.",
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
# Tensor dtypes and integer widths affect Triton specialization.
|
||||
watermark_kwargs = {
|
||||
"contexts": torch.zeros(
|
||||
(num_logits, watermark_config.context_width),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
"watermarking": torch.zeros(num_reqs, dtype=torch.bool, device=device),
|
||||
"watermark_key": watermark_key,
|
||||
}
|
||||
|
||||
# Sampling-parameter processing may promote either distribution to fp32.
|
||||
model_dtype = model_config.dtype
|
||||
warmup_dtype_pairs: set[tuple[torch.dtype, torch.dtype | None]]
|
||||
if use_draft_logits:
|
||||
warmup_dtype_pairs = {
|
||||
(model_dtype, model_dtype),
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, model_dtype),
|
||||
(model_dtype, torch.float32),
|
||||
}
|
||||
else:
|
||||
warmup_dtype_pairs = {(model_dtype, None), (torch.float32, None)}
|
||||
|
||||
logger.info(
|
||||
"Warming up spec-decode rejection sampler kernels "
|
||||
"(vocab=%d, num_spec=%d, dtype_pairs=%s, block_verify=%s, watermark=%s).",
|
||||
vocab_size,
|
||||
num_spec,
|
||||
[(str(t), str(d)) for t, d in warmup_dtype_pairs],
|
||||
use_block_verification,
|
||||
watermark_config is not None,
|
||||
)
|
||||
for tgt_dtype, draft_dtype in warmup_dtype_pairs:
|
||||
target_logits = torch.zeros(
|
||||
(num_logits, vocab_size), dtype=tgt_dtype, device=device
|
||||
)
|
||||
draft_logits = (
|
||||
torch.zeros(
|
||||
(num_reqs, num_spec, vocab_size), dtype=draft_dtype, device=device
|
||||
)
|
||||
if draft_dtype is not None
|
||||
else None
|
||||
)
|
||||
synthetic_rates = (
|
||||
torch.full((num_spec,), 0.5, dtype=torch.float32, device=device)
|
||||
if use_synthetic
|
||||
else None
|
||||
)
|
||||
try:
|
||||
rejection_sample(
|
||||
target_logits=target_logits,
|
||||
draft_logits=draft_logits,
|
||||
draft_sampled=torch.zeros(num_logits, dtype=torch.int32, device=device),
|
||||
cu_num_logits=torch.tensor(
|
||||
[0, num_logits], dtype=torch.int32, device=device
|
||||
),
|
||||
pos=torch.zeros(num_logits, dtype=torch.int64, device=device),
|
||||
idx_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
|
||||
expanded_idx_mapping=torch.zeros(
|
||||
num_logits, dtype=torch.int64, device=device
|
||||
),
|
||||
expanded_local_pos=torch.arange(
|
||||
num_logits, dtype=torch.int32, device=device
|
||||
),
|
||||
temperature=torch.zeros(num_reqs, dtype=torch.float32, device=device),
|
||||
seed=torch.full((num_reqs,), 42, dtype=torch.int64, device=device),
|
||||
num_speculative_steps=num_spec,
|
||||
synthetic_conditional_rates=synthetic_rates,
|
||||
use_fp64=False,
|
||||
use_block_verification=use_block_verification,
|
||||
**watermark_kwargs,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Skipping spec-decode rejection sampler warmup.", exc_info=True
|
||||
)
|
||||
return
|
||||
Reference in New Issue
Block a user