[Feat][Perf] Add new warmup infrastructure for JITs (#47451)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Roberto L. Castro
2026-07-20 13:21:55 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fbfe58133d
commit 7ca017778f
14 changed files with 1855 additions and 693 deletions
@@ -2,6 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Run registered CuTeDSL warmup compile units."""
# TODO(roberto): Remove this compatibility registry after registered CuTeDSL
# warmups are migrated to the shared JIT warmup infrastructure.
# https://github.com/vllm-project/vllm/pull/47451
from __future__ import annotations
import time
@@ -1,204 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FA4 MLA prefill CuTeDSL compile warmup config."""
from __future__ import annotations
from collections.abc import Hashable, Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Literal
import torch
if TYPE_CHECKING:
from vllm.v1.attention.backends.fa_utils import (
FlashAttentionCuTeDSLCompileSpec,
)
FA4ArchitectureFamily = Literal["sm90", "sm100f", "sm120"]
FA4_STANDARD_DTYPES = (torch.bfloat16, torch.float16)
# Current vLLM MLA prefill expands K/V to num_heads before FA4, so this plan
# covers qhead_per_kvhead=1.
# Batch is not a current FA4 MLA-prefill key field. Use b1 for compile-only
# specs because it is the conservative case for Split-KV shape heuristics.
# TODO(roberto): FA4 also has direct-GQA and qv/top-k absorbed-MLA paths, but vLLM
# does not use them in this backend yet; they need a separate
# num_kv_heads/qv/top-k-aware warmup plan if wired in later.
FA4_MLA_PREFILL_COMPILE_BATCH_SIZE = 1
FA4_MLA_PREFILL_Q_TILE = 128
FA4_MLA_PREFILL_K_TILE = 128
FA4_MLA_PREFILL_LONG_K_BLOCKS = 32
FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS = 64
FA4_MLA_PREFILL_CAUSAL_OPTIONS = (False, True)
FA4_MLA_PREFILL_LSE_OPTIONS = (False, True)
@dataclass(frozen=True)
class FA4MLAPrefillCompileContext:
dtype: torch.dtype
num_heads: int
qk_head_dim: int
v_head_dim: int
kv_nope_head_dim: int
requires_v_padding: bool
scale: float
num_splits: int
fa_version: int
# Return the V head dim FA4 sees.
@property
def effective_v_head_dim(self) -> int:
if self.requires_v_padding:
return self.qk_head_dim
return self.v_head_dim
@dataclass(frozen=True)
class FA4MLAPrefillCompileRequest:
"""One compile-only FA4 MLA prefill request."""
key: Hashable
compile_spec: FlashAttentionCuTeDSLCompileSpec
# Compile this request.
def compile(self) -> None:
self.compile_spec.compile()
# Yield deduped compile requests.
def iter_fa4_mla_prefill_compile_requests(
ctx: FA4MLAPrefillCompileContext,
) -> Iterator[FA4MLAPrefillCompileRequest]:
"""Yield compile requests for this fixed MLA backend.
FA4 dedupes duplicate atomic kernel selections in its own JIT cache.
"""
seen: set[Hashable] = set()
for compile_spec in iter_fa4_mla_prefill_compile_specs(ctx):
key = compile_spec.request_key()
if key in seen:
continue
seen.add(key)
yield FA4MLAPrefillCompileRequest(
key=key,
compile_spec=compile_spec,
)
# Build compile specs for this setup.
def iter_fa4_mla_prefill_compile_specs(
ctx: FA4MLAPrefillCompileContext,
) -> Iterator[FlashAttentionCuTeDSLCompileSpec]:
"""Yield compile-only FA4 MLA prefill requests for this fixed setup."""
arch_family = _fa4_architecture_family_from_compute_capability(
*torch.cuda.get_device_capability()
)
if not _supports_fa4_mla_prefill(ctx, arch_family):
return
from vllm.v1.attention.backends.fa_utils import (
FlashAttentionCuTeDSLCompileSpec,
)
batch_size = FA4_MLA_PREFILL_COMPILE_BATCH_SIZE
v_stride = None
if not ctx.requires_v_padding:
v_stride = (
ctx.num_heads * ctx.kv_nope_head_dim,
ctx.kv_nope_head_dim,
1,
)
for _, max_seqlen_q, max_seqlen_k in _shape_probes_for_context(ctx, arch_family):
total_q_tokens = batch_size * max_seqlen_q
total_kv_tokens = batch_size * max_seqlen_k
for causal in FA4_MLA_PREFILL_CAUSAL_OPTIONS:
for return_lse in FA4_MLA_PREFILL_LSE_OPTIONS:
yield FlashAttentionCuTeDSLCompileSpec(
q_shape=(total_q_tokens, ctx.num_heads, ctx.qk_head_dim),
k_shape=(total_kv_tokens, ctx.num_heads, ctx.qk_head_dim),
v_shape=(
total_kv_tokens,
ctx.num_heads,
ctx.effective_v_head_dim,
),
v_stride=v_stride,
q_dtype=ctx.dtype,
cu_seqlens_q_shape=(batch_size + 1,),
cu_seqlens_k_shape=(batch_size + 1,),
max_seqlen_q=max_seqlen_q,
max_seqlen_k=max_seqlen_k,
softmax_scale=ctx.scale,
causal=causal,
return_softmax_lse=return_lse,
num_splits=ctx.num_splits,
fa_version=ctx.fa_version,
)
# Pick one q/k point per current FA4 MLA-prefill shape regime.
def _shape_probes_for_context(
ctx: FA4MLAPrefillCompileContext,
arch_family: FA4ArchitectureFamily,
) -> tuple[tuple[str, int, int], ...]:
q_stage1_q = 1
q_stage2_q = FA4_MLA_PREFILL_Q_TILE + 1
# FA4 never auto-splits when ceil(max_seqlen_k / tile_n) <= 4.
no_split_k = 4 * FA4_MLA_PREFILL_K_TILE
long_k = FA4_MLA_PREFILL_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
# Diff-head-dim Blackwell Split-KV switches tile_n at 64 K blocks.
very_long_k = FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
base_probes = (
("q_stage1", q_stage1_q, FA4_MLA_PREFILL_K_TILE),
("q_stage2", q_stage2_q, no_split_k),
)
# SM120 currently rejects Split-KV in FA4; num_splits=1 also has no split
# shape regimes on any architecture.
if ctx.num_splits == 1 or arch_family == "sm120":
return base_probes
long_k_probes = (
("q_stage1_long_k", q_stage1_q, long_k),
("q_stage2_long_k", q_stage2_q, long_k),
)
# SM90 does not have the SM100 q_stage or diff-head-dim tile_n=64 branch.
# Same-dim SM100-family MLA also does not need the very-long-K probe.
if arch_family == "sm90" or ctx.qk_head_dim == ctx.effective_v_head_dim:
return (*base_probes, *long_k_probes)
very_long_k_probes = (
("q_stage1_very_long_k", q_stage1_q, very_long_k),
("q_stage2_very_long_k", q_stage2_q, very_long_k),
)
return (*base_probes, *long_k_probes, *very_long_k_probes)
# Check whether this setup can use FA4 MLA prefill.
def _supports_fa4_mla_prefill(
ctx: FA4MLAPrefillCompileContext,
arch_family: FA4ArchitectureFamily,
) -> bool:
return (
ctx.dtype in FA4_STANDARD_DTYPES
and ctx.num_heads > 0
and (arch_family != "sm120" or ctx.num_splits == 1)
)
# Map CUDA capability to the FA4 arch family used by warmup checks.
def _fa4_architecture_family_from_compute_capability(
major: int,
minor: int,
) -> FA4ArchitectureFamily:
if (major, minor) == (9, 0):
return "sm90"
if major == 10:
return "sm100f"
if (major, minor) == (12, 0):
return "sm120"
raise ValueError(f"FA4 warmup does not know CUDA capability {major}.{minor}")
@@ -0,0 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Warm up FA4 CuTeDSL MLA prefill compile keys."""
from __future__ import annotations
from typing import TYPE_CHECKING
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
if TYPE_CHECKING:
from vllm.v1.worker.gpu_worker import Worker
def fa4_cutedsl_warmup(worker: Worker) -> None:
runner = worker.model_runner
if runner.is_pooling_model:
return
vllm_config = runner.vllm_config
if not vllm_config.model_config.use_mla:
return
backend_cls = get_mla_prefill_backend(vllm_config)
if backend_cls.get_name() != "FLASH_ATTN":
return
from vllm.v1.attention.backends.mla.prefill import flash_attn
flash_attn.FA4_MLA_PREFILL_KERNEL.warmup(vllm_config)
+474
View File
@@ -0,0 +1,474 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared interfaces and tracing helpers for explicit JIT warmup keys."""
from __future__ import annotations
import ast
import inspect
import itertools
import operator
import textwrap
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass
from typing import Any, Generic, TypeVar
__all__ = [
"VllmJitKernel",
"WarmupIntRange",
"get_ast_full_name",
"get_function_source_node",
"zip_inputs",
]
CompileKeyT = TypeVar("CompileKeyT")
@dataclass(frozen=True)
class WarmupIntRange:
start: int
stop: int
step: int = 1
WarmupValues = Any
CompileKeyDispatchFn = Callable[..., CompileKeyT]
@dataclass(frozen=True)
class _WarmupInputRows:
"""Warmup dispatch inputs expanded in lockstep."""
rows: tuple[Mapping[str, WarmupValues], ...]
def _expand_warmup_values(values: WarmupValues) -> tuple[Any, ...]:
if isinstance(values, WarmupIntRange):
return tuple(range(values.start, values.stop, values.step))
if isinstance(values, (list, tuple)):
return tuple(values)
return (values,)
def zip_inputs(*rows: Mapping[str, WarmupValues]) -> _WarmupInputRows:
"""Group row-wise dispatch inputs that should be expanded in lockstep."""
if not rows:
raise ValueError("zip_inputs requires at least one dispatch input row")
if not all(isinstance(row, Mapping) for row in rows):
raise ValueError("zip_inputs rows must be mappings")
first_names = frozenset(rows[0])
if not first_names:
raise ValueError("zip_inputs rows require at least one dispatch input name")
if not all(isinstance(name, str) for name in first_names):
raise ValueError("zip_inputs dispatch input names must be strings")
input_rows: list[Mapping[str, WarmupValues]] = []
for row in rows:
names = frozenset(row)
if names != first_names:
raise ValueError("zip_inputs rows must use the same dispatch input names")
input_rows.append(dict(row))
return _WarmupInputRows(rows=tuple(input_rows))
def _expand_warmup_value_grid(
values: Mapping[str, WarmupValues],
input_names: frozenset[str],
) -> tuple[dict[str, Any], ...]:
names = tuple(name for name in values if name in input_names)
if not names:
return ({},)
expanded_values = tuple(_expand_warmup_values(values[name]) for name in names)
return tuple(
dict(zip(names, value_set)) for value_set in itertools.product(*expanded_values)
)
def _expand_warmup_input_rows(
rows: tuple[Mapping[str, WarmupValues], ...],
input_names: frozenset[str],
) -> tuple[dict[str, Any], ...]:
active_names = frozenset(name for name in rows[0] if name in input_names)
if not active_names:
return ({},)
return tuple(
{name: value for name, value in row.items() if name in active_names}
for row in rows
)
def _merge_warmup_kwargs(parts: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
merged: dict[str, Any] = {}
for part in parts:
for name, value in part.items():
if name in merged:
raise ValueError(
f"Warmup dispatch input '{name}' is specified more than once"
)
merged[name] = value
return merged
@dataclass(frozen=True)
class _CompileKeyDispatchTrace:
local_exprs: tuple[tuple[str, ast.AST], ...]
field_exprs: tuple[tuple[str, ast.AST], ...]
globals: Mapping[str, Any]
input_names: frozenset[str]
defaults: Mapping[str, Any]
def compile_key(
self,
compile_key_type: type[CompileKeyT],
kwargs: Mapping[str, Any],
) -> CompileKeyT:
dispatch_values = {**self.defaults, **kwargs}
for name, expr in self.local_exprs:
dispatch_values[name] = _eval_dispatch_expr(
expr, dispatch_values, self.globals
)
return compile_key_type(
**{
field: _eval_dispatch_expr(expr, dispatch_values, self.globals)
for field, expr in self.field_exprs
}
)
_BIN_OPS: dict[type[ast.operator], Callable[[Any, Any], Any]] = {
ast.Add: operator.add,
ast.Sub: operator.sub,
ast.Mult: operator.mul,
ast.FloorDiv: operator.floordiv,
ast.Mod: operator.mod,
ast.Pow: operator.pow,
}
_CMP_OPS: dict[type[ast.cmpop], Callable[[Any, Any], bool]] = {
ast.Eq: operator.eq,
ast.NotEq: operator.ne,
ast.Lt: operator.lt,
ast.LtE: operator.le,
ast.Gt: operator.gt,
ast.GtE: operator.ge,
}
def _dispatch_expr_source(node: ast.AST) -> str:
try:
return ast.unparse(node)
except Exception:
return ast.dump(node)
def _dispatch_expr_error(node: ast.AST, reason: str) -> ValueError:
return ValueError(
f"{reason}: {_dispatch_expr_source(node)}. "
"Supported dispatch expressions are names, constants, attributes, "
"tuple/list literals, conditional expressions, comparisons, boolean "
"operators, unary not/minus, arithmetic, and calls without **kwargs."
)
class _DispatchExprEvaluator(ast.NodeVisitor):
def __init__(
self,
values: Mapping[str, Any],
globals_: Mapping[str, Any],
) -> None:
self.values = values
self.globals = globals_
def eval(self, node: ast.AST) -> Any:
return self.visit(node)
def generic_visit(self, node: ast.AST) -> Any:
raise _dispatch_expr_error(node, "Unsupported dispatch expression")
def visit_Name(self, node: ast.Name) -> Any:
if node.id in self.values:
return self.values[node.id]
if node.id in self.globals:
return self.globals[node.id]
raise _dispatch_expr_error(node, f"Unknown dispatch name '{node.id}'")
def visit_Constant(self, node: ast.Constant) -> Any:
return node.value
def visit_IfExp(self, node: ast.IfExp) -> Any:
return self.visit(node.body if self.visit(node.test) else node.orelse)
def visit_Tuple(self, node: ast.Tuple) -> tuple[Any, ...]:
return tuple(self.visit(elt) for elt in node.elts)
def visit_List(self, node: ast.List) -> list[Any]:
return [self.visit(elt) for elt in node.elts]
def visit_BoolOp(self, node: ast.BoolOp) -> Any:
if isinstance(node.op, ast.And):
result = None
for value in node.values:
result = self.visit(value)
if not result:
return result
return result
if isinstance(node.op, ast.Or):
result = None
for value in node.values:
result = self.visit(value)
if result:
return result
return result
raise _dispatch_expr_error(node, "Unsupported dispatch boolean operator")
def visit_Compare(self, node: ast.Compare) -> bool:
left = self.visit(node.left)
for op_node, comparator in zip(node.ops, node.comparators):
right = self.visit(comparator)
op = _CMP_OPS.get(type(op_node))
if op is None:
raise _dispatch_expr_error(
node, "Unsupported dispatch comparison operator"
)
if not op(left, right):
return False
left = right
return True
def visit_UnaryOp(self, node: ast.UnaryOp) -> Any:
operand = self.visit(node.operand)
if isinstance(node.op, ast.Not):
return not operand
if isinstance(node.op, ast.USub):
return -operand
raise _dispatch_expr_error(node, "Unsupported dispatch unary operator")
def visit_BinOp(self, node: ast.BinOp) -> Any:
op = _BIN_OPS.get(type(node.op))
if op is None:
raise _dispatch_expr_error(node, "Unsupported dispatch binary operator")
return op(self.visit(node.left), self.visit(node.right))
def visit_Call(self, node: ast.Call) -> Any:
args = [self.visit(arg) for arg in node.args]
fn = self.visit(node.func)
call_kwargs: dict[str, Any] = {}
for keyword in node.keywords:
if keyword.arg is None:
raise _dispatch_expr_error(
node, "Dispatch helper calls cannot use **kwargs"
)
call_kwargs[keyword.arg] = self.visit(keyword.value)
return fn(*args, **call_kwargs)
def visit_Attribute(self, node: ast.Attribute) -> Any:
return getattr(self.visit(node.value), node.attr)
def get_ast_full_name(node: ast.AST) -> str | None:
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
parent = get_ast_full_name(node.value)
if parent is not None:
return f"{parent}.{node.attr}"
return None
def get_function_source_node(fn: Callable[..., Any]) -> ast.FunctionDef:
source_fn = getattr(fn, "fn", fn)
source = textwrap.dedent(inspect.getsource(source_fn))
tree = ast.parse(source)
function_defs = [node for node in tree.body if isinstance(node, ast.FunctionDef)]
if len(function_defs) != 1:
name = getattr(source_fn, "__name__", type(source_fn).__name__)
raise ValueError(f"Expected one function in {name}, found {len(function_defs)}")
return function_defs[0]
def _eval_dispatch_expr(
node: ast.AST,
kwargs: Mapping[str, Any],
globals_: Mapping[str, Any],
) -> Any:
return _DispatchExprEvaluator(kwargs, globals_).eval(node)
def _collect_input_names(
node: ast.AST,
candidate_names: set[str],
local_names: set[str] | None = None,
) -> set[str]:
if local_names is None:
local_names = set()
return {
child.id
for child in ast.walk(node)
if (
isinstance(child, ast.Name)
and child.id in candidate_names
and child.id not in local_names
)
}
def _collect_dispatch_body(
fn: CompileKeyDispatchFn[Any],
function_def: ast.FunctionDef,
) -> tuple[list[tuple[str, ast.AST]], ast.Call]:
local_exprs: list[tuple[str, ast.AST]] = []
for statement in function_def.body:
if (
isinstance(statement, ast.Expr)
and isinstance(statement.value, ast.Constant)
and isinstance(statement.value.value, str)
):
continue
if isinstance(statement, ast.Assign):
if len(statement.targets) != 1 or not isinstance(
statement.targets[0], ast.Name
):
raise _dispatch_expr_error(
statement, "Dispatch assignments must target one local name"
)
local_exprs.append((statement.targets[0].id, statement.value))
continue
if isinstance(statement, ast.AnnAssign):
if statement.value is None:
raise _dispatch_expr_error(
statement, "Dispatch annotations must assign a value"
)
if not isinstance(statement.target, ast.Name):
raise _dispatch_expr_error(
statement, "Dispatch assignments must target one local name"
)
local_exprs.append((statement.target.id, statement.value))
continue
if isinstance(statement, ast.Return) and isinstance(statement.value, ast.Call):
return local_exprs, statement.value
if isinstance(statement, ast.Return):
raise _dispatch_expr_error(
statement, "Dispatch must return one CompileKey(...) call"
)
raise _dispatch_expr_error(
statement,
"Dispatch may only contain local assignments before CompileKey return",
)
raise ValueError(f"Expected {fn.__name__} to return one CompileKey(...) call")
def _trace_compile_key_dispatch(
fn: CompileKeyDispatchFn[Any],
) -> _CompileKeyDispatchTrace:
source_fn = getattr(fn, "__func__", fn)
globals_ = source_fn.__globals__
function_def = get_function_source_node(fn)
local_exprs, return_call = _collect_dispatch_body(fn, function_def)
field_exprs: list[tuple[str, ast.AST]] = []
signature = inspect.signature(fn)
defaults = {
name: parameter.default
for name, parameter in signature.parameters.items()
if parameter.default is not inspect.Parameter.empty
}
candidate_names = set(signature.parameters)
input_names: set[str] = set()
local_names = {name for name, _ in local_exprs}
for _, expr in local_exprs:
input_names.update(_collect_input_names(expr, candidate_names))
for keyword in return_call.keywords:
if keyword.arg is None:
raise ValueError(f"{fn.__name__} cannot use **kwargs in CompileKey")
field_exprs.append((keyword.arg, keyword.value))
input_names.update(
_collect_input_names(keyword.value, candidate_names, local_names)
)
return _CompileKeyDispatchTrace(
tuple(local_exprs),
tuple(field_exprs),
globals_,
frozenset(input_names),
defaults,
)
class VllmJitKernel(Generic[CompileKeyT], ABC):
"""Kernel wrapper that owns dispatch, warmup keys, and compilation."""
CompileKey: type[CompileKeyT]
def __init__(self) -> None:
self.compile_key_dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
def compile_key(self, kwargs: Mapping[str, Any]) -> CompileKeyT:
return self.compile_key_dispatch_trace.compile_key(self.CompileKey, kwargs)
def _trace_dispatch(
self, dispatch: CompileKeyDispatchFn[CompileKeyT]
) -> Callable[..., list[CompileKeyT]]:
compile_key_dispatch_trace = _trace_compile_key_dispatch(dispatch)
def traced(
*input_groups: _WarmupInputRows,
**kwargs: WarmupValues,
) -> list[CompileKeyT]:
for group in input_groups:
if not isinstance(group, _WarmupInputRows):
raise TypeError(
"_trace_dispatch positional arguments must be "
"zip_inputs(...) groups"
)
expanded_input_groups = tuple(
_expand_warmup_input_rows(
group.rows, compile_key_dispatch_trace.input_names
)
for group in input_groups
)
expanded_kwargs = _expand_warmup_value_grid(
kwargs, compile_key_dispatch_trace.input_names
)
dispatch_value_groups = (*expanded_input_groups, expanded_kwargs)
return list(
dict.fromkeys(
compile_key_dispatch_trace.compile_key(
self.CompileKey, _merge_warmup_kwargs(dispatch_values)
)
for dispatch_values in itertools.product(*dispatch_value_groups)
)
)
return traced
@abstractmethod
def dispatch(self, **kwargs: Any) -> CompileKeyT:
"""Build one compile key from one concrete dispatch point."""
raise NotImplementedError
@abstractmethod
def get_warmup_keys(self, *args: Any, **kwargs: Any) -> list[CompileKeyT]:
"""Return compile keys that should be warmed for this kernel."""
raise NotImplementedError
@abstractmethod
def compile(self, compile_key: CompileKeyT) -> None:
"""Compile one warmup key."""
raise NotImplementedError
def warmup(self, *args: Any, **kwargs: Any) -> None:
"""Compile this kernel's warmup keys."""
for compile_key in self.get_warmup_keys(*args, **kwargs):
self.compile(compile_key)
@@ -0,0 +1,183 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ast
import inspect
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from vllm.model_executor.warmup.jit_warmup import (
get_ast_full_name,
get_function_source_node,
)
@dataclass(frozen=True)
class TritonWarmupTensor:
# Compile-only tensor descriptor for Triton pointer specialization.
dtype: Any
aligned: bool = True
shape: tuple[int, ...] = (1,)
def data_ptr(self) -> int:
return 0 if self.aligned else 1
def ptr_range(self) -> int:
return 0
def stride(self) -> tuple[int, ...]:
strides: list[int] = []
stride = 1
for size in reversed(self.shape):
strides.append(stride)
stride *= size
return tuple(reversed(strides))
@dataclass(frozen=True)
class TritonPointerInputVariant:
# Named pointer-alignment variant for compile-only Triton warmup.
alignments: tuple[tuple[str, bool], ...]
@classmethod
def from_alignment(cls, **aligned: bool) -> "TritonPointerInputVariant":
return cls(tuple(aligned.items()))
def is_aligned(self, name: str) -> bool:
for alignment_name, aligned in self.alignments:
if alignment_name == name:
return aligned
raise KeyError(f"Unknown Triton pointer input variant: {name}")
def pointer(
self,
name: str,
dtype: Any,
shape: tuple[int, ...] = (1,),
) -> TritonWarmupTensor:
return TritonWarmupTensor(dtype, aligned=self.is_aligned(name), shape=shape)
def _literal_str_refs(node: ast.AST) -> tuple[str | int, ...]:
if isinstance(node, ast.Constant) and isinstance(node.value, str | int):
return (node.value,)
if isinstance(node, ast.List | ast.Tuple):
refs: list[str | int] = []
for elt in node.elts:
if isinstance(elt, ast.Constant) and isinstance(elt.value, str | int):
refs.append(elt.value)
else:
raise ValueError(
f"Unsupported Triton specialization ref: {ast.dump(elt)}"
)
return tuple(refs)
raise ValueError(f"Unsupported Triton specialization refs: {ast.dump(node)}")
def _normalize_arg_refs(
refs: tuple[str | int, ...],
arg_names: tuple[str, ...],
) -> frozenset[str]:
names: set[str] = set()
for ref in refs:
if isinstance(ref, int):
names.add(arg_names[ref])
else:
names.add(ref)
return frozenset(names)
def _decorator_keyword_refs(
function_def: ast.FunctionDef,
keyword_name: str,
) -> tuple[str | int, ...]:
for decorator in function_def.decorator_list:
if not isinstance(decorator, ast.Call):
continue
decorator_name = get_ast_full_name(decorator.func)
if decorator_name not in ("triton.jit", "jit"):
continue
for keyword in decorator.keywords:
if keyword.arg == keyword_name:
return _literal_str_refs(keyword.value)
return ()
def _triton_do_not_specialize_args(
kernel: Callable[..., Any],
function_def: ast.FunctionDef,
arg_names: tuple[str, ...],
) -> frozenset[str]:
refs = getattr(kernel, "do_not_specialize", None)
if refs is not None:
return _normalize_arg_refs(tuple(refs), arg_names)
return _normalize_arg_refs(
_decorator_keyword_refs(function_def, "do_not_specialize"),
arg_names,
)
def _triton_constexpr_arg_names(
kernel: Callable[..., Any],
function_def: ast.FunctionDef,
arg_names: tuple[str, ...],
) -> frozenset[str]:
constexprs = getattr(kernel, "constexprs", None)
if constexprs is not None:
return frozenset(arg_names[index] for index in constexprs)
names: set[str] = set()
for arg in function_def.args.args + function_def.args.kwonlyargs:
if arg.annotation is None:
continue
annotation = get_ast_full_name(arg.annotation)
if annotation in ("tl.constexpr", "triton.language.constexpr", "constexpr"):
names.add(arg.arg)
return frozenset(names)
def _leftmost_name(node: ast.AST) -> str | None:
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.BinOp):
return _leftmost_name(node.left)
return None
def _pointer_arg_names(
function_def: ast.FunctionDef,
arg_names: tuple[str, ...],
) -> frozenset[str]:
candidate_names = set(arg_names)
pointer_names = {name for name in arg_names if name.endswith("_ptr")}
for node in ast.walk(function_def):
if not isinstance(node, ast.Call):
continue
if get_ast_full_name(node.func) not in ("tl.load", "tl.store"):
continue
if not node.args:
continue
name = _leftmost_name(node.args[0])
if name in candidate_names:
pointer_names.add(name)
return frozenset(pointer_names)
def trace_triton_kernel_specialization_args(
kernel: Callable[..., Any],
) -> tuple[str, ...]:
function_def = get_function_source_node(kernel)
source_fn = getattr(kernel, "fn", kernel)
arg_names = tuple(inspect.signature(source_fn).parameters)
constexpr_args = _triton_constexpr_arg_names(kernel, function_def, arg_names)
do_not_specialize_args = _triton_do_not_specialize_args(
kernel, function_def, arg_names
)
pointer_args = _pointer_arg_names(function_def, arg_names)
return tuple(
name
for name in arg_names
if name in constexpr_args
or (name not in pointer_args and name not in do_not_specialize_args)
)
+11 -2
View File
@@ -17,6 +17,9 @@ from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
deepseek_v4_mhc_warmup,
)
from vllm.model_executor.warmup.fa4_cutedsl_warmup import (
fa4_cutedsl_warmup,
)
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
resolve_flashinfer_autotune_file,
write_flashinfer_autotune_cache,
@@ -27,7 +30,7 @@ from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
)
from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
from vllm.model_executor.warmup.sparse_mla_triton_warmup import (
sparse_mla_triton_warmup_if_needed,
sparse_mla_triton_warmup,
)
from vllm.model_executor.warmup.v1_block_table_warmup import (
warm_v1_block_table_kernels,
@@ -94,7 +97,6 @@ def kernel_warmup(worker: "Worker"):
)
# Run next so input-prep kernels JIT against pristine runner state.
sparse_mla_triton_warmup_if_needed(worker)
flashinfer_sparse_mla_decode_autotune_warmup(worker)
deepseek_v4_sparse_mla_attention_warmup(worker)
@@ -156,8 +158,15 @@ def kernel_warmup(worker: "Worker"):
)
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
# to the shared JIT warmup infrastructure.
# https://github.com/vllm-project/vllm/pull/47451
cutedsl_warmup()
if worker.vllm_config.kernel_config.enable_jit_warmup:
fa4_cutedsl_warmup(worker)
sparse_mla_triton_warmup(worker)
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
@@ -4,11 +4,10 @@
from typing import TYPE_CHECKING
import torch
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.gpu_worker import Worker
@@ -31,47 +30,7 @@ _GENERIC_SPARSE_MLA_BACKENDS = frozenset(
)
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
_SPARSE_PREFILL_METADATA_NUM_PREFILLS = (1, 2, 4, 8)
_SPARSE_PREFILL_METADATA_NUM_DECODES = (0, 1, 2)
_DSV4_PREFILL_CHUNK_METADATA_COMPRESS_RATIOS = (4, 128)
_PREFILL_CHUNK_METADATA_SEQ_LEN_MULTIPLIERS = (2, 3)
_PREFILL_CHUNK_METADATA_QUERY_SLICE_OFFSETS = (
# query_slice_start offset, query_slice_stop offset
(0, 0),
(0, -1),
(1, 0),
(1, -1),
)
_COMBINE_TOPK_SWA_INPUT_VARIANTS = (
# offset_topk, offset_query_and_seq, offset_gather
(False, False, False),
(False, True, False),
(True, True, True),
)
_DSV4_COMBINE_TOPK_SWA_WARMUP_CASES = (
# compress_ratio, topk, topk_width, N
(1, 0, 512, 512),
(4, 512, 512, 512 * 4),
# DSv4-Pro C4A traffic uses top-k 1024 with N=1024.
(4, 1024, 1024, 1024),
(128, 8192, 8192, 8192 * 128),
# Real C128A traffic also specializes N=1 in one call path.
(128, 8192, 8192, 1),
)
def _clamp_warmup_tokens(num_tokens: int, max_tokens: int) -> int:
return max(0, min(num_tokens, max_tokens))
def _next_power_of_2(x: int) -> int:
return 1 << (x - 1).bit_length()
def _hf_config_int(runner: "GPUModelRunner", name: str, default: int) -> int:
model_config = getattr(runner.vllm_config, "model_config", None)
hf_config = getattr(model_config, "hf_config", None)
return int(getattr(hf_config, name, default) or default)
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
def _attention_backend_name(backend: object) -> str | None:
@@ -96,242 +55,57 @@ def _has_attention_backend(
return False
def _warm_sparse_swa_prefill_metadata_kernel(
device: torch.device,
window_size: int,
prefill_tokens: int,
def _compile_sparse_swa_prefill_metadata_kernel(
vllm_config: "VllmConfig",
) -> None:
from vllm.v1.attention.backends.mla.sparse_swa import (
_compute_prefill_metadata_kernel,
_COMPUTE_PREFILL_METADATA_KERNEL,
)
for num_prefills in _SPARSE_PREFILL_METADATA_NUM_PREFILLS:
for num_decodes in _SPARSE_PREFILL_METADATA_NUM_DECODES:
query_lens = [1] * num_decodes
query_lens += [prefill_tokens] * num_prefills
query_start_locs = [0]
for query_len in query_lens:
query_start_locs.append(query_start_locs[-1] + query_len)
query_start_loc = torch.tensor(
query_start_locs,
dtype=torch.int32,
device=device,
)
seq_lens = torch.tensor(
[1] * num_decodes + [window_size + q for q in query_lens[num_decodes:]],
dtype=torch.int32,
device=device,
)
prefill_gather_lens = torch.empty(
num_prefills, dtype=torch.int32, device=device
)
_compute_prefill_metadata_kernel[(1,)](
prefill_gather_lens,
seq_lens,
query_start_loc,
num_prefills,
num_decodes,
window_size,
BLOCK_SIZE=_next_power_of_2(num_prefills),
)
_COMPUTE_PREFILL_METADATA_KERNEL.warmup(vllm_config)
def _warm_prefill_chunk_metadata_kernel(
device: torch.device,
compress_ratio: int,
query_len: int,
def _compile_prefill_chunk_metadata_kernel(
vllm_config: "VllmConfig",
) -> None:
from vllm.v1.attention.backends.mla.indexer import build_prefill_chunk_metadata
num_reqs = 2
query_start_loc_cpu = torch.arange(
0, (num_reqs + 1) * query_len, query_len, dtype=torch.int32
)
query_start_loc = query_start_loc_cpu.to(device=device)
uncompressed_seq_lens_cpu = torch.tensor(
[
compress_ratio * multiplier + query_len
for multiplier in _PREFILL_CHUNK_METADATA_SEQ_LEN_MULTIPLIERS
],
dtype=torch.int32,
)
compressed_seq_lens_cpu = uncompressed_seq_lens_cpu // compress_ratio
uncompressed_seq_lens = uncompressed_seq_lens_cpu.to(device=device)
compressed_seq_lens = compressed_seq_lens_cpu.to(device=device)
block_table = torch.zeros(
(num_reqs, int(compressed_seq_lens_cpu.max().item())),
dtype=torch.int32,
device=device,
from vllm.v1.attention.backends.mla.indexer import (
_BUILD_PREFILL_CHUNK_METADATA_KERNEL,
)
offset_uncompressed_seq_lens = torch.empty(
num_reqs + 1, dtype=torch.int32, device=device
)[1:]
offset_uncompressed_seq_lens.copy_(uncompressed_seq_lens)
query_slices = tuple(
slice(start, num_reqs * query_len + stop)
for start, stop in _PREFILL_CHUNK_METADATA_QUERY_SLICE_OFFSETS
)
for warmup_uncompressed_seq_lens in (
uncompressed_seq_lens,
offset_uncompressed_seq_lens,
):
for query_slice in query_slices:
build_prefill_chunk_metadata(
0,
num_reqs,
query_start_loc,
query_start_loc_cpu,
warmup_uncompressed_seq_lens,
compressed_seq_lens,
compressed_seq_lens_cpu,
block_table,
compress_ratio,
query_slice=query_slice,
)
_BUILD_PREFILL_CHUNK_METADATA_KERNEL.warmup(vllm_config)
def _warm_combine_topk_swa_indices_kernel(
device: torch.device,
num_tokens: int,
window_size: int,
compress_ratio: int,
topk: int,
topk_width: int,
n: int,
def _compile_combine_topk_swa_indices_kernel(
vllm_config: "VllmConfig",
) -> None:
from vllm.models.deepseek_v4.common.ops.cache_utils import combine_topk_swa_indices
if num_tokens <= 0:
return
def _make_topk_indices(*, offset: bool) -> torch.Tensor:
if offset:
topk_storage = torch.full(
(num_tokens * topk_width + 1,),
-1,
dtype=torch.int32,
device=device,
)
topk_indices = topk_storage[1:].reshape(num_tokens, topk_width)
else:
topk_indices = torch.full(
(num_tokens, topk_width), -1, dtype=torch.int32, device=device
)
if topk > 0:
topk_indices.copy_(
torch.arange(num_tokens * topk_width, dtype=torch.int32, device=device)
.reshape(num_tokens, topk_width)
.remainder(topk_width)
)
return topk_indices
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
seq_lens = torch.tensor(
[window_size + num_tokens], dtype=torch.int32, device=device
)
gather_lens = torch.tensor(
[min(window_size + num_tokens, window_size + num_tokens - 1)],
dtype=torch.int32,
device=device,
)
offset_query_start_loc = torch.empty(3, dtype=torch.int32, device=device)[1:]
offset_query_start_loc.copy_(query_start_loc)
offset_seq_lens = torch.empty(2, dtype=torch.int32, device=device)[1:]
offset_seq_lens.copy_(seq_lens)
offset_gather_lens = torch.empty(2, dtype=torch.int32, device=device)[1:]
offset_gather_lens.copy_(gather_lens)
for (
offset_topk,
offset_query_and_seq,
offset_gather,
) in _COMBINE_TOPK_SWA_INPUT_VARIANTS:
warmup_topk_indices = _make_topk_indices(offset=offset_topk)
warmup_query_start_loc = (
offset_query_start_loc if offset_query_and_seq else query_start_loc
)
warmup_seq_lens = offset_seq_lens if offset_query_and_seq else seq_lens
warmup_gather_lens = offset_gather_lens if offset_gather else gather_lens
n_values = (n,) if n == 1 else (n, n + 1)
for m in (window_size + num_tokens, topk_width):
for n_value in n_values:
combine_topk_swa_indices(
warmup_topk_indices,
warmup_query_start_loc,
warmup_seq_lens,
warmup_gather_lens,
window_size,
compress_ratio,
topk,
M=m,
N=n_value,
)
@torch.inference_mode()
def sparse_mla_triton_warmup(
runner: "GPUModelRunner",
num_tokens: int,
*,
compress_ratios: tuple[int, ...],
combine_topk_swa_cases: tuple[tuple[int, int, int, int], ...] = (),
) -> None:
device = getattr(runner, "device", torch.device("cuda"))
window_size = _hf_config_int(runner, "sliding_window", 128)
_warm_sparse_swa_prefill_metadata_kernel(device, window_size, num_tokens)
for compress_ratio in compress_ratios:
_warm_prefill_chunk_metadata_kernel(device, compress_ratio, num_tokens)
for compress_ratio, topk, topk_width, n in combine_topk_swa_cases:
_warm_combine_topk_swa_indices_kernel(
device,
num_tokens,
window_size,
compress_ratio,
topk,
topk_width,
n,
)
def deepseek_v4_sparse_triton_warmup(
runner: "GPUModelRunner",
num_tokens: int,
) -> None:
sparse_mla_triton_warmup(
runner,
num_tokens,
compress_ratios=_DSV4_PREFILL_CHUNK_METADATA_COMPRESS_RATIOS,
combine_topk_swa_cases=_DSV4_COMBINE_TOPK_SWA_WARMUP_CASES,
from vllm.models.deepseek_v4.common.ops.cache_utils import (
_COMBINE_TOPK_SWA_INDICES_KERNEL,
)
_COMBINE_TOPK_SWA_INDICES_KERNEL.warmup(vllm_config)
def sparse_mla_triton_warmup_if_needed(worker: "Worker") -> None:
def sparse_mla_triton_warmup(worker: "Worker") -> None:
runner = worker.model_runner
if runner.is_pooling_model:
return
max_tokens = worker.scheduler_config.max_num_batched_tokens
num_tokens = _clamp_warmup_tokens(8, max_tokens)
if num_tokens <= 0:
max_num_prefills = min(worker.scheduler_config.max_num_seqs, max_tokens)
if max_tokens <= 0 or max_num_prefills <= 0:
return
vllm_config = runner.vllm_config
try:
if _has_attention_backend(runner, _DEEPSEEK_V4_SPARSE_MLA_BACKENDS):
deepseek_v4_sparse_triton_warmup(runner, num_tokens)
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
_compile_prefill_chunk_metadata_kernel(vllm_config)
_compile_combine_topk_swa_indices_kernel(vllm_config)
elif _has_attention_backend(runner, _GENERIC_SPARSE_MLA_BACKENDS):
sparse_mla_triton_warmup(
runner,
num_tokens,
compress_ratios=(1,),
)
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
_compile_prefill_chunk_metadata_kernel(vllm_config)
elif _has_attention_backend(runner, _INDEXER_PREFILL_CHUNK_METADATA_BACKENDS):
_warm_prefill_chunk_metadata_kernel(
getattr(runner, "device", torch.device("cuda")),
compress_ratio=1,
query_len=num_tokens,
)
_compile_prefill_chunk_metadata_kernel(vllm_config)
except Exception:
logger.warning("Skipping sparse MLA Triton warmup.", exc_info=True)