[Feat][Perf] Add new warmup infrastructure for JITs (#47451)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fbfe58133d
commit
7ca017778f
@@ -2,6 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Run registered CuTeDSL warmup compile units."""
|
||||
|
||||
# TODO(roberto): Remove this compatibility registry after registered CuTeDSL
|
||||
# warmups are migrated to the shared JIT warmup infrastructure.
|
||||
# https://github.com/vllm-project/vllm/pull/47451
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FA4 MLA prefill CuTeDSL compile warmup config."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Hashable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
FlashAttentionCuTeDSLCompileSpec,
|
||||
)
|
||||
|
||||
FA4ArchitectureFamily = Literal["sm90", "sm100f", "sm120"]
|
||||
|
||||
FA4_STANDARD_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
# Current vLLM MLA prefill expands K/V to num_heads before FA4, so this plan
|
||||
# covers qhead_per_kvhead=1.
|
||||
# Batch is not a current FA4 MLA-prefill key field. Use b1 for compile-only
|
||||
# specs because it is the conservative case for Split-KV shape heuristics.
|
||||
# TODO(roberto): FA4 also has direct-GQA and qv/top-k absorbed-MLA paths, but vLLM
|
||||
# does not use them in this backend yet; they need a separate
|
||||
# num_kv_heads/qv/top-k-aware warmup plan if wired in later.
|
||||
FA4_MLA_PREFILL_COMPILE_BATCH_SIZE = 1
|
||||
FA4_MLA_PREFILL_Q_TILE = 128
|
||||
FA4_MLA_PREFILL_K_TILE = 128
|
||||
FA4_MLA_PREFILL_LONG_K_BLOCKS = 32
|
||||
FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS = 64
|
||||
FA4_MLA_PREFILL_CAUSAL_OPTIONS = (False, True)
|
||||
FA4_MLA_PREFILL_LSE_OPTIONS = (False, True)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FA4MLAPrefillCompileContext:
|
||||
dtype: torch.dtype
|
||||
num_heads: int
|
||||
qk_head_dim: int
|
||||
v_head_dim: int
|
||||
kv_nope_head_dim: int
|
||||
requires_v_padding: bool
|
||||
scale: float
|
||||
num_splits: int
|
||||
fa_version: int
|
||||
|
||||
# Return the V head dim FA4 sees.
|
||||
@property
|
||||
def effective_v_head_dim(self) -> int:
|
||||
if self.requires_v_padding:
|
||||
return self.qk_head_dim
|
||||
return self.v_head_dim
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FA4MLAPrefillCompileRequest:
|
||||
"""One compile-only FA4 MLA prefill request."""
|
||||
|
||||
key: Hashable
|
||||
compile_spec: FlashAttentionCuTeDSLCompileSpec
|
||||
|
||||
# Compile this request.
|
||||
def compile(self) -> None:
|
||||
self.compile_spec.compile()
|
||||
|
||||
|
||||
# Yield deduped compile requests.
|
||||
def iter_fa4_mla_prefill_compile_requests(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
) -> Iterator[FA4MLAPrefillCompileRequest]:
|
||||
"""Yield compile requests for this fixed MLA backend.
|
||||
|
||||
FA4 dedupes duplicate atomic kernel selections in its own JIT cache.
|
||||
"""
|
||||
seen: set[Hashable] = set()
|
||||
for compile_spec in iter_fa4_mla_prefill_compile_specs(ctx):
|
||||
key = compile_spec.request_key()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
yield FA4MLAPrefillCompileRequest(
|
||||
key=key,
|
||||
compile_spec=compile_spec,
|
||||
)
|
||||
|
||||
|
||||
# Build compile specs for this setup.
|
||||
def iter_fa4_mla_prefill_compile_specs(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
) -> Iterator[FlashAttentionCuTeDSLCompileSpec]:
|
||||
"""Yield compile-only FA4 MLA prefill requests for this fixed setup."""
|
||||
|
||||
arch_family = _fa4_architecture_family_from_compute_capability(
|
||||
*torch.cuda.get_device_capability()
|
||||
)
|
||||
if not _supports_fa4_mla_prefill(ctx, arch_family):
|
||||
return
|
||||
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
FlashAttentionCuTeDSLCompileSpec,
|
||||
)
|
||||
|
||||
batch_size = FA4_MLA_PREFILL_COMPILE_BATCH_SIZE
|
||||
v_stride = None
|
||||
if not ctx.requires_v_padding:
|
||||
v_stride = (
|
||||
ctx.num_heads * ctx.kv_nope_head_dim,
|
||||
ctx.kv_nope_head_dim,
|
||||
1,
|
||||
)
|
||||
|
||||
for _, max_seqlen_q, max_seqlen_k in _shape_probes_for_context(ctx, arch_family):
|
||||
total_q_tokens = batch_size * max_seqlen_q
|
||||
total_kv_tokens = batch_size * max_seqlen_k
|
||||
for causal in FA4_MLA_PREFILL_CAUSAL_OPTIONS:
|
||||
for return_lse in FA4_MLA_PREFILL_LSE_OPTIONS:
|
||||
yield FlashAttentionCuTeDSLCompileSpec(
|
||||
q_shape=(total_q_tokens, ctx.num_heads, ctx.qk_head_dim),
|
||||
k_shape=(total_kv_tokens, ctx.num_heads, ctx.qk_head_dim),
|
||||
v_shape=(
|
||||
total_kv_tokens,
|
||||
ctx.num_heads,
|
||||
ctx.effective_v_head_dim,
|
||||
),
|
||||
v_stride=v_stride,
|
||||
q_dtype=ctx.dtype,
|
||||
cu_seqlens_q_shape=(batch_size + 1,),
|
||||
cu_seqlens_k_shape=(batch_size + 1,),
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
max_seqlen_k=max_seqlen_k,
|
||||
softmax_scale=ctx.scale,
|
||||
causal=causal,
|
||||
return_softmax_lse=return_lse,
|
||||
num_splits=ctx.num_splits,
|
||||
fa_version=ctx.fa_version,
|
||||
)
|
||||
|
||||
|
||||
# Pick one q/k point per current FA4 MLA-prefill shape regime.
|
||||
def _shape_probes_for_context(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
arch_family: FA4ArchitectureFamily,
|
||||
) -> tuple[tuple[str, int, int], ...]:
|
||||
q_stage1_q = 1
|
||||
q_stage2_q = FA4_MLA_PREFILL_Q_TILE + 1
|
||||
# FA4 never auto-splits when ceil(max_seqlen_k / tile_n) <= 4.
|
||||
no_split_k = 4 * FA4_MLA_PREFILL_K_TILE
|
||||
long_k = FA4_MLA_PREFILL_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
|
||||
# Diff-head-dim Blackwell Split-KV switches tile_n at 64 K blocks.
|
||||
very_long_k = FA4_MLA_PREFILL_VERY_LONG_K_BLOCKS * FA4_MLA_PREFILL_K_TILE
|
||||
|
||||
base_probes = (
|
||||
("q_stage1", q_stage1_q, FA4_MLA_PREFILL_K_TILE),
|
||||
("q_stage2", q_stage2_q, no_split_k),
|
||||
)
|
||||
# SM120 currently rejects Split-KV in FA4; num_splits=1 also has no split
|
||||
# shape regimes on any architecture.
|
||||
if ctx.num_splits == 1 or arch_family == "sm120":
|
||||
return base_probes
|
||||
|
||||
long_k_probes = (
|
||||
("q_stage1_long_k", q_stage1_q, long_k),
|
||||
("q_stage2_long_k", q_stage2_q, long_k),
|
||||
)
|
||||
|
||||
# SM90 does not have the SM100 q_stage or diff-head-dim tile_n=64 branch.
|
||||
# Same-dim SM100-family MLA also does not need the very-long-K probe.
|
||||
if arch_family == "sm90" or ctx.qk_head_dim == ctx.effective_v_head_dim:
|
||||
return (*base_probes, *long_k_probes)
|
||||
|
||||
very_long_k_probes = (
|
||||
("q_stage1_very_long_k", q_stage1_q, very_long_k),
|
||||
("q_stage2_very_long_k", q_stage2_q, very_long_k),
|
||||
)
|
||||
return (*base_probes, *long_k_probes, *very_long_k_probes)
|
||||
|
||||
|
||||
# Check whether this setup can use FA4 MLA prefill.
|
||||
def _supports_fa4_mla_prefill(
|
||||
ctx: FA4MLAPrefillCompileContext,
|
||||
arch_family: FA4ArchitectureFamily,
|
||||
) -> bool:
|
||||
return (
|
||||
ctx.dtype in FA4_STANDARD_DTYPES
|
||||
and ctx.num_heads > 0
|
||||
and (arch_family != "sm120" or ctx.num_splits == 1)
|
||||
)
|
||||
|
||||
|
||||
# Map CUDA capability to the FA4 arch family used by warmup checks.
|
||||
def _fa4_architecture_family_from_compute_capability(
|
||||
major: int,
|
||||
minor: int,
|
||||
) -> FA4ArchitectureFamily:
|
||||
if (major, minor) == (9, 0):
|
||||
return "sm90"
|
||||
if major == 10:
|
||||
return "sm100f"
|
||||
if (major, minor) == (12, 0):
|
||||
return "sm120"
|
||||
raise ValueError(f"FA4 warmup does not know CUDA capability {major}.{minor}")
|
||||
@@ -0,0 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Warm up FA4 CuTeDSL MLA prefill compile keys."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
|
||||
def fa4_cutedsl_warmup(worker: Worker) -> None:
|
||||
runner = worker.model_runner
|
||||
if runner.is_pooling_model:
|
||||
return
|
||||
|
||||
vllm_config = runner.vllm_config
|
||||
if not vllm_config.model_config.use_mla:
|
||||
return
|
||||
|
||||
backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
if backend_cls.get_name() != "FLASH_ATTN":
|
||||
return
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill import flash_attn
|
||||
|
||||
flash_attn.FA4_MLA_PREFILL_KERNEL.warmup(vllm_config)
|
||||
@@ -0,0 +1,474 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared interfaces and tracing helpers for explicit JIT warmup keys."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import itertools
|
||||
import operator
|
||||
import textwrap
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Generic, TypeVar
|
||||
|
||||
__all__ = [
|
||||
"VllmJitKernel",
|
||||
"WarmupIntRange",
|
||||
"get_ast_full_name",
|
||||
"get_function_source_node",
|
||||
"zip_inputs",
|
||||
]
|
||||
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WarmupIntRange:
|
||||
start: int
|
||||
stop: int
|
||||
step: int = 1
|
||||
|
||||
|
||||
WarmupValues = Any
|
||||
CompileKeyDispatchFn = Callable[..., CompileKeyT]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _WarmupInputRows:
|
||||
"""Warmup dispatch inputs expanded in lockstep."""
|
||||
|
||||
rows: tuple[Mapping[str, WarmupValues], ...]
|
||||
|
||||
|
||||
def _expand_warmup_values(values: WarmupValues) -> tuple[Any, ...]:
|
||||
if isinstance(values, WarmupIntRange):
|
||||
return tuple(range(values.start, values.stop, values.step))
|
||||
if isinstance(values, (list, tuple)):
|
||||
return tuple(values)
|
||||
return (values,)
|
||||
|
||||
|
||||
def zip_inputs(*rows: Mapping[str, WarmupValues]) -> _WarmupInputRows:
|
||||
"""Group row-wise dispatch inputs that should be expanded in lockstep."""
|
||||
if not rows:
|
||||
raise ValueError("zip_inputs requires at least one dispatch input row")
|
||||
if not all(isinstance(row, Mapping) for row in rows):
|
||||
raise ValueError("zip_inputs rows must be mappings")
|
||||
|
||||
first_names = frozenset(rows[0])
|
||||
if not first_names:
|
||||
raise ValueError("zip_inputs rows require at least one dispatch input name")
|
||||
if not all(isinstance(name, str) for name in first_names):
|
||||
raise ValueError("zip_inputs dispatch input names must be strings")
|
||||
|
||||
input_rows: list[Mapping[str, WarmupValues]] = []
|
||||
for row in rows:
|
||||
names = frozenset(row)
|
||||
if names != first_names:
|
||||
raise ValueError("zip_inputs rows must use the same dispatch input names")
|
||||
input_rows.append(dict(row))
|
||||
|
||||
return _WarmupInputRows(rows=tuple(input_rows))
|
||||
|
||||
|
||||
def _expand_warmup_value_grid(
|
||||
values: Mapping[str, WarmupValues],
|
||||
input_names: frozenset[str],
|
||||
) -> tuple[dict[str, Any], ...]:
|
||||
names = tuple(name for name in values if name in input_names)
|
||||
if not names:
|
||||
return ({},)
|
||||
|
||||
expanded_values = tuple(_expand_warmup_values(values[name]) for name in names)
|
||||
return tuple(
|
||||
dict(zip(names, value_set)) for value_set in itertools.product(*expanded_values)
|
||||
)
|
||||
|
||||
|
||||
def _expand_warmup_input_rows(
|
||||
rows: tuple[Mapping[str, WarmupValues], ...],
|
||||
input_names: frozenset[str],
|
||||
) -> tuple[dict[str, Any], ...]:
|
||||
active_names = frozenset(name for name in rows[0] if name in input_names)
|
||||
if not active_names:
|
||||
return ({},)
|
||||
|
||||
return tuple(
|
||||
{name: value for name, value in row.items() if name in active_names}
|
||||
for row in rows
|
||||
)
|
||||
|
||||
|
||||
def _merge_warmup_kwargs(parts: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
merged: dict[str, Any] = {}
|
||||
for part in parts:
|
||||
for name, value in part.items():
|
||||
if name in merged:
|
||||
raise ValueError(
|
||||
f"Warmup dispatch input '{name}' is specified more than once"
|
||||
)
|
||||
merged[name] = value
|
||||
return merged
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _CompileKeyDispatchTrace:
|
||||
local_exprs: tuple[tuple[str, ast.AST], ...]
|
||||
field_exprs: tuple[tuple[str, ast.AST], ...]
|
||||
globals: Mapping[str, Any]
|
||||
input_names: frozenset[str]
|
||||
defaults: Mapping[str, Any]
|
||||
|
||||
def compile_key(
|
||||
self,
|
||||
compile_key_type: type[CompileKeyT],
|
||||
kwargs: Mapping[str, Any],
|
||||
) -> CompileKeyT:
|
||||
dispatch_values = {**self.defaults, **kwargs}
|
||||
for name, expr in self.local_exprs:
|
||||
dispatch_values[name] = _eval_dispatch_expr(
|
||||
expr, dispatch_values, self.globals
|
||||
)
|
||||
return compile_key_type(
|
||||
**{
|
||||
field: _eval_dispatch_expr(expr, dispatch_values, self.globals)
|
||||
for field, expr in self.field_exprs
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_BIN_OPS: dict[type[ast.operator], Callable[[Any, Any], Any]] = {
|
||||
ast.Add: operator.add,
|
||||
ast.Sub: operator.sub,
|
||||
ast.Mult: operator.mul,
|
||||
ast.FloorDiv: operator.floordiv,
|
||||
ast.Mod: operator.mod,
|
||||
ast.Pow: operator.pow,
|
||||
}
|
||||
_CMP_OPS: dict[type[ast.cmpop], Callable[[Any, Any], bool]] = {
|
||||
ast.Eq: operator.eq,
|
||||
ast.NotEq: operator.ne,
|
||||
ast.Lt: operator.lt,
|
||||
ast.LtE: operator.le,
|
||||
ast.Gt: operator.gt,
|
||||
ast.GtE: operator.ge,
|
||||
}
|
||||
|
||||
|
||||
def _dispatch_expr_source(node: ast.AST) -> str:
|
||||
try:
|
||||
return ast.unparse(node)
|
||||
except Exception:
|
||||
return ast.dump(node)
|
||||
|
||||
|
||||
def _dispatch_expr_error(node: ast.AST, reason: str) -> ValueError:
|
||||
return ValueError(
|
||||
f"{reason}: {_dispatch_expr_source(node)}. "
|
||||
"Supported dispatch expressions are names, constants, attributes, "
|
||||
"tuple/list literals, conditional expressions, comparisons, boolean "
|
||||
"operators, unary not/minus, arithmetic, and calls without **kwargs."
|
||||
)
|
||||
|
||||
|
||||
class _DispatchExprEvaluator(ast.NodeVisitor):
|
||||
def __init__(
|
||||
self,
|
||||
values: Mapping[str, Any],
|
||||
globals_: Mapping[str, Any],
|
||||
) -> None:
|
||||
self.values = values
|
||||
self.globals = globals_
|
||||
|
||||
def eval(self, node: ast.AST) -> Any:
|
||||
return self.visit(node)
|
||||
|
||||
def generic_visit(self, node: ast.AST) -> Any:
|
||||
raise _dispatch_expr_error(node, "Unsupported dispatch expression")
|
||||
|
||||
def visit_Name(self, node: ast.Name) -> Any:
|
||||
if node.id in self.values:
|
||||
return self.values[node.id]
|
||||
if node.id in self.globals:
|
||||
return self.globals[node.id]
|
||||
raise _dispatch_expr_error(node, f"Unknown dispatch name '{node.id}'")
|
||||
|
||||
def visit_Constant(self, node: ast.Constant) -> Any:
|
||||
return node.value
|
||||
|
||||
def visit_IfExp(self, node: ast.IfExp) -> Any:
|
||||
return self.visit(node.body if self.visit(node.test) else node.orelse)
|
||||
|
||||
def visit_Tuple(self, node: ast.Tuple) -> tuple[Any, ...]:
|
||||
return tuple(self.visit(elt) for elt in node.elts)
|
||||
|
||||
def visit_List(self, node: ast.List) -> list[Any]:
|
||||
return [self.visit(elt) for elt in node.elts]
|
||||
|
||||
def visit_BoolOp(self, node: ast.BoolOp) -> Any:
|
||||
if isinstance(node.op, ast.And):
|
||||
result = None
|
||||
for value in node.values:
|
||||
result = self.visit(value)
|
||||
if not result:
|
||||
return result
|
||||
return result
|
||||
if isinstance(node.op, ast.Or):
|
||||
result = None
|
||||
for value in node.values:
|
||||
result = self.visit(value)
|
||||
if result:
|
||||
return result
|
||||
return result
|
||||
raise _dispatch_expr_error(node, "Unsupported dispatch boolean operator")
|
||||
|
||||
def visit_Compare(self, node: ast.Compare) -> bool:
|
||||
left = self.visit(node.left)
|
||||
for op_node, comparator in zip(node.ops, node.comparators):
|
||||
right = self.visit(comparator)
|
||||
op = _CMP_OPS.get(type(op_node))
|
||||
if op is None:
|
||||
raise _dispatch_expr_error(
|
||||
node, "Unsupported dispatch comparison operator"
|
||||
)
|
||||
if not op(left, right):
|
||||
return False
|
||||
left = right
|
||||
return True
|
||||
|
||||
def visit_UnaryOp(self, node: ast.UnaryOp) -> Any:
|
||||
operand = self.visit(node.operand)
|
||||
if isinstance(node.op, ast.Not):
|
||||
return not operand
|
||||
if isinstance(node.op, ast.USub):
|
||||
return -operand
|
||||
raise _dispatch_expr_error(node, "Unsupported dispatch unary operator")
|
||||
|
||||
def visit_BinOp(self, node: ast.BinOp) -> Any:
|
||||
op = _BIN_OPS.get(type(node.op))
|
||||
if op is None:
|
||||
raise _dispatch_expr_error(node, "Unsupported dispatch binary operator")
|
||||
return op(self.visit(node.left), self.visit(node.right))
|
||||
|
||||
def visit_Call(self, node: ast.Call) -> Any:
|
||||
args = [self.visit(arg) for arg in node.args]
|
||||
fn = self.visit(node.func)
|
||||
call_kwargs: dict[str, Any] = {}
|
||||
for keyword in node.keywords:
|
||||
if keyword.arg is None:
|
||||
raise _dispatch_expr_error(
|
||||
node, "Dispatch helper calls cannot use **kwargs"
|
||||
)
|
||||
call_kwargs[keyword.arg] = self.visit(keyword.value)
|
||||
return fn(*args, **call_kwargs)
|
||||
|
||||
def visit_Attribute(self, node: ast.Attribute) -> Any:
|
||||
return getattr(self.visit(node.value), node.attr)
|
||||
|
||||
|
||||
def get_ast_full_name(node: ast.AST) -> str | None:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id
|
||||
if isinstance(node, ast.Attribute):
|
||||
parent = get_ast_full_name(node.value)
|
||||
if parent is not None:
|
||||
return f"{parent}.{node.attr}"
|
||||
return None
|
||||
|
||||
|
||||
def get_function_source_node(fn: Callable[..., Any]) -> ast.FunctionDef:
|
||||
source_fn = getattr(fn, "fn", fn)
|
||||
source = textwrap.dedent(inspect.getsource(source_fn))
|
||||
tree = ast.parse(source)
|
||||
function_defs = [node for node in tree.body if isinstance(node, ast.FunctionDef)]
|
||||
if len(function_defs) != 1:
|
||||
name = getattr(source_fn, "__name__", type(source_fn).__name__)
|
||||
raise ValueError(f"Expected one function in {name}, found {len(function_defs)}")
|
||||
return function_defs[0]
|
||||
|
||||
|
||||
def _eval_dispatch_expr(
|
||||
node: ast.AST,
|
||||
kwargs: Mapping[str, Any],
|
||||
globals_: Mapping[str, Any],
|
||||
) -> Any:
|
||||
return _DispatchExprEvaluator(kwargs, globals_).eval(node)
|
||||
|
||||
|
||||
def _collect_input_names(
|
||||
node: ast.AST,
|
||||
candidate_names: set[str],
|
||||
local_names: set[str] | None = None,
|
||||
) -> set[str]:
|
||||
if local_names is None:
|
||||
local_names = set()
|
||||
return {
|
||||
child.id
|
||||
for child in ast.walk(node)
|
||||
if (
|
||||
isinstance(child, ast.Name)
|
||||
and child.id in candidate_names
|
||||
and child.id not in local_names
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _collect_dispatch_body(
|
||||
fn: CompileKeyDispatchFn[Any],
|
||||
function_def: ast.FunctionDef,
|
||||
) -> tuple[list[tuple[str, ast.AST]], ast.Call]:
|
||||
local_exprs: list[tuple[str, ast.AST]] = []
|
||||
for statement in function_def.body:
|
||||
if (
|
||||
isinstance(statement, ast.Expr)
|
||||
and isinstance(statement.value, ast.Constant)
|
||||
and isinstance(statement.value.value, str)
|
||||
):
|
||||
continue
|
||||
|
||||
if isinstance(statement, ast.Assign):
|
||||
if len(statement.targets) != 1 or not isinstance(
|
||||
statement.targets[0], ast.Name
|
||||
):
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Dispatch assignments must target one local name"
|
||||
)
|
||||
local_exprs.append((statement.targets[0].id, statement.value))
|
||||
continue
|
||||
|
||||
if isinstance(statement, ast.AnnAssign):
|
||||
if statement.value is None:
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Dispatch annotations must assign a value"
|
||||
)
|
||||
if not isinstance(statement.target, ast.Name):
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Dispatch assignments must target one local name"
|
||||
)
|
||||
local_exprs.append((statement.target.id, statement.value))
|
||||
continue
|
||||
|
||||
if isinstance(statement, ast.Return) and isinstance(statement.value, ast.Call):
|
||||
return local_exprs, statement.value
|
||||
|
||||
if isinstance(statement, ast.Return):
|
||||
raise _dispatch_expr_error(
|
||||
statement, "Dispatch must return one CompileKey(...) call"
|
||||
)
|
||||
|
||||
raise _dispatch_expr_error(
|
||||
statement,
|
||||
"Dispatch may only contain local assignments before CompileKey return",
|
||||
)
|
||||
|
||||
raise ValueError(f"Expected {fn.__name__} to return one CompileKey(...) call")
|
||||
|
||||
|
||||
def _trace_compile_key_dispatch(
|
||||
fn: CompileKeyDispatchFn[Any],
|
||||
) -> _CompileKeyDispatchTrace:
|
||||
source_fn = getattr(fn, "__func__", fn)
|
||||
globals_ = source_fn.__globals__
|
||||
function_def = get_function_source_node(fn)
|
||||
|
||||
local_exprs, return_call = _collect_dispatch_body(fn, function_def)
|
||||
|
||||
field_exprs: list[tuple[str, ast.AST]] = []
|
||||
signature = inspect.signature(fn)
|
||||
defaults = {
|
||||
name: parameter.default
|
||||
for name, parameter in signature.parameters.items()
|
||||
if parameter.default is not inspect.Parameter.empty
|
||||
}
|
||||
candidate_names = set(signature.parameters)
|
||||
input_names: set[str] = set()
|
||||
local_names = {name for name, _ in local_exprs}
|
||||
for _, expr in local_exprs:
|
||||
input_names.update(_collect_input_names(expr, candidate_names))
|
||||
for keyword in return_call.keywords:
|
||||
if keyword.arg is None:
|
||||
raise ValueError(f"{fn.__name__} cannot use **kwargs in CompileKey")
|
||||
field_exprs.append((keyword.arg, keyword.value))
|
||||
input_names.update(
|
||||
_collect_input_names(keyword.value, candidate_names, local_names)
|
||||
)
|
||||
|
||||
return _CompileKeyDispatchTrace(
|
||||
tuple(local_exprs),
|
||||
tuple(field_exprs),
|
||||
globals_,
|
||||
frozenset(input_names),
|
||||
defaults,
|
||||
)
|
||||
|
||||
|
||||
class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
"""Kernel wrapper that owns dispatch, warmup keys, and compilation."""
|
||||
|
||||
CompileKey: type[CompileKeyT]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.compile_key_dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
|
||||
|
||||
def compile_key(self, kwargs: Mapping[str, Any]) -> CompileKeyT:
|
||||
return self.compile_key_dispatch_trace.compile_key(self.CompileKey, kwargs)
|
||||
|
||||
def _trace_dispatch(
|
||||
self, dispatch: CompileKeyDispatchFn[CompileKeyT]
|
||||
) -> Callable[..., list[CompileKeyT]]:
|
||||
compile_key_dispatch_trace = _trace_compile_key_dispatch(dispatch)
|
||||
|
||||
def traced(
|
||||
*input_groups: _WarmupInputRows,
|
||||
**kwargs: WarmupValues,
|
||||
) -> list[CompileKeyT]:
|
||||
for group in input_groups:
|
||||
if not isinstance(group, _WarmupInputRows):
|
||||
raise TypeError(
|
||||
"_trace_dispatch positional arguments must be "
|
||||
"zip_inputs(...) groups"
|
||||
)
|
||||
expanded_input_groups = tuple(
|
||||
_expand_warmup_input_rows(
|
||||
group.rows, compile_key_dispatch_trace.input_names
|
||||
)
|
||||
for group in input_groups
|
||||
)
|
||||
expanded_kwargs = _expand_warmup_value_grid(
|
||||
kwargs, compile_key_dispatch_trace.input_names
|
||||
)
|
||||
dispatch_value_groups = (*expanded_input_groups, expanded_kwargs)
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
compile_key_dispatch_trace.compile_key(
|
||||
self.CompileKey, _merge_warmup_kwargs(dispatch_values)
|
||||
)
|
||||
for dispatch_values in itertools.product(*dispatch_value_groups)
|
||||
)
|
||||
)
|
||||
|
||||
return traced
|
||||
|
||||
@abstractmethod
|
||||
def dispatch(self, **kwargs: Any) -> CompileKeyT:
|
||||
"""Build one compile key from one concrete dispatch point."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_warmup_keys(self, *args: Any, **kwargs: Any) -> list[CompileKeyT]:
|
||||
"""Return compile keys that should be warmed for this kernel."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def compile(self, compile_key: CompileKeyT) -> None:
|
||||
"""Compile one warmup key."""
|
||||
raise NotImplementedError
|
||||
|
||||
def warmup(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""Compile this kernel's warmup keys."""
|
||||
for compile_key in self.get_warmup_keys(*args, **kwargs):
|
||||
self.compile(compile_key)
|
||||
@@ -0,0 +1,183 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import ast
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
get_ast_full_name,
|
||||
get_function_source_node,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonWarmupTensor:
|
||||
# Compile-only tensor descriptor for Triton pointer specialization.
|
||||
dtype: Any
|
||||
aligned: bool = True
|
||||
shape: tuple[int, ...] = (1,)
|
||||
|
||||
def data_ptr(self) -> int:
|
||||
return 0 if self.aligned else 1
|
||||
|
||||
def ptr_range(self) -> int:
|
||||
return 0
|
||||
|
||||
def stride(self) -> tuple[int, ...]:
|
||||
strides: list[int] = []
|
||||
stride = 1
|
||||
for size in reversed(self.shape):
|
||||
strides.append(stride)
|
||||
stride *= size
|
||||
return tuple(reversed(strides))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonPointerInputVariant:
|
||||
# Named pointer-alignment variant for compile-only Triton warmup.
|
||||
alignments: tuple[tuple[str, bool], ...]
|
||||
|
||||
@classmethod
|
||||
def from_alignment(cls, **aligned: bool) -> "TritonPointerInputVariant":
|
||||
return cls(tuple(aligned.items()))
|
||||
|
||||
def is_aligned(self, name: str) -> bool:
|
||||
for alignment_name, aligned in self.alignments:
|
||||
if alignment_name == name:
|
||||
return aligned
|
||||
raise KeyError(f"Unknown Triton pointer input variant: {name}")
|
||||
|
||||
def pointer(
|
||||
self,
|
||||
name: str,
|
||||
dtype: Any,
|
||||
shape: tuple[int, ...] = (1,),
|
||||
) -> TritonWarmupTensor:
|
||||
return TritonWarmupTensor(dtype, aligned=self.is_aligned(name), shape=shape)
|
||||
|
||||
|
||||
def _literal_str_refs(node: ast.AST) -> tuple[str | int, ...]:
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str | int):
|
||||
return (node.value,)
|
||||
if isinstance(node, ast.List | ast.Tuple):
|
||||
refs: list[str | int] = []
|
||||
for elt in node.elts:
|
||||
if isinstance(elt, ast.Constant) and isinstance(elt.value, str | int):
|
||||
refs.append(elt.value)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported Triton specialization ref: {ast.dump(elt)}"
|
||||
)
|
||||
return tuple(refs)
|
||||
raise ValueError(f"Unsupported Triton specialization refs: {ast.dump(node)}")
|
||||
|
||||
|
||||
def _normalize_arg_refs(
|
||||
refs: tuple[str | int, ...],
|
||||
arg_names: tuple[str, ...],
|
||||
) -> frozenset[str]:
|
||||
names: set[str] = set()
|
||||
for ref in refs:
|
||||
if isinstance(ref, int):
|
||||
names.add(arg_names[ref])
|
||||
else:
|
||||
names.add(ref)
|
||||
return frozenset(names)
|
||||
|
||||
|
||||
def _decorator_keyword_refs(
|
||||
function_def: ast.FunctionDef,
|
||||
keyword_name: str,
|
||||
) -> tuple[str | int, ...]:
|
||||
for decorator in function_def.decorator_list:
|
||||
if not isinstance(decorator, ast.Call):
|
||||
continue
|
||||
decorator_name = get_ast_full_name(decorator.func)
|
||||
if decorator_name not in ("triton.jit", "jit"):
|
||||
continue
|
||||
for keyword in decorator.keywords:
|
||||
if keyword.arg == keyword_name:
|
||||
return _literal_str_refs(keyword.value)
|
||||
return ()
|
||||
|
||||
|
||||
def _triton_do_not_specialize_args(
|
||||
kernel: Callable[..., Any],
|
||||
function_def: ast.FunctionDef,
|
||||
arg_names: tuple[str, ...],
|
||||
) -> frozenset[str]:
|
||||
refs = getattr(kernel, "do_not_specialize", None)
|
||||
if refs is not None:
|
||||
return _normalize_arg_refs(tuple(refs), arg_names)
|
||||
return _normalize_arg_refs(
|
||||
_decorator_keyword_refs(function_def, "do_not_specialize"),
|
||||
arg_names,
|
||||
)
|
||||
|
||||
|
||||
def _triton_constexpr_arg_names(
|
||||
kernel: Callable[..., Any],
|
||||
function_def: ast.FunctionDef,
|
||||
arg_names: tuple[str, ...],
|
||||
) -> frozenset[str]:
|
||||
constexprs = getattr(kernel, "constexprs", None)
|
||||
if constexprs is not None:
|
||||
return frozenset(arg_names[index] for index in constexprs)
|
||||
|
||||
names: set[str] = set()
|
||||
for arg in function_def.args.args + function_def.args.kwonlyargs:
|
||||
if arg.annotation is None:
|
||||
continue
|
||||
annotation = get_ast_full_name(arg.annotation)
|
||||
if annotation in ("tl.constexpr", "triton.language.constexpr", "constexpr"):
|
||||
names.add(arg.arg)
|
||||
return frozenset(names)
|
||||
|
||||
|
||||
def _leftmost_name(node: ast.AST) -> str | None:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id
|
||||
if isinstance(node, ast.BinOp):
|
||||
return _leftmost_name(node.left)
|
||||
return None
|
||||
|
||||
|
||||
def _pointer_arg_names(
|
||||
function_def: ast.FunctionDef,
|
||||
arg_names: tuple[str, ...],
|
||||
) -> frozenset[str]:
|
||||
candidate_names = set(arg_names)
|
||||
pointer_names = {name for name in arg_names if name.endswith("_ptr")}
|
||||
for node in ast.walk(function_def):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
if get_ast_full_name(node.func) not in ("tl.load", "tl.store"):
|
||||
continue
|
||||
if not node.args:
|
||||
continue
|
||||
name = _leftmost_name(node.args[0])
|
||||
if name in candidate_names:
|
||||
pointer_names.add(name)
|
||||
return frozenset(pointer_names)
|
||||
|
||||
|
||||
def trace_triton_kernel_specialization_args(
|
||||
kernel: Callable[..., Any],
|
||||
) -> tuple[str, ...]:
|
||||
function_def = get_function_source_node(kernel)
|
||||
source_fn = getattr(kernel, "fn", kernel)
|
||||
arg_names = tuple(inspect.signature(source_fn).parameters)
|
||||
constexpr_args = _triton_constexpr_arg_names(kernel, function_def, arg_names)
|
||||
do_not_specialize_args = _triton_do_not_specialize_args(
|
||||
kernel, function_def, arg_names
|
||||
)
|
||||
pointer_args = _pointer_arg_names(function_def, arg_names)
|
||||
|
||||
return tuple(
|
||||
name
|
||||
for name in arg_names
|
||||
if name in constexpr_args
|
||||
or (name not in pointer_args and name not in do_not_specialize_args)
|
||||
)
|
||||
@@ -17,6 +17,9 @@ from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.deepseek_v4_mhc_warmup import (
|
||||
deepseek_v4_mhc_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.fa4_cutedsl_warmup import (
|
||||
fa4_cutedsl_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
write_flashinfer_autotune_cache,
|
||||
@@ -27,7 +30,7 @@ from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
)
|
||||
from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
|
||||
from vllm.model_executor.warmup.sparse_mla_triton_warmup import (
|
||||
sparse_mla_triton_warmup_if_needed,
|
||||
sparse_mla_triton_warmup,
|
||||
)
|
||||
from vllm.model_executor.warmup.v1_block_table_warmup import (
|
||||
warm_v1_block_table_kernels,
|
||||
@@ -94,7 +97,6 @@ def kernel_warmup(worker: "Worker"):
|
||||
)
|
||||
|
||||
# Run next so input-prep kernels JIT against pristine runner state.
|
||||
sparse_mla_triton_warmup_if_needed(worker)
|
||||
flashinfer_sparse_mla_decode_autotune_warmup(worker)
|
||||
deepseek_v4_sparse_mla_attention_warmup(worker)
|
||||
|
||||
@@ -156,8 +158,15 @@ def kernel_warmup(worker: "Worker"):
|
||||
)
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
|
||||
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
|
||||
# to the shared JIT warmup infrastructure.
|
||||
# https://github.com/vllm-project/vllm/pull/47451
|
||||
cutedsl_warmup()
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_jit_warmup:
|
||||
fa4_cutedsl_warmup(worker)
|
||||
sparse_mla_triton_warmup(worker)
|
||||
|
||||
|
||||
def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
if envs.VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS is not None:
|
||||
|
||||
@@ -4,11 +4,10 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
@@ -31,47 +30,7 @@ _GENERIC_SPARSE_MLA_BACKENDS = frozenset(
|
||||
)
|
||||
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
|
||||
|
||||
_SPARSE_PREFILL_METADATA_NUM_PREFILLS = (1, 2, 4, 8)
|
||||
_SPARSE_PREFILL_METADATA_NUM_DECODES = (0, 1, 2)
|
||||
_DSV4_PREFILL_CHUNK_METADATA_COMPRESS_RATIOS = (4, 128)
|
||||
_PREFILL_CHUNK_METADATA_SEQ_LEN_MULTIPLIERS = (2, 3)
|
||||
_PREFILL_CHUNK_METADATA_QUERY_SLICE_OFFSETS = (
|
||||
# query_slice_start offset, query_slice_stop offset
|
||||
(0, 0),
|
||||
(0, -1),
|
||||
(1, 0),
|
||||
(1, -1),
|
||||
)
|
||||
_COMBINE_TOPK_SWA_INPUT_VARIANTS = (
|
||||
# offset_topk, offset_query_and_seq, offset_gather
|
||||
(False, False, False),
|
||||
(False, True, False),
|
||||
(True, True, True),
|
||||
)
|
||||
_DSV4_COMBINE_TOPK_SWA_WARMUP_CASES = (
|
||||
# compress_ratio, topk, topk_width, N
|
||||
(1, 0, 512, 512),
|
||||
(4, 512, 512, 512 * 4),
|
||||
# DSv4-Pro C4A traffic uses top-k 1024 with N=1024.
|
||||
(4, 1024, 1024, 1024),
|
||||
(128, 8192, 8192, 8192 * 128),
|
||||
# Real C128A traffic also specializes N=1 in one call path.
|
||||
(128, 8192, 8192, 1),
|
||||
)
|
||||
|
||||
|
||||
def _clamp_warmup_tokens(num_tokens: int, max_tokens: int) -> int:
|
||||
return max(0, min(num_tokens, max_tokens))
|
||||
|
||||
|
||||
def _next_power_of_2(x: int) -> int:
|
||||
return 1 << (x - 1).bit_length()
|
||||
|
||||
|
||||
def _hf_config_int(runner: "GPUModelRunner", name: str, default: int) -> int:
|
||||
model_config = getattr(runner.vllm_config, "model_config", None)
|
||||
hf_config = getattr(model_config, "hf_config", None)
|
||||
return int(getattr(hf_config, name, default) or default)
|
||||
_INDEXER_PREFILL_CHUNK_METADATA_BACKENDS = frozenset({"DEEPSEEK_V32_INDEXER"})
|
||||
|
||||
|
||||
def _attention_backend_name(backend: object) -> str | None:
|
||||
@@ -96,242 +55,57 @@ def _has_attention_backend(
|
||||
return False
|
||||
|
||||
|
||||
def _warm_sparse_swa_prefill_metadata_kernel(
|
||||
device: torch.device,
|
||||
window_size: int,
|
||||
prefill_tokens: int,
|
||||
def _compile_sparse_swa_prefill_metadata_kernel(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
_compute_prefill_metadata_kernel,
|
||||
_COMPUTE_PREFILL_METADATA_KERNEL,
|
||||
)
|
||||
|
||||
for num_prefills in _SPARSE_PREFILL_METADATA_NUM_PREFILLS:
|
||||
for num_decodes in _SPARSE_PREFILL_METADATA_NUM_DECODES:
|
||||
query_lens = [1] * num_decodes
|
||||
query_lens += [prefill_tokens] * num_prefills
|
||||
query_start_locs = [0]
|
||||
for query_len in query_lens:
|
||||
query_start_locs.append(query_start_locs[-1] + query_len)
|
||||
query_start_loc = torch.tensor(
|
||||
query_start_locs,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
seq_lens = torch.tensor(
|
||||
[1] * num_decodes + [window_size + q for q in query_lens[num_decodes:]],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
prefill_gather_lens = torch.empty(
|
||||
num_prefills, dtype=torch.int32, device=device
|
||||
)
|
||||
_compute_prefill_metadata_kernel[(1,)](
|
||||
prefill_gather_lens,
|
||||
seq_lens,
|
||||
query_start_loc,
|
||||
num_prefills,
|
||||
num_decodes,
|
||||
window_size,
|
||||
BLOCK_SIZE=_next_power_of_2(num_prefills),
|
||||
)
|
||||
_COMPUTE_PREFILL_METADATA_KERNEL.warmup(vllm_config)
|
||||
|
||||
|
||||
def _warm_prefill_chunk_metadata_kernel(
|
||||
device: torch.device,
|
||||
compress_ratio: int,
|
||||
query_len: int,
|
||||
def _compile_prefill_chunk_metadata_kernel(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
from vllm.v1.attention.backends.mla.indexer import build_prefill_chunk_metadata
|
||||
|
||||
num_reqs = 2
|
||||
query_start_loc_cpu = torch.arange(
|
||||
0, (num_reqs + 1) * query_len, query_len, dtype=torch.int32
|
||||
)
|
||||
query_start_loc = query_start_loc_cpu.to(device=device)
|
||||
|
||||
uncompressed_seq_lens_cpu = torch.tensor(
|
||||
[
|
||||
compress_ratio * multiplier + query_len
|
||||
for multiplier in _PREFILL_CHUNK_METADATA_SEQ_LEN_MULTIPLIERS
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
compressed_seq_lens_cpu = uncompressed_seq_lens_cpu // compress_ratio
|
||||
uncompressed_seq_lens = uncompressed_seq_lens_cpu.to(device=device)
|
||||
compressed_seq_lens = compressed_seq_lens_cpu.to(device=device)
|
||||
block_table = torch.zeros(
|
||||
(num_reqs, int(compressed_seq_lens_cpu.max().item())),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
_BUILD_PREFILL_CHUNK_METADATA_KERNEL,
|
||||
)
|
||||
|
||||
offset_uncompressed_seq_lens = torch.empty(
|
||||
num_reqs + 1, dtype=torch.int32, device=device
|
||||
)[1:]
|
||||
offset_uncompressed_seq_lens.copy_(uncompressed_seq_lens)
|
||||
query_slices = tuple(
|
||||
slice(start, num_reqs * query_len + stop)
|
||||
for start, stop in _PREFILL_CHUNK_METADATA_QUERY_SLICE_OFFSETS
|
||||
)
|
||||
for warmup_uncompressed_seq_lens in (
|
||||
uncompressed_seq_lens,
|
||||
offset_uncompressed_seq_lens,
|
||||
):
|
||||
for query_slice in query_slices:
|
||||
build_prefill_chunk_metadata(
|
||||
0,
|
||||
num_reqs,
|
||||
query_start_loc,
|
||||
query_start_loc_cpu,
|
||||
warmup_uncompressed_seq_lens,
|
||||
compressed_seq_lens,
|
||||
compressed_seq_lens_cpu,
|
||||
block_table,
|
||||
compress_ratio,
|
||||
query_slice=query_slice,
|
||||
)
|
||||
_BUILD_PREFILL_CHUNK_METADATA_KERNEL.warmup(vllm_config)
|
||||
|
||||
|
||||
def _warm_combine_topk_swa_indices_kernel(
|
||||
device: torch.device,
|
||||
num_tokens: int,
|
||||
window_size: int,
|
||||
compress_ratio: int,
|
||||
topk: int,
|
||||
topk_width: int,
|
||||
n: int,
|
||||
def _compile_combine_topk_swa_indices_kernel(
|
||||
vllm_config: "VllmConfig",
|
||||
) -> None:
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import combine_topk_swa_indices
|
||||
|
||||
if num_tokens <= 0:
|
||||
return
|
||||
|
||||
def _make_topk_indices(*, offset: bool) -> torch.Tensor:
|
||||
if offset:
|
||||
topk_storage = torch.full(
|
||||
(num_tokens * topk_width + 1,),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
topk_indices = topk_storage[1:].reshape(num_tokens, topk_width)
|
||||
else:
|
||||
topk_indices = torch.full(
|
||||
(num_tokens, topk_width), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
if topk > 0:
|
||||
topk_indices.copy_(
|
||||
torch.arange(num_tokens * topk_width, dtype=torch.int32, device=device)
|
||||
.reshape(num_tokens, topk_width)
|
||||
.remainder(topk_width)
|
||||
)
|
||||
return topk_indices
|
||||
|
||||
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
|
||||
seq_lens = torch.tensor(
|
||||
[window_size + num_tokens], dtype=torch.int32, device=device
|
||||
)
|
||||
gather_lens = torch.tensor(
|
||||
[min(window_size + num_tokens, window_size + num_tokens - 1)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
offset_query_start_loc = torch.empty(3, dtype=torch.int32, device=device)[1:]
|
||||
offset_query_start_loc.copy_(query_start_loc)
|
||||
offset_seq_lens = torch.empty(2, dtype=torch.int32, device=device)[1:]
|
||||
offset_seq_lens.copy_(seq_lens)
|
||||
offset_gather_lens = torch.empty(2, dtype=torch.int32, device=device)[1:]
|
||||
offset_gather_lens.copy_(gather_lens)
|
||||
|
||||
for (
|
||||
offset_topk,
|
||||
offset_query_and_seq,
|
||||
offset_gather,
|
||||
) in _COMBINE_TOPK_SWA_INPUT_VARIANTS:
|
||||
warmup_topk_indices = _make_topk_indices(offset=offset_topk)
|
||||
warmup_query_start_loc = (
|
||||
offset_query_start_loc if offset_query_and_seq else query_start_loc
|
||||
)
|
||||
warmup_seq_lens = offset_seq_lens if offset_query_and_seq else seq_lens
|
||||
warmup_gather_lens = offset_gather_lens if offset_gather else gather_lens
|
||||
n_values = (n,) if n == 1 else (n, n + 1)
|
||||
for m in (window_size + num_tokens, topk_width):
|
||||
for n_value in n_values:
|
||||
combine_topk_swa_indices(
|
||||
warmup_topk_indices,
|
||||
warmup_query_start_loc,
|
||||
warmup_seq_lens,
|
||||
warmup_gather_lens,
|
||||
window_size,
|
||||
compress_ratio,
|
||||
topk,
|
||||
M=m,
|
||||
N=n_value,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def sparse_mla_triton_warmup(
|
||||
runner: "GPUModelRunner",
|
||||
num_tokens: int,
|
||||
*,
|
||||
compress_ratios: tuple[int, ...],
|
||||
combine_topk_swa_cases: tuple[tuple[int, int, int, int], ...] = (),
|
||||
) -> None:
|
||||
device = getattr(runner, "device", torch.device("cuda"))
|
||||
window_size = _hf_config_int(runner, "sliding_window", 128)
|
||||
|
||||
_warm_sparse_swa_prefill_metadata_kernel(device, window_size, num_tokens)
|
||||
for compress_ratio in compress_ratios:
|
||||
_warm_prefill_chunk_metadata_kernel(device, compress_ratio, num_tokens)
|
||||
for compress_ratio, topk, topk_width, n in combine_topk_swa_cases:
|
||||
_warm_combine_topk_swa_indices_kernel(
|
||||
device,
|
||||
num_tokens,
|
||||
window_size,
|
||||
compress_ratio,
|
||||
topk,
|
||||
topk_width,
|
||||
n,
|
||||
)
|
||||
|
||||
|
||||
def deepseek_v4_sparse_triton_warmup(
|
||||
runner: "GPUModelRunner",
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
sparse_mla_triton_warmup(
|
||||
runner,
|
||||
num_tokens,
|
||||
compress_ratios=_DSV4_PREFILL_CHUNK_METADATA_COMPRESS_RATIOS,
|
||||
combine_topk_swa_cases=_DSV4_COMBINE_TOPK_SWA_WARMUP_CASES,
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
_COMBINE_TOPK_SWA_INDICES_KERNEL,
|
||||
)
|
||||
|
||||
_COMBINE_TOPK_SWA_INDICES_KERNEL.warmup(vllm_config)
|
||||
|
||||
def sparse_mla_triton_warmup_if_needed(worker: "Worker") -> None:
|
||||
|
||||
def sparse_mla_triton_warmup(worker: "Worker") -> None:
|
||||
runner = worker.model_runner
|
||||
if runner.is_pooling_model:
|
||||
return
|
||||
|
||||
max_tokens = worker.scheduler_config.max_num_batched_tokens
|
||||
num_tokens = _clamp_warmup_tokens(8, max_tokens)
|
||||
if num_tokens <= 0:
|
||||
max_num_prefills = min(worker.scheduler_config.max_num_seqs, max_tokens)
|
||||
if max_tokens <= 0 or max_num_prefills <= 0:
|
||||
return
|
||||
|
||||
vllm_config = runner.vllm_config
|
||||
try:
|
||||
if _has_attention_backend(runner, _DEEPSEEK_V4_SPARSE_MLA_BACKENDS):
|
||||
deepseek_v4_sparse_triton_warmup(runner, num_tokens)
|
||||
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
|
||||
_compile_prefill_chunk_metadata_kernel(vllm_config)
|
||||
_compile_combine_topk_swa_indices_kernel(vllm_config)
|
||||
elif _has_attention_backend(runner, _GENERIC_SPARSE_MLA_BACKENDS):
|
||||
sparse_mla_triton_warmup(
|
||||
runner,
|
||||
num_tokens,
|
||||
compress_ratios=(1,),
|
||||
)
|
||||
_compile_sparse_swa_prefill_metadata_kernel(vllm_config)
|
||||
_compile_prefill_chunk_metadata_kernel(vllm_config)
|
||||
elif _has_attention_backend(runner, _INDEXER_PREFILL_CHUNK_METADATA_BACKENDS):
|
||||
_warm_prefill_chunk_metadata_kernel(
|
||||
getattr(runner, "device", torch.device("cuda")),
|
||||
compress_ratio=1,
|
||||
query_len=num_tokens,
|
||||
)
|
||||
_compile_prefill_chunk_metadata_kernel(vllm_config)
|
||||
|
||||
except Exception:
|
||||
logger.warning("Skipping sparse MLA Triton warmup.", exc_info=True)
|
||||
|
||||
Reference in New Issue
Block a user