[torch.compile] Canonicalize functionalized split slices for fusion pa… (#57299)
Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
@@ -1,21 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
input tensor with the same split sizes.
|
||||
"""Canonicalize tensor partitions to a single ``split_with_sizes`` node.
|
||||
|
||||
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
|
||||
graph may contain multiple ``split_with_sizes`` calls on the same tensor
|
||||
that CSE fails to merge. This pass detects and replaces the duplicates
|
||||
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
|
||||
see a single split node with all users attached.
|
||||
Functionalization may replay the individual outputs of a multi-output split as
|
||||
contiguous ``slice`` nodes. This pass converts complete slice partitions back
|
||||
to ``split_with_sizes`` and coalesces equivalent split nodes so downstream
|
||||
pattern-matching passes see one canonical split with all users attached.
|
||||
|
||||
See Also:
|
||||
- vLLM #33295 (original issue)
|
||||
- vLLM #33295 (original duplicate-split issue)
|
||||
- PyTorch #174472 (upstream CSE gap)
|
||||
- PyTorch #194037 (multi-output views replayed as individual slices)
|
||||
|
||||
"""
|
||||
|
||||
import operator
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import fx
|
||||
@@ -28,35 +31,177 @@ from ..vllm_inductor_pass import VllmInductorPass
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SliceInfo:
|
||||
node: fx.Node
|
||||
dim: int
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
class SplitCoalescingPass(VllmInductorPass):
|
||||
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
|
||||
node when they share the same input tensor and split sizes."""
|
||||
"""Canonicalize tensor partitions to one shared ``split_with_sizes``.
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
count = 0
|
||||
A complete contiguous slice partition is rewritten from::
|
||||
|
||||
# Map from input tensor node -> list of split nodes seen so far.
|
||||
split_nodes: dict[fx.Node, list[fx.Node]] = {}
|
||||
q = slice(qkv, 0, q_size)
|
||||
k = slice(qkv, q_size, q_size + kv_size)
|
||||
v = slice(qkv, q_size + kv_size, q_size + 2 * kv_size)
|
||||
|
||||
for node in graph.nodes:
|
||||
if not is_func(node, torch.ops.aten.split_with_sizes.default):
|
||||
to::
|
||||
|
||||
qkv_split = split_with_sizes(qkv, [q_size, kv_size, kv_size])
|
||||
q = qkv_split[0]
|
||||
k = qkv_split[1]
|
||||
v = qkv_split[2]
|
||||
|
||||
Equivalent splits are also coalesced from::
|
||||
|
||||
q = split_with_sizes(qkv, sizes)[0]
|
||||
k = split_with_sizes(qkv, sizes)[1]
|
||||
|
||||
to::
|
||||
|
||||
qkv_split = split_with_sizes(qkv, sizes)
|
||||
q = qkv_split[0]
|
||||
k = qkv_split[1]
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _get_arg(node: fx.Node, index: int, name: str, default: Any) -> Any:
|
||||
"""Read an FX argument by position, keyword, or schema default."""
|
||||
if len(node.args) > index:
|
||||
return node.args[index]
|
||||
return node.kwargs.get(name, default)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_dim(dim: int, rank: int) -> int:
|
||||
return dim + rank if dim < 0 else dim
|
||||
|
||||
@staticmethod
|
||||
def _tensor_val(node: fx.Node) -> torch.Tensor | None:
|
||||
value = node.meta.get("val")
|
||||
return value if isinstance(value, torch.Tensor) else None
|
||||
|
||||
def _slice_info(self, node: fx.Node) -> _SliceInfo | None:
|
||||
source = node.args[0]
|
||||
source_val = self._tensor_val(source)
|
||||
slice_val = self._tensor_val(node)
|
||||
# in case someone forgot to set meta !
|
||||
if source_val is None or slice_val is None:
|
||||
return None
|
||||
|
||||
dim = self._normalize_dim(self._get_arg(node, 1, "dim", 0), source_val.dim())
|
||||
extent = source_val.shape[dim]
|
||||
|
||||
# do not support dynamic slices.
|
||||
if not isinstance(extent, int):
|
||||
return None
|
||||
|
||||
start = self._get_arg(node, 2, "start", None)
|
||||
end = self._get_arg(node, 3, "end", None)
|
||||
step = self._get_arg(node, 4, "step", 1)
|
||||
start = 0 if start is None else start
|
||||
end = extent if end is None else end
|
||||
if not isinstance(start, int) or not isinstance(end, int) or step != 1:
|
||||
return None
|
||||
|
||||
if start < 0 or end < 0:
|
||||
return None
|
||||
|
||||
# aten.slice clamps an oversized end to the selected dimension.
|
||||
end = min(end, extent)
|
||||
if start >= end:
|
||||
return None
|
||||
|
||||
return _SliceInfo(node=node, dim=dim, start=start, end=end)
|
||||
|
||||
def _canonicalize_slices(self, graph: fx.Graph) -> int:
|
||||
groups: dict[tuple[fx.Node, int], list[_SliceInfo]] = defaultdict(list)
|
||||
|
||||
for node in graph.find_nodes(
|
||||
op="call_function", target=torch.ops.aten.slice.Tensor
|
||||
):
|
||||
slice_info = self._slice_info(node)
|
||||
if slice_info is None:
|
||||
continue
|
||||
groups[(node.args[0], slice_info.dim)].append(slice_info)
|
||||
|
||||
count = 0
|
||||
for slice_group in groups.values():
|
||||
if len(slice_group) < 2:
|
||||
continue
|
||||
|
||||
source = slice_group[0].node.args[0]
|
||||
dim = slice_group[0].dim
|
||||
source_val = self._tensor_val(source)
|
||||
assert source_val is not None
|
||||
extent = source_val.shape[dim]
|
||||
assert isinstance(extent, int)
|
||||
|
||||
ordered = sorted(slice_group, key=lambda info: info.start)
|
||||
# Ensure they are contiguous partitions.
|
||||
if (
|
||||
ordered[0].start != 0
|
||||
or ordered[-1].end != extent
|
||||
or any(
|
||||
left.end != right.start for left, right in zip(ordered, ordered[1:])
|
||||
)
|
||||
):
|
||||
continue
|
||||
|
||||
split_sizes = [info.end - info.start for info in ordered]
|
||||
# Use -1 for the last dimension; otherwise use the positive dimension index.
|
||||
split_dim = -1 if dim == source_val.dim() - 1 else dim
|
||||
first_slice = slice_group[0].node
|
||||
|
||||
with graph.inserting_before(first_slice):
|
||||
split = graph.call_function(
|
||||
torch.ops.aten.split_with_sizes.default,
|
||||
args=(source, split_sizes, split_dim),
|
||||
)
|
||||
split.meta["val"] = [info.node.meta["val"] for info in ordered]
|
||||
|
||||
replacements: list[tuple[fx.Node, fx.Node]] = []
|
||||
for index, info in enumerate(ordered):
|
||||
getitem = graph.call_function(
|
||||
operator.getitem,
|
||||
args=(split, index),
|
||||
type_expr=info.node.type,
|
||||
)
|
||||
getitem.meta = copy(info.node.meta)
|
||||
replacements.append((info.node, getitem))
|
||||
|
||||
for slice_node, getitem in replacements:
|
||||
slice_node.replace_all_uses_with(getitem)
|
||||
graph.erase_node(slice_node)
|
||||
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
def _coalesce_splits(self, graph: fx.Graph) -> int:
|
||||
count = 0
|
||||
split_nodes: dict[tuple[fx.Node, int], list[fx.Node]] = defaultdict(list)
|
||||
|
||||
for node in graph.find_nodes(
|
||||
op="call_function", target=torch.ops.aten.split_with_sizes.default
|
||||
):
|
||||
if not all(is_func(user, operator.getitem) for user in node.users):
|
||||
continue
|
||||
|
||||
arg_node, split_sizes = node.args[:2]
|
||||
|
||||
if arg_node not in split_nodes:
|
||||
split_nodes[arg_node] = [node]
|
||||
source, split_sizes = node.args[:2]
|
||||
source_val = self._tensor_val(source)
|
||||
if source_val is None:
|
||||
continue
|
||||
|
||||
# Find existing node with same split_sizes
|
||||
raw_dim = self._get_arg(node, 2, "dim", 0)
|
||||
dim = self._normalize_dim(raw_dim, source_val.dim())
|
||||
candidates = split_nodes[(source, dim)]
|
||||
canonical = next(
|
||||
(
|
||||
n
|
||||
for n in split_nodes[arg_node]
|
||||
if list(n.args[1]) == list(split_sizes)
|
||||
candidate
|
||||
for candidate in candidates
|
||||
if list(candidate.args[1]) == list(split_sizes)
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -65,6 +210,16 @@ class SplitCoalescingPass(VllmInductorPass):
|
||||
graph.erase_node(node)
|
||||
count += 1
|
||||
else:
|
||||
split_nodes[arg_node].append(node)
|
||||
candidates.append(node)
|
||||
|
||||
logger.debug("Coalesced %d duplicate split_with_sizes nodes", count)
|
||||
return count
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
canonicalized = self._canonicalize_slices(graph)
|
||||
coalesced = self._coalesce_splits(graph)
|
||||
logger.debug(
|
||||
"Canonicalized %d slice partitions and coalesced %d split nodes",
|
||||
canonicalized,
|
||||
coalesced,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user