[torch.compile] Canonicalize functionalized split slices for fusion pa… (#57299)

Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
Laith Sakka
2026-09-28 20:21:59 -04:00
committed by GitHub
parent 171d1b9ef6
commit b39f760cc7
2 changed files with 397 additions and 34 deletions
@@ -1,21 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
input tensor with the same split sizes.
"""Canonicalize tensor partitions to a single ``split_with_sizes`` node.
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
graph may contain multiple ``split_with_sizes`` calls on the same tensor
that CSE fails to merge. This pass detects and replaces the duplicates
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
see a single split node with all users attached.
Functionalization may replay the individual outputs of a multi-output split as
contiguous ``slice`` nodes. This pass converts complete slice partitions back
to ``split_with_sizes`` and coalesces equivalent split nodes so downstream
pattern-matching passes see one canonical split with all users attached.
See Also:
- vLLM #33295 (original issue)
- vLLM #33295 (original duplicate-split issue)
- PyTorch #174472 (upstream CSE gap)
- PyTorch #194037 (multi-output views replayed as individual slices)
"""
import operator
from collections import defaultdict
from copy import copy
from dataclasses import dataclass
from typing import Any
import torch
from torch import fx
@@ -28,35 +31,177 @@ from ..vllm_inductor_pass import VllmInductorPass
logger = init_logger(__name__)
@dataclass(frozen=True)
class _SliceInfo:
node: fx.Node
dim: int
start: int
end: int
class SplitCoalescingPass(VllmInductorPass):
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
node when they share the same input tensor and split sizes."""
"""Canonicalize tensor partitions to one shared ``split_with_sizes``.
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
count = 0
A complete contiguous slice partition is rewritten from::
# Map from input tensor node -> list of split nodes seen so far.
split_nodes: dict[fx.Node, list[fx.Node]] = {}
q = slice(qkv, 0, q_size)
k = slice(qkv, q_size, q_size + kv_size)
v = slice(qkv, q_size + kv_size, q_size + 2 * kv_size)
for node in graph.nodes:
if not is_func(node, torch.ops.aten.split_with_sizes.default):
to::
qkv_split = split_with_sizes(qkv, [q_size, kv_size, kv_size])
q = qkv_split[0]
k = qkv_split[1]
v = qkv_split[2]
Equivalent splits are also coalesced from::
q = split_with_sizes(qkv, sizes)[0]
k = split_with_sizes(qkv, sizes)[1]
to::
qkv_split = split_with_sizes(qkv, sizes)
q = qkv_split[0]
k = qkv_split[1]
"""
@staticmethod
def _get_arg(node: fx.Node, index: int, name: str, default: Any) -> Any:
"""Read an FX argument by position, keyword, or schema default."""
if len(node.args) > index:
return node.args[index]
return node.kwargs.get(name, default)
@staticmethod
def _normalize_dim(dim: int, rank: int) -> int:
return dim + rank if dim < 0 else dim
@staticmethod
def _tensor_val(node: fx.Node) -> torch.Tensor | None:
value = node.meta.get("val")
return value if isinstance(value, torch.Tensor) else None
def _slice_info(self, node: fx.Node) -> _SliceInfo | None:
source = node.args[0]
source_val = self._tensor_val(source)
slice_val = self._tensor_val(node)
# in case someone forgot to set meta !
if source_val is None or slice_val is None:
return None
dim = self._normalize_dim(self._get_arg(node, 1, "dim", 0), source_val.dim())
extent = source_val.shape[dim]
# do not support dynamic slices.
if not isinstance(extent, int):
return None
start = self._get_arg(node, 2, "start", None)
end = self._get_arg(node, 3, "end", None)
step = self._get_arg(node, 4, "step", 1)
start = 0 if start is None else start
end = extent if end is None else end
if not isinstance(start, int) or not isinstance(end, int) or step != 1:
return None
if start < 0 or end < 0:
return None
# aten.slice clamps an oversized end to the selected dimension.
end = min(end, extent)
if start >= end:
return None
return _SliceInfo(node=node, dim=dim, start=start, end=end)
def _canonicalize_slices(self, graph: fx.Graph) -> int:
groups: dict[tuple[fx.Node, int], list[_SliceInfo]] = defaultdict(list)
for node in graph.find_nodes(
op="call_function", target=torch.ops.aten.slice.Tensor
):
slice_info = self._slice_info(node)
if slice_info is None:
continue
groups[(node.args[0], slice_info.dim)].append(slice_info)
count = 0
for slice_group in groups.values():
if len(slice_group) < 2:
continue
source = slice_group[0].node.args[0]
dim = slice_group[0].dim
source_val = self._tensor_val(source)
assert source_val is not None
extent = source_val.shape[dim]
assert isinstance(extent, int)
ordered = sorted(slice_group, key=lambda info: info.start)
# Ensure they are contiguous partitions.
if (
ordered[0].start != 0
or ordered[-1].end != extent
or any(
left.end != right.start for left, right in zip(ordered, ordered[1:])
)
):
continue
split_sizes = [info.end - info.start for info in ordered]
# Use -1 for the last dimension; otherwise use the positive dimension index.
split_dim = -1 if dim == source_val.dim() - 1 else dim
first_slice = slice_group[0].node
with graph.inserting_before(first_slice):
split = graph.call_function(
torch.ops.aten.split_with_sizes.default,
args=(source, split_sizes, split_dim),
)
split.meta["val"] = [info.node.meta["val"] for info in ordered]
replacements: list[tuple[fx.Node, fx.Node]] = []
for index, info in enumerate(ordered):
getitem = graph.call_function(
operator.getitem,
args=(split, index),
type_expr=info.node.type,
)
getitem.meta = copy(info.node.meta)
replacements.append((info.node, getitem))
for slice_node, getitem in replacements:
slice_node.replace_all_uses_with(getitem)
graph.erase_node(slice_node)
count += 1
return count
def _coalesce_splits(self, graph: fx.Graph) -> int:
count = 0
split_nodes: dict[tuple[fx.Node, int], list[fx.Node]] = defaultdict(list)
for node in graph.find_nodes(
op="call_function", target=torch.ops.aten.split_with_sizes.default
):
if not all(is_func(user, operator.getitem) for user in node.users):
continue
arg_node, split_sizes = node.args[:2]
if arg_node not in split_nodes:
split_nodes[arg_node] = [node]
source, split_sizes = node.args[:2]
source_val = self._tensor_val(source)
if source_val is None:
continue
# Find existing node with same split_sizes
raw_dim = self._get_arg(node, 2, "dim", 0)
dim = self._normalize_dim(raw_dim, source_val.dim())
candidates = split_nodes[(source, dim)]
canonical = next(
(
n
for n in split_nodes[arg_node]
if list(n.args[1]) == list(split_sizes)
candidate
for candidate in candidates
if list(candidate.args[1]) == list(split_sizes)
),
None,
)
@@ -65,6 +210,16 @@ class SplitCoalescingPass(VllmInductorPass):
graph.erase_node(node)
count += 1
else:
split_nodes[arg_node].append(node)
candidates.append(node)
logger.debug("Coalesced %d duplicate split_with_sizes nodes", count)
return count
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
canonicalized = self._canonicalize_slices(graph)
coalesced = self._coalesce_splits(graph)
logger.debug(
"Canonicalized %d slice partitions and coalesced %d split nodes",
canonicalized,
coalesced,
)