[torch.compile] Canonicalize functionalized split slices for fusion pa… (#57299)

Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
Laith Sakka
2026-09-28 20:21:59 -04:00
committed by GitHub
parent 171d1b9ef6
commit b39f760cc7
2 changed files with 397 additions and 34 deletions
+214 -6
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import operator
import pytest
import torch
@@ -29,6 +31,33 @@ class SplitCoalescingModel(torch.nn.Module):
return q + 1, k + 2, v + 3
class SliceCoalescingModel(torch.nn.Module):
"""Model with a complete QKV partition represented as contiguous slices."""
def __init__(self, q_size: int, kv_size: int) -> None:
super().__init__()
self.q_size = q_size
self.kv_size = kv_size
def forward(self, qkv: torch.Tensor):
q_end = self.q_size
k_end = q_end + self.kv_size
v_end = k_end + self.kv_size
q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end)
k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end)
v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end)
return q + 1, k + 2, v + 3
def make_vllm_config() -> VllmConfig:
return VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(),
)
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_split_coalescing(dtype):
torch.set_default_device(DEVICE_TYPE)
@@ -37,12 +66,7 @@ def test_split_coalescing(dtype):
q_size, kv_size = 2048, 512
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(),
)
)
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
@@ -63,3 +87,187 @@ def test_split_coalescing(dtype):
torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL)
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_slice_partition_canonicalization(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
q_size, kv_size = 2048, 512
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
model = SliceCoalescingModel(q_size, kv_size)
qkv = torch.randn(5, q_size + 2 * kv_size)
torch._dynamo.mark_dynamic(qkv, 0)
result_eager = model(qkv)
result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3)
assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3
assert backend.op_count(torch.ops.aten.slice.Tensor) == 0
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
split = next(
node
for node in backend.final_graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
)
assert list(split.args[1]) == [q_size, kv_size, kv_size]
assert split.args[2] == -1
getitems = [
node
for node in split.users
if node.op == "call_function" and node.target == operator.getitem
]
assert sorted(node.args[1] for node in getitems) == [0, 1, 2]
assert isinstance(split.meta["val"], list)
for getitem in getitems:
index = getitem.args[1]
assert getitem.meta["val"] is split.meta["val"][index]
def make_slice_graph(
ranges: list[tuple[int, int, int]],
) -> torch.fx.Graph:
graph = torch.fx.Graph()
source = graph.placeholder("qkv")
source.meta["val"] = torch.empty((2, 8), device="meta")
slices = []
for start, end, step in ranges:
node = graph.call_function(
torch.ops.aten.slice.Tensor,
args=(source, -1, start, end, step),
)
node.meta["val"] = torch.empty(
(2, len(range(start, min(end, 8), step))), device="meta"
)
slices.append(node)
graph.output(tuple(slices))
return graph
def test_slice_partition_accepts_safe_group():
graph = make_slice_graph([(0, 4, 1), (4, 8, 1)])
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
assert (
sum(
node.target == torch.ops.aten.split_with_sizes.default
for node in graph.nodes
)
== 1
)
@pytest.mark.parametrize(
"ranges",
[
[(0, 3, 1), (4, 8, 1)], # gap
[(0, 5, 1), (4, 8, 1)], # overlap
[(1, 4, 1), (4, 8, 1)], # missing prefix
[(0, 4, 1), (4, 7, 1)], # missing suffix
[(0, 8, 1)], # single full-range slice
[(0, 8, 2)], # non-unit step
],
)
def test_slice_partition_rejects_unsafe_groups(ranges):
graph = make_slice_graph(ranges)
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
slice_count = sum(
node.target == torch.ops.aten.slice.Tensor for node in graph.nodes
)
split_count = sum(
node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes
)
assert slice_count == len(ranges)
assert split_count == 0
def test_nested_slice_partitions_use_live_source():
"""Use the rewired source when a slice partition consumes another slice."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((2, 8), device="meta")
outer_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4)
)
outer_left.meta["val"] = torch.empty((2, 4), device="meta")
outer_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8)
)
outer_right.meta["val"] = torch.empty((2, 4), device="meta")
inner_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2)
)
inner_left.meta["val"] = torch.empty((2, 2), device="meta")
inner_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4)
)
inner_right.meta["val"] = torch.empty((2, 2), device="meta")
graph.output((inner_left, inner_right, outer_right))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
outer_split = next(node for node in splits if list(node.args[1]) == [4, 4])
inner_split = next(node for node in splits if list(node.args[1]) == [2, 2])
assert outer_split.args[0] is source
assert inner_split.args[0].target == operator.getitem
assert inner_split.args[0].args == (outer_split, 0)
def test_split_coalescing_preserves_different_dimensions():
"""Keep equal-sized splits on different dimensions as separate nodes."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((8, 8), device="meta")
outputs = []
for dim in (0, 1):
split = graph.call_function(
torch.ops.aten.split_with_sizes.default,
args=(source, [4, 4], dim),
)
for index in range(2):
outputs.append(graph.call_function(operator.getitem, args=(split, index)))
graph.output(tuple(outputs))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
assert all(split.args[0] is source for split in splits)
assert {split.args[2] for split in splits} == {0, 1}
@@ -1,21 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
input tensor with the same split sizes.
"""Canonicalize tensor partitions to a single ``split_with_sizes`` node.
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
graph may contain multiple ``split_with_sizes`` calls on the same tensor
that CSE fails to merge. This pass detects and replaces the duplicates
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
see a single split node with all users attached.
Functionalization may replay the individual outputs of a multi-output split as
contiguous ``slice`` nodes. This pass converts complete slice partitions back
to ``split_with_sizes`` and coalesces equivalent split nodes so downstream
pattern-matching passes see one canonical split with all users attached.
See Also:
- vLLM #33295 (original issue)
- vLLM #33295 (original duplicate-split issue)
- PyTorch #174472 (upstream CSE gap)
- PyTorch #194037 (multi-output views replayed as individual slices)
"""
import operator
from collections import defaultdict
from copy import copy
from dataclasses import dataclass
from typing import Any
import torch
from torch import fx
@@ -28,35 +31,177 @@ from ..vllm_inductor_pass import VllmInductorPass
logger = init_logger(__name__)
@dataclass(frozen=True)
class _SliceInfo:
node: fx.Node
dim: int
start: int
end: int
class SplitCoalescingPass(VllmInductorPass):
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
node when they share the same input tensor and split sizes."""
"""Canonicalize tensor partitions to one shared ``split_with_sizes``.
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
count = 0
A complete contiguous slice partition is rewritten from::
# Map from input tensor node -> list of split nodes seen so far.
split_nodes: dict[fx.Node, list[fx.Node]] = {}
q = slice(qkv, 0, q_size)
k = slice(qkv, q_size, q_size + kv_size)
v = slice(qkv, q_size + kv_size, q_size + 2 * kv_size)
for node in graph.nodes:
if not is_func(node, torch.ops.aten.split_with_sizes.default):
to::
qkv_split = split_with_sizes(qkv, [q_size, kv_size, kv_size])
q = qkv_split[0]
k = qkv_split[1]
v = qkv_split[2]
Equivalent splits are also coalesced from::
q = split_with_sizes(qkv, sizes)[0]
k = split_with_sizes(qkv, sizes)[1]
to::
qkv_split = split_with_sizes(qkv, sizes)
q = qkv_split[0]
k = qkv_split[1]
"""
@staticmethod
def _get_arg(node: fx.Node, index: int, name: str, default: Any) -> Any:
"""Read an FX argument by position, keyword, or schema default."""
if len(node.args) > index:
return node.args[index]
return node.kwargs.get(name, default)
@staticmethod
def _normalize_dim(dim: int, rank: int) -> int:
return dim + rank if dim < 0 else dim
@staticmethod
def _tensor_val(node: fx.Node) -> torch.Tensor | None:
value = node.meta.get("val")
return value if isinstance(value, torch.Tensor) else None
def _slice_info(self, node: fx.Node) -> _SliceInfo | None:
source = node.args[0]
source_val = self._tensor_val(source)
slice_val = self._tensor_val(node)
# in case someone forgot to set meta !
if source_val is None or slice_val is None:
return None
dim = self._normalize_dim(self._get_arg(node, 1, "dim", 0), source_val.dim())
extent = source_val.shape[dim]
# do not support dynamic slices.
if not isinstance(extent, int):
return None
start = self._get_arg(node, 2, "start", None)
end = self._get_arg(node, 3, "end", None)
step = self._get_arg(node, 4, "step", 1)
start = 0 if start is None else start
end = extent if end is None else end
if not isinstance(start, int) or not isinstance(end, int) or step != 1:
return None
if start < 0 or end < 0:
return None
# aten.slice clamps an oversized end to the selected dimension.
end = min(end, extent)
if start >= end:
return None
return _SliceInfo(node=node, dim=dim, start=start, end=end)
def _canonicalize_slices(self, graph: fx.Graph) -> int:
groups: dict[tuple[fx.Node, int], list[_SliceInfo]] = defaultdict(list)
for node in graph.find_nodes(
op="call_function", target=torch.ops.aten.slice.Tensor
):
slice_info = self._slice_info(node)
if slice_info is None:
continue
groups[(node.args[0], slice_info.dim)].append(slice_info)
count = 0
for slice_group in groups.values():
if len(slice_group) < 2:
continue
source = slice_group[0].node.args[0]
dim = slice_group[0].dim
source_val = self._tensor_val(source)
assert source_val is not None
extent = source_val.shape[dim]
assert isinstance(extent, int)
ordered = sorted(slice_group, key=lambda info: info.start)
# Ensure they are contiguous partitions.
if (
ordered[0].start != 0
or ordered[-1].end != extent
or any(
left.end != right.start for left, right in zip(ordered, ordered[1:])
)
):
continue
split_sizes = [info.end - info.start for info in ordered]
# Use -1 for the last dimension; otherwise use the positive dimension index.
split_dim = -1 if dim == source_val.dim() - 1 else dim
first_slice = slice_group[0].node
with graph.inserting_before(first_slice):
split = graph.call_function(
torch.ops.aten.split_with_sizes.default,
args=(source, split_sizes, split_dim),
)
split.meta["val"] = [info.node.meta["val"] for info in ordered]
replacements: list[tuple[fx.Node, fx.Node]] = []
for index, info in enumerate(ordered):
getitem = graph.call_function(
operator.getitem,
args=(split, index),
type_expr=info.node.type,
)
getitem.meta = copy(info.node.meta)
replacements.append((info.node, getitem))
for slice_node, getitem in replacements:
slice_node.replace_all_uses_with(getitem)
graph.erase_node(slice_node)
count += 1
return count
def _coalesce_splits(self, graph: fx.Graph) -> int:
count = 0
split_nodes: dict[tuple[fx.Node, int], list[fx.Node]] = defaultdict(list)
for node in graph.find_nodes(
op="call_function", target=torch.ops.aten.split_with_sizes.default
):
if not all(is_func(user, operator.getitem) for user in node.users):
continue
arg_node, split_sizes = node.args[:2]
if arg_node not in split_nodes:
split_nodes[arg_node] = [node]
source, split_sizes = node.args[:2]
source_val = self._tensor_val(source)
if source_val is None:
continue
# Find existing node with same split_sizes
raw_dim = self._get_arg(node, 2, "dim", 0)
dim = self._normalize_dim(raw_dim, source_val.dim())
candidates = split_nodes[(source, dim)]
canonical = next(
(
n
for n in split_nodes[arg_node]
if list(n.args[1]) == list(split_sizes)
candidate
for candidate in candidates
if list(candidate.args[1]) == list(split_sizes)
),
None,
)
@@ -65,6 +210,16 @@ class SplitCoalescingPass(VllmInductorPass):
graph.erase_node(node)
count += 1
else:
split_nodes[arg_node].append(node)
candidates.append(node)
logger.debug("Coalesced %d duplicate split_with_sizes nodes", count)
return count
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
canonicalized = self._canonicalize_slices(graph)
coalesced = self._coalesce_splits(graph)
logger.debug(
"Canonicalized %d slice partitions and coalesced %d split nodes",
canonicalized,
coalesced,
)