[torch.compile] Canonicalize functionalized split slices for fusion pa… (#57299)
Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import operator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -29,6 +31,33 @@ class SplitCoalescingModel(torch.nn.Module):
|
||||
return q + 1, k + 2, v + 3
|
||||
|
||||
|
||||
class SliceCoalescingModel(torch.nn.Module):
|
||||
"""Model with a complete QKV partition represented as contiguous slices."""
|
||||
|
||||
def __init__(self, q_size: int, kv_size: int) -> None:
|
||||
super().__init__()
|
||||
self.q_size = q_size
|
||||
self.kv_size = kv_size
|
||||
|
||||
def forward(self, qkv: torch.Tensor):
|
||||
q_end = self.q_size
|
||||
k_end = q_end + self.kv_size
|
||||
v_end = k_end + self.kv_size
|
||||
q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end)
|
||||
k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end)
|
||||
v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end)
|
||||
return q + 1, k + 2, v + 3
|
||||
|
||||
|
||||
def make_vllm_config() -> VllmConfig:
|
||||
return VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_split_coalescing(dtype):
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
@@ -37,12 +66,7 @@ def test_split_coalescing(dtype):
|
||||
|
||||
q_size, kv_size = 2048, 512
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(),
|
||||
)
|
||||
)
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
coalesce_pass = SplitCoalescingPass(vllm_config)
|
||||
backend = TestBackend(coalesce_pass)
|
||||
@@ -63,3 +87,187 @@ def test_split_coalescing(dtype):
|
||||
torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL)
|
||||
|
||||
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_slice_partition_canonicalization(dtype):
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
q_size, kv_size = 2048, 512
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
coalesce_pass = SplitCoalescingPass(vllm_config)
|
||||
backend = TestBackend(coalesce_pass)
|
||||
model = SliceCoalescingModel(q_size, kv_size)
|
||||
|
||||
qkv = torch.randn(5, q_size + 2 * kv_size)
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
result_eager = model(qkv)
|
||||
result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv)
|
||||
|
||||
for eager, compiled in zip(result_eager, result_compiled):
|
||||
torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3)
|
||||
|
||||
assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3
|
||||
assert backend.op_count(torch.ops.aten.slice.Tensor) == 0
|
||||
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
|
||||
|
||||
split = next(
|
||||
node
|
||||
for node in backend.final_graph.nodes
|
||||
if node.target == torch.ops.aten.split_with_sizes.default
|
||||
)
|
||||
assert list(split.args[1]) == [q_size, kv_size, kv_size]
|
||||
assert split.args[2] == -1
|
||||
|
||||
getitems = [
|
||||
node
|
||||
for node in split.users
|
||||
if node.op == "call_function" and node.target == operator.getitem
|
||||
]
|
||||
assert sorted(node.args[1] for node in getitems) == [0, 1, 2]
|
||||
assert isinstance(split.meta["val"], list)
|
||||
for getitem in getitems:
|
||||
index = getitem.args[1]
|
||||
assert getitem.meta["val"] is split.meta["val"][index]
|
||||
|
||||
|
||||
def make_slice_graph(
|
||||
ranges: list[tuple[int, int, int]],
|
||||
) -> torch.fx.Graph:
|
||||
graph = torch.fx.Graph()
|
||||
source = graph.placeholder("qkv")
|
||||
source.meta["val"] = torch.empty((2, 8), device="meta")
|
||||
|
||||
slices = []
|
||||
for start, end, step in ranges:
|
||||
node = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor,
|
||||
args=(source, -1, start, end, step),
|
||||
)
|
||||
node.meta["val"] = torch.empty(
|
||||
(2, len(range(start, min(end, 8), step))), device="meta"
|
||||
)
|
||||
slices.append(node)
|
||||
graph.output(tuple(slices))
|
||||
return graph
|
||||
|
||||
|
||||
def test_slice_partition_accepts_safe_group():
|
||||
graph = make_slice_graph([(0, 4, 1), (4, 8, 1)])
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
graph.lint()
|
||||
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
|
||||
assert (
|
||||
sum(
|
||||
node.target == torch.ops.aten.split_with_sizes.default
|
||||
for node in graph.nodes
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ranges",
|
||||
[
|
||||
[(0, 3, 1), (4, 8, 1)], # gap
|
||||
[(0, 5, 1), (4, 8, 1)], # overlap
|
||||
[(1, 4, 1), (4, 8, 1)], # missing prefix
|
||||
[(0, 4, 1), (4, 7, 1)], # missing suffix
|
||||
[(0, 8, 1)], # single full-range slice
|
||||
[(0, 8, 2)], # non-unit step
|
||||
],
|
||||
)
|
||||
def test_slice_partition_rejects_unsafe_groups(ranges):
|
||||
graph = make_slice_graph(ranges)
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
slice_count = sum(
|
||||
node.target == torch.ops.aten.slice.Tensor for node in graph.nodes
|
||||
)
|
||||
split_count = sum(
|
||||
node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes
|
||||
)
|
||||
assert slice_count == len(ranges)
|
||||
assert split_count == 0
|
||||
|
||||
|
||||
def test_nested_slice_partitions_use_live_source():
|
||||
"""Use the rewired source when a slice partition consumes another slice."""
|
||||
graph = torch.fx.Graph()
|
||||
source = graph.placeholder("input")
|
||||
source.meta["val"] = torch.empty((2, 8), device="meta")
|
||||
|
||||
outer_left = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4)
|
||||
)
|
||||
outer_left.meta["val"] = torch.empty((2, 4), device="meta")
|
||||
outer_right = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8)
|
||||
)
|
||||
outer_right.meta["val"] = torch.empty((2, 4), device="meta")
|
||||
inner_left = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2)
|
||||
)
|
||||
inner_left.meta["val"] = torch.empty((2, 2), device="meta")
|
||||
inner_right = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4)
|
||||
)
|
||||
inner_right.meta["val"] = torch.empty((2, 2), device="meta")
|
||||
graph.output((inner_left, inner_right, outer_right))
|
||||
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
graph.lint()
|
||||
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
|
||||
splits = [
|
||||
node
|
||||
for node in graph.nodes
|
||||
if node.target == torch.ops.aten.split_with_sizes.default
|
||||
]
|
||||
assert len(splits) == 2
|
||||
|
||||
outer_split = next(node for node in splits if list(node.args[1]) == [4, 4])
|
||||
inner_split = next(node for node in splits if list(node.args[1]) == [2, 2])
|
||||
assert outer_split.args[0] is source
|
||||
assert inner_split.args[0].target == operator.getitem
|
||||
assert inner_split.args[0].args == (outer_split, 0)
|
||||
|
||||
|
||||
def test_split_coalescing_preserves_different_dimensions():
|
||||
"""Keep equal-sized splits on different dimensions as separate nodes."""
|
||||
graph = torch.fx.Graph()
|
||||
source = graph.placeholder("input")
|
||||
source.meta["val"] = torch.empty((8, 8), device="meta")
|
||||
|
||||
outputs = []
|
||||
for dim in (0, 1):
|
||||
split = graph.call_function(
|
||||
torch.ops.aten.split_with_sizes.default,
|
||||
args=(source, [4, 4], dim),
|
||||
)
|
||||
for index in range(2):
|
||||
outputs.append(graph.call_function(operator.getitem, args=(split, index)))
|
||||
graph.output(tuple(outputs))
|
||||
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
splits = [
|
||||
node
|
||||
for node in graph.nodes
|
||||
if node.target == torch.ops.aten.split_with_sizes.default
|
||||
]
|
||||
assert len(splits) == 2
|
||||
assert all(split.args[0] is source for split in splits)
|
||||
assert {split.args[2] for split in splits} == {0, 1}
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same
|
||||
input tensor with the same split sizes.
|
||||
"""Canonicalize tensor partitions to a single ``split_with_sizes`` node.
|
||||
|
||||
On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor
|
||||
graph may contain multiple ``split_with_sizes`` calls on the same tensor
|
||||
that CSE fails to merge. This pass detects and replaces the duplicates
|
||||
so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion)
|
||||
see a single split node with all users attached.
|
||||
Functionalization may replay the individual outputs of a multi-output split as
|
||||
contiguous ``slice`` nodes. This pass converts complete slice partitions back
|
||||
to ``split_with_sizes`` and coalesces equivalent split nodes so downstream
|
||||
pattern-matching passes see one canonical split with all users attached.
|
||||
|
||||
See Also:
|
||||
- vLLM #33295 (original issue)
|
||||
- vLLM #33295 (original duplicate-split issue)
|
||||
- PyTorch #174472 (upstream CSE gap)
|
||||
- PyTorch #194037 (multi-output views replayed as individual slices)
|
||||
|
||||
"""
|
||||
|
||||
import operator
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import fx
|
||||
@@ -28,35 +31,177 @@ from ..vllm_inductor_pass import VllmInductorPass
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SliceInfo:
|
||||
node: fx.Node
|
||||
dim: int
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
class SplitCoalescingPass(VllmInductorPass):
|
||||
"""Replace duplicate ``split_with_sizes`` nodes with a single canonical
|
||||
node when they share the same input tensor and split sizes."""
|
||||
"""Canonicalize tensor partitions to one shared ``split_with_sizes``.
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
count = 0
|
||||
A complete contiguous slice partition is rewritten from::
|
||||
|
||||
# Map from input tensor node -> list of split nodes seen so far.
|
||||
split_nodes: dict[fx.Node, list[fx.Node]] = {}
|
||||
q = slice(qkv, 0, q_size)
|
||||
k = slice(qkv, q_size, q_size + kv_size)
|
||||
v = slice(qkv, q_size + kv_size, q_size + 2 * kv_size)
|
||||
|
||||
for node in graph.nodes:
|
||||
if not is_func(node, torch.ops.aten.split_with_sizes.default):
|
||||
to::
|
||||
|
||||
qkv_split = split_with_sizes(qkv, [q_size, kv_size, kv_size])
|
||||
q = qkv_split[0]
|
||||
k = qkv_split[1]
|
||||
v = qkv_split[2]
|
||||
|
||||
Equivalent splits are also coalesced from::
|
||||
|
||||
q = split_with_sizes(qkv, sizes)[0]
|
||||
k = split_with_sizes(qkv, sizes)[1]
|
||||
|
||||
to::
|
||||
|
||||
qkv_split = split_with_sizes(qkv, sizes)
|
||||
q = qkv_split[0]
|
||||
k = qkv_split[1]
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _get_arg(node: fx.Node, index: int, name: str, default: Any) -> Any:
|
||||
"""Read an FX argument by position, keyword, or schema default."""
|
||||
if len(node.args) > index:
|
||||
return node.args[index]
|
||||
return node.kwargs.get(name, default)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_dim(dim: int, rank: int) -> int:
|
||||
return dim + rank if dim < 0 else dim
|
||||
|
||||
@staticmethod
|
||||
def _tensor_val(node: fx.Node) -> torch.Tensor | None:
|
||||
value = node.meta.get("val")
|
||||
return value if isinstance(value, torch.Tensor) else None
|
||||
|
||||
def _slice_info(self, node: fx.Node) -> _SliceInfo | None:
|
||||
source = node.args[0]
|
||||
source_val = self._tensor_val(source)
|
||||
slice_val = self._tensor_val(node)
|
||||
# in case someone forgot to set meta !
|
||||
if source_val is None or slice_val is None:
|
||||
return None
|
||||
|
||||
dim = self._normalize_dim(self._get_arg(node, 1, "dim", 0), source_val.dim())
|
||||
extent = source_val.shape[dim]
|
||||
|
||||
# do not support dynamic slices.
|
||||
if not isinstance(extent, int):
|
||||
return None
|
||||
|
||||
start = self._get_arg(node, 2, "start", None)
|
||||
end = self._get_arg(node, 3, "end", None)
|
||||
step = self._get_arg(node, 4, "step", 1)
|
||||
start = 0 if start is None else start
|
||||
end = extent if end is None else end
|
||||
if not isinstance(start, int) or not isinstance(end, int) or step != 1:
|
||||
return None
|
||||
|
||||
if start < 0 or end < 0:
|
||||
return None
|
||||
|
||||
# aten.slice clamps an oversized end to the selected dimension.
|
||||
end = min(end, extent)
|
||||
if start >= end:
|
||||
return None
|
||||
|
||||
return _SliceInfo(node=node, dim=dim, start=start, end=end)
|
||||
|
||||
def _canonicalize_slices(self, graph: fx.Graph) -> int:
|
||||
groups: dict[tuple[fx.Node, int], list[_SliceInfo]] = defaultdict(list)
|
||||
|
||||
for node in graph.find_nodes(
|
||||
op="call_function", target=torch.ops.aten.slice.Tensor
|
||||
):
|
||||
slice_info = self._slice_info(node)
|
||||
if slice_info is None:
|
||||
continue
|
||||
groups[(node.args[0], slice_info.dim)].append(slice_info)
|
||||
|
||||
count = 0
|
||||
for slice_group in groups.values():
|
||||
if len(slice_group) < 2:
|
||||
continue
|
||||
|
||||
source = slice_group[0].node.args[0]
|
||||
dim = slice_group[0].dim
|
||||
source_val = self._tensor_val(source)
|
||||
assert source_val is not None
|
||||
extent = source_val.shape[dim]
|
||||
assert isinstance(extent, int)
|
||||
|
||||
ordered = sorted(slice_group, key=lambda info: info.start)
|
||||
# Ensure they are contiguous partitions.
|
||||
if (
|
||||
ordered[0].start != 0
|
||||
or ordered[-1].end != extent
|
||||
or any(
|
||||
left.end != right.start for left, right in zip(ordered, ordered[1:])
|
||||
)
|
||||
):
|
||||
continue
|
||||
|
||||
split_sizes = [info.end - info.start for info in ordered]
|
||||
# Use -1 for the last dimension; otherwise use the positive dimension index.
|
||||
split_dim = -1 if dim == source_val.dim() - 1 else dim
|
||||
first_slice = slice_group[0].node
|
||||
|
||||
with graph.inserting_before(first_slice):
|
||||
split = graph.call_function(
|
||||
torch.ops.aten.split_with_sizes.default,
|
||||
args=(source, split_sizes, split_dim),
|
||||
)
|
||||
split.meta["val"] = [info.node.meta["val"] for info in ordered]
|
||||
|
||||
replacements: list[tuple[fx.Node, fx.Node]] = []
|
||||
for index, info in enumerate(ordered):
|
||||
getitem = graph.call_function(
|
||||
operator.getitem,
|
||||
args=(split, index),
|
||||
type_expr=info.node.type,
|
||||
)
|
||||
getitem.meta = copy(info.node.meta)
|
||||
replacements.append((info.node, getitem))
|
||||
|
||||
for slice_node, getitem in replacements:
|
||||
slice_node.replace_all_uses_with(getitem)
|
||||
graph.erase_node(slice_node)
|
||||
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
def _coalesce_splits(self, graph: fx.Graph) -> int:
|
||||
count = 0
|
||||
split_nodes: dict[tuple[fx.Node, int], list[fx.Node]] = defaultdict(list)
|
||||
|
||||
for node in graph.find_nodes(
|
||||
op="call_function", target=torch.ops.aten.split_with_sizes.default
|
||||
):
|
||||
if not all(is_func(user, operator.getitem) for user in node.users):
|
||||
continue
|
||||
|
||||
arg_node, split_sizes = node.args[:2]
|
||||
|
||||
if arg_node not in split_nodes:
|
||||
split_nodes[arg_node] = [node]
|
||||
source, split_sizes = node.args[:2]
|
||||
source_val = self._tensor_val(source)
|
||||
if source_val is None:
|
||||
continue
|
||||
|
||||
# Find existing node with same split_sizes
|
||||
raw_dim = self._get_arg(node, 2, "dim", 0)
|
||||
dim = self._normalize_dim(raw_dim, source_val.dim())
|
||||
candidates = split_nodes[(source, dim)]
|
||||
canonical = next(
|
||||
(
|
||||
n
|
||||
for n in split_nodes[arg_node]
|
||||
if list(n.args[1]) == list(split_sizes)
|
||||
candidate
|
||||
for candidate in candidates
|
||||
if list(candidate.args[1]) == list(split_sizes)
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -65,6 +210,16 @@ class SplitCoalescingPass(VllmInductorPass):
|
||||
graph.erase_node(node)
|
||||
count += 1
|
||||
else:
|
||||
split_nodes[arg_node].append(node)
|
||||
candidates.append(node)
|
||||
|
||||
logger.debug("Coalesced %d duplicate split_with_sizes nodes", count)
|
||||
return count
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
canonicalized = self._canonicalize_slices(graph)
|
||||
coalesced = self._coalesce_splits(graph)
|
||||
logger.debug(
|
||||
"Canonicalized %d slice partitions and coalesced %d split nodes",
|
||||
canonicalized,
|
||||
coalesced,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user