From b39f760cc705d945ab6eb56bd8dd15c931890725 Mon Sep 17 00:00:00 2001 From: Laith Sakka Date: Mon, 28 Sep 2026 17:21:59 -0700 Subject: [PATCH] =?UTF-8?q?[torch.compile]=20Canonicalize=20functionalized?= =?UTF-8?q?=20split=20slices=20for=20fusion=20pa=E2=80=A6=20(#57299)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Laith Sakka --- tests/compile/passes/test_split_coalescing.py | 220 +++++++++++++++++- .../passes/utility/split_coalescing.py | 211 ++++++++++++++--- 2 files changed, 397 insertions(+), 34 deletions(-) diff --git a/tests/compile/passes/test_split_coalescing.py b/tests/compile/passes/test_split_coalescing.py index ab7a0be1a2..f8c2891045 100644 --- a/tests/compile/passes/test_split_coalescing.py +++ b/tests/compile/passes/test_split_coalescing.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import operator + import pytest import torch @@ -29,6 +31,33 @@ class SplitCoalescingModel(torch.nn.Module): return q + 1, k + 2, v + 3 +class SliceCoalescingModel(torch.nn.Module): + """Model with a complete QKV partition represented as contiguous slices.""" + + def __init__(self, q_size: int, kv_size: int) -> None: + super().__init__() + self.q_size = q_size + self.kv_size = kv_size + + def forward(self, qkv: torch.Tensor): + q_end = self.q_size + k_end = q_end + self.kv_size + v_end = k_end + self.kv_size + q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end) + k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end) + v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end) + return q + 1, k + 2, v + 3 + + +def make_vllm_config() -> VllmConfig: + return VllmConfig( + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + pass_config=PassConfig(), + ) + ) + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) def test_split_coalescing(dtype): torch.set_default_device(DEVICE_TYPE) @@ -37,12 +66,7 @@ def test_split_coalescing(dtype): q_size, kv_size = 2048, 512 - vllm_config = VllmConfig( - compilation_config=CompilationConfig( - mode=CompilationMode.VLLM_COMPILE, - pass_config=PassConfig(), - ) - ) + vllm_config = make_vllm_config() with vllm.config.set_current_vllm_config(vllm_config): coalesce_pass = SplitCoalescingPass(vllm_config) backend = TestBackend(coalesce_pass) @@ -63,3 +87,187 @@ def test_split_coalescing(dtype): torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL) assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1 + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_slice_partition_canonicalization(dtype): + torch.set_default_device(DEVICE_TYPE) + torch.set_default_dtype(dtype) + torch.manual_seed(0) + + q_size, kv_size = 2048, 512 + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + coalesce_pass = SplitCoalescingPass(vllm_config) + backend = TestBackend(coalesce_pass) + model = SliceCoalescingModel(q_size, kv_size) + + qkv = torch.randn(5, q_size + 2 * kv_size) + torch._dynamo.mark_dynamic(qkv, 0) + result_eager = model(qkv) + result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv) + + for eager, compiled in zip(result_eager, result_compiled): + torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3) + + assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3 + assert backend.op_count(torch.ops.aten.slice.Tensor) == 0 + assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1 + + split = next( + node + for node in backend.final_graph.nodes + if node.target == torch.ops.aten.split_with_sizes.default + ) + assert list(split.args[1]) == [q_size, kv_size, kv_size] + assert split.args[2] == -1 + + getitems = [ + node + for node in split.users + if node.op == "call_function" and node.target == operator.getitem + ] + assert sorted(node.args[1] for node in getitems) == [0, 1, 2] + assert isinstance(split.meta["val"], list) + for getitem in getitems: + index = getitem.args[1] + assert getitem.meta["val"] is split.meta["val"][index] + + +def make_slice_graph( + ranges: list[tuple[int, int, int]], +) -> torch.fx.Graph: + graph = torch.fx.Graph() + source = graph.placeholder("qkv") + source.meta["val"] = torch.empty((2, 8), device="meta") + + slices = [] + for start, end, step in ranges: + node = graph.call_function( + torch.ops.aten.slice.Tensor, + args=(source, -1, start, end, step), + ) + node.meta["val"] = torch.empty( + (2, len(range(start, min(end, 8), step))), device="meta" + ) + slices.append(node) + graph.output(tuple(slices)) + return graph + + +def test_slice_partition_accepts_safe_group(): + graph = make_slice_graph([(0, 4, 1), (4, 8, 1)]) + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + graph.lint() + assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0 + assert ( + sum( + node.target == torch.ops.aten.split_with_sizes.default + for node in graph.nodes + ) + == 1 + ) + + +@pytest.mark.parametrize( + "ranges", + [ + [(0, 3, 1), (4, 8, 1)], # gap + [(0, 5, 1), (4, 8, 1)], # overlap + [(1, 4, 1), (4, 8, 1)], # missing prefix + [(0, 4, 1), (4, 7, 1)], # missing suffix + [(0, 8, 1)], # single full-range slice + [(0, 8, 2)], # non-unit step + ], +) +def test_slice_partition_rejects_unsafe_groups(ranges): + graph = make_slice_graph(ranges) + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + slice_count = sum( + node.target == torch.ops.aten.slice.Tensor for node in graph.nodes + ) + split_count = sum( + node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes + ) + assert slice_count == len(ranges) + assert split_count == 0 + + +def test_nested_slice_partitions_use_live_source(): + """Use the rewired source when a slice partition consumes another slice.""" + graph = torch.fx.Graph() + source = graph.placeholder("input") + source.meta["val"] = torch.empty((2, 8), device="meta") + + outer_left = graph.call_function( + torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4) + ) + outer_left.meta["val"] = torch.empty((2, 4), device="meta") + outer_right = graph.call_function( + torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8) + ) + outer_right.meta["val"] = torch.empty((2, 4), device="meta") + inner_left = graph.call_function( + torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2) + ) + inner_left.meta["val"] = torch.empty((2, 2), device="meta") + inner_right = graph.call_function( + torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4) + ) + inner_right.meta["val"] = torch.empty((2, 2), device="meta") + graph.output((inner_left, inner_right, outer_right)) + + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + graph.lint() + assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0 + splits = [ + node + for node in graph.nodes + if node.target == torch.ops.aten.split_with_sizes.default + ] + assert len(splits) == 2 + + outer_split = next(node for node in splits if list(node.args[1]) == [4, 4]) + inner_split = next(node for node in splits if list(node.args[1]) == [2, 2]) + assert outer_split.args[0] is source + assert inner_split.args[0].target == operator.getitem + assert inner_split.args[0].args == (outer_split, 0) + + +def test_split_coalescing_preserves_different_dimensions(): + """Keep equal-sized splits on different dimensions as separate nodes.""" + graph = torch.fx.Graph() + source = graph.placeholder("input") + source.meta["val"] = torch.empty((8, 8), device="meta") + + outputs = [] + for dim in (0, 1): + split = graph.call_function( + torch.ops.aten.split_with_sizes.default, + args=(source, [4, 4], dim), + ) + for index in range(2): + outputs.append(graph.call_function(operator.getitem, args=(split, index))) + graph.output(tuple(outputs)) + + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + splits = [ + node + for node in graph.nodes + if node.target == torch.ops.aten.split_with_sizes.default + ] + assert len(splits) == 2 + assert all(split.args[0] is source for split in splits) + assert {split.args[2] for split in splits} == {0, 1} diff --git a/vllm/compilation/passes/utility/split_coalescing.py b/vllm/compilation/passes/utility/split_coalescing.py index f16623a1aa..e9423830d1 100644 --- a/vllm/compilation/passes/utility/split_coalescing.py +++ b/vllm/compilation/passes/utility/split_coalescing.py @@ -1,21 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same -input tensor with the same split sizes. +"""Canonicalize tensor partitions to a single ``split_with_sizes`` node. -On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor -graph may contain multiple ``split_with_sizes`` calls on the same tensor -that CSE fails to merge. This pass detects and replaces the duplicates -so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion) -see a single split node with all users attached. +Functionalization may replay the individual outputs of a multi-output split as +contiguous ``slice`` nodes. This pass converts complete slice partitions back +to ``split_with_sizes`` and coalesces equivalent split nodes so downstream +pattern-matching passes see one canonical split with all users attached. See Also: - - vLLM #33295 (original issue) + - vLLM #33295 (original duplicate-split issue) - PyTorch #174472 (upstream CSE gap) + - PyTorch #194037 (multi-output views replayed as individual slices) """ import operator +from collections import defaultdict +from copy import copy +from dataclasses import dataclass +from typing import Any import torch from torch import fx @@ -28,35 +31,177 @@ from ..vllm_inductor_pass import VllmInductorPass logger = init_logger(__name__) +@dataclass(frozen=True) +class _SliceInfo: + node: fx.Node + dim: int + start: int + end: int + + class SplitCoalescingPass(VllmInductorPass): - """Replace duplicate ``split_with_sizes`` nodes with a single canonical - node when they share the same input tensor and split sizes.""" + """Canonicalize tensor partitions to one shared ``split_with_sizes``. - @VllmInductorPass.time_and_log - def __call__(self, graph: fx.Graph) -> None: - count = 0 + A complete contiguous slice partition is rewritten from:: - # Map from input tensor node -> list of split nodes seen so far. - split_nodes: dict[fx.Node, list[fx.Node]] = {} + q = slice(qkv, 0, q_size) + k = slice(qkv, q_size, q_size + kv_size) + v = slice(qkv, q_size + kv_size, q_size + 2 * kv_size) - for node in graph.nodes: - if not is_func(node, torch.ops.aten.split_with_sizes.default): + to:: + + qkv_split = split_with_sizes(qkv, [q_size, kv_size, kv_size]) + q = qkv_split[0] + k = qkv_split[1] + v = qkv_split[2] + + Equivalent splits are also coalesced from:: + + q = split_with_sizes(qkv, sizes)[0] + k = split_with_sizes(qkv, sizes)[1] + + to:: + + qkv_split = split_with_sizes(qkv, sizes) + q = qkv_split[0] + k = qkv_split[1] + """ + + @staticmethod + def _get_arg(node: fx.Node, index: int, name: str, default: Any) -> Any: + """Read an FX argument by position, keyword, or schema default.""" + if len(node.args) > index: + return node.args[index] + return node.kwargs.get(name, default) + + @staticmethod + def _normalize_dim(dim: int, rank: int) -> int: + return dim + rank if dim < 0 else dim + + @staticmethod + def _tensor_val(node: fx.Node) -> torch.Tensor | None: + value = node.meta.get("val") + return value if isinstance(value, torch.Tensor) else None + + def _slice_info(self, node: fx.Node) -> _SliceInfo | None: + source = node.args[0] + source_val = self._tensor_val(source) + slice_val = self._tensor_val(node) + # in case someone forgot to set meta ! + if source_val is None or slice_val is None: + return None + + dim = self._normalize_dim(self._get_arg(node, 1, "dim", 0), source_val.dim()) + extent = source_val.shape[dim] + + # do not support dynamic slices. + if not isinstance(extent, int): + return None + + start = self._get_arg(node, 2, "start", None) + end = self._get_arg(node, 3, "end", None) + step = self._get_arg(node, 4, "step", 1) + start = 0 if start is None else start + end = extent if end is None else end + if not isinstance(start, int) or not isinstance(end, int) or step != 1: + return None + + if start < 0 or end < 0: + return None + + # aten.slice clamps an oversized end to the selected dimension. + end = min(end, extent) + if start >= end: + return None + + return _SliceInfo(node=node, dim=dim, start=start, end=end) + + def _canonicalize_slices(self, graph: fx.Graph) -> int: + groups: dict[tuple[fx.Node, int], list[_SliceInfo]] = defaultdict(list) + + for node in graph.find_nodes( + op="call_function", target=torch.ops.aten.slice.Tensor + ): + slice_info = self._slice_info(node) + if slice_info is None: continue + groups[(node.args[0], slice_info.dim)].append(slice_info) + + count = 0 + for slice_group in groups.values(): + if len(slice_group) < 2: + continue + + source = slice_group[0].node.args[0] + dim = slice_group[0].dim + source_val = self._tensor_val(source) + assert source_val is not None + extent = source_val.shape[dim] + assert isinstance(extent, int) + + ordered = sorted(slice_group, key=lambda info: info.start) + # Ensure they are contiguous partitions. + if ( + ordered[0].start != 0 + or ordered[-1].end != extent + or any( + left.end != right.start for left, right in zip(ordered, ordered[1:]) + ) + ): + continue + + split_sizes = [info.end - info.start for info in ordered] + # Use -1 for the last dimension; otherwise use the positive dimension index. + split_dim = -1 if dim == source_val.dim() - 1 else dim + first_slice = slice_group[0].node + + with graph.inserting_before(first_slice): + split = graph.call_function( + torch.ops.aten.split_with_sizes.default, + args=(source, split_sizes, split_dim), + ) + split.meta["val"] = [info.node.meta["val"] for info in ordered] + + replacements: list[tuple[fx.Node, fx.Node]] = [] + for index, info in enumerate(ordered): + getitem = graph.call_function( + operator.getitem, + args=(split, index), + type_expr=info.node.type, + ) + getitem.meta = copy(info.node.meta) + replacements.append((info.node, getitem)) + + for slice_node, getitem in replacements: + slice_node.replace_all_uses_with(getitem) + graph.erase_node(slice_node) + + count += 1 + + return count + + def _coalesce_splits(self, graph: fx.Graph) -> int: + count = 0 + split_nodes: dict[tuple[fx.Node, int], list[fx.Node]] = defaultdict(list) + + for node in graph.find_nodes( + op="call_function", target=torch.ops.aten.split_with_sizes.default + ): if not all(is_func(user, operator.getitem) for user in node.users): continue - arg_node, split_sizes = node.args[:2] - - if arg_node not in split_nodes: - split_nodes[arg_node] = [node] + source, split_sizes = node.args[:2] + source_val = self._tensor_val(source) + if source_val is None: continue - - # Find existing node with same split_sizes + raw_dim = self._get_arg(node, 2, "dim", 0) + dim = self._normalize_dim(raw_dim, source_val.dim()) + candidates = split_nodes[(source, dim)] canonical = next( ( - n - for n in split_nodes[arg_node] - if list(n.args[1]) == list(split_sizes) + candidate + for candidate in candidates + if list(candidate.args[1]) == list(split_sizes) ), None, ) @@ -65,6 +210,16 @@ class SplitCoalescingPass(VllmInductorPass): graph.erase_node(node) count += 1 else: - split_nodes[arg_node].append(node) + candidates.append(node) - logger.debug("Coalesced %d duplicate split_with_sizes nodes", count) + return count + + @VllmInductorPass.time_and_log + def __call__(self, graph: fx.Graph) -> None: + canonicalized = self._canonicalize_slices(graph) + coalesced = self._coalesce_splits(graph) + logger.debug( + "Canonicalized %d slice partitions and coalesced %d split nodes", + canonicalized, + coalesced, + )