[Bugfix] Avoid merging empty-only partitions into splitting-op subgraphs (#36595)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -431,6 +431,7 @@ def _is_empty_allocation_node(node: fx.Node) -> bool:
|
||||
|
||||
def _merge_empty_only_subgraphs(
|
||||
node_to_subgraph_id: dict[fx.Node, int],
|
||||
split_op_graphs: list[int],
|
||||
) -> None:
|
||||
"""
|
||||
Merge a partition that only contains an empty allocation op into the
|
||||
@@ -439,23 +440,35 @@ def _merge_empty_only_subgraphs(
|
||||
"""
|
||||
|
||||
nodes_by_subgraph_id: dict[int, list[fx.Node]] = defaultdict(list)
|
||||
subgraph_id_order: list[int] = []
|
||||
for node, subgraph_id in node_to_subgraph_id.items():
|
||||
if subgraph_id not in nodes_by_subgraph_id:
|
||||
subgraph_id_order.append(subgraph_id)
|
||||
nodes_by_subgraph_id[subgraph_id].append(node)
|
||||
|
||||
prev_subgraph_id: int | None = None
|
||||
for subgraph_id in subgraph_id_order:
|
||||
nodes = nodes_by_subgraph_id[subgraph_id]
|
||||
if (
|
||||
len(nodes) == 1
|
||||
and _is_empty_allocation_node(nodes[0])
|
||||
and prev_subgraph_id is not None
|
||||
):
|
||||
node_to_subgraph_id[nodes[0]] = prev_subgraph_id
|
||||
splitting_subgraphs = set(split_op_graphs)
|
||||
prev_non_splitting_subgraph_id: int | None = None
|
||||
|
||||
max_subgraph_id = max(node_to_subgraph_id.values(), default=-1)
|
||||
for subgraph_id in range(max_subgraph_id + 1):
|
||||
nodes = nodes_by_subgraph_id.get(subgraph_id, [])
|
||||
if not nodes:
|
||||
continue
|
||||
prev_subgraph_id = subgraph_id
|
||||
|
||||
is_non_splitting_subgraph = subgraph_id not in splitting_subgraphs
|
||||
is_empty_only_subgraph = len(nodes) == 1 and _is_empty_allocation_node(nodes[0])
|
||||
merged = False
|
||||
|
||||
if is_empty_only_subgraph and prev_non_splitting_subgraph_id is not None:
|
||||
# Safety check: don't move allocation before any input producer.
|
||||
empty_node = nodes[0]
|
||||
if all(
|
||||
input_node.op == "placeholder"
|
||||
or node_to_subgraph_id[input_node] <= prev_non_splitting_subgraph_id
|
||||
for input_node in empty_node.all_input_nodes
|
||||
):
|
||||
node_to_subgraph_id[empty_node] = prev_non_splitting_subgraph_id
|
||||
merged = True
|
||||
|
||||
if not merged and is_non_splitting_subgraph:
|
||||
prev_non_splitting_subgraph_id = subgraph_id
|
||||
|
||||
|
||||
def split_graph(
|
||||
@@ -496,7 +509,7 @@ def split_graph(
|
||||
else:
|
||||
node_to_subgraph_id[node] = subgraph_id
|
||||
|
||||
_merge_empty_only_subgraphs(node_to_subgraph_id)
|
||||
_merge_empty_only_subgraphs(node_to_subgraph_id, split_op_graphs)
|
||||
|
||||
# `keep_original_order` is important!
|
||||
# otherwise pytorch might reorder the nodes and
|
||||
|
||||
Reference in New Issue
Block a user