Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
e519281920
commit
cddce79fda
@@ -15,6 +15,11 @@ import torch.fx as fx
|
||||
from torch._dispatch.python import enable_python_dispatcher
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.inductor_pass import pass_context
|
||||
from vllm.compilation.partition_rules import (
|
||||
inductor_partition_rule_context,
|
||||
resolve_defined_ops,
|
||||
)
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
@@ -76,6 +81,21 @@ class CompilerManager:
|
||||
def compute_hash(self, vllm_config: VllmConfig) -> str:
|
||||
return self.compiler.compute_hash(vllm_config)
|
||||
|
||||
@contextmanager
|
||||
def compile_context(self, runtime_shape: Optional[int] = None):
|
||||
"""Provide compilation context for the duration of compilation to set
|
||||
any torch global properties we want to scope to a single Inductor
|
||||
compilation (e.g. partition rules, pass context)."""
|
||||
with pass_context(runtime_shape):
|
||||
if self.compilation_config.use_inductor_graph_partition:
|
||||
inductor_partition_ops = resolve_defined_ops(
|
||||
self.compilation_config.splitting_ops
|
||||
)
|
||||
with inductor_partition_rule_context(inductor_partition_ops):
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
|
||||
def initialize_cache(
|
||||
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
|
||||
):
|
||||
@@ -197,9 +217,15 @@ class CompilerManager:
|
||||
maybe_key = None
|
||||
else:
|
||||
maybe_key = f"artifact_shape_{runtime_shape}_subgraph_{graph_index}"
|
||||
compiled_graph, handle = self.compiler.compile(
|
||||
graph, example_inputs, additional_inductor_config, runtime_shape, maybe_key
|
||||
)
|
||||
|
||||
with self.compile_context(runtime_shape):
|
||||
compiled_graph, handle = self.compiler.compile(
|
||||
graph,
|
||||
example_inputs,
|
||||
additional_inductor_config,
|
||||
runtime_shape,
|
||||
maybe_key,
|
||||
)
|
||||
|
||||
assert compiled_graph is not None, "Failed to compile the graph"
|
||||
|
||||
@@ -258,7 +284,7 @@ class SplitItem:
|
||||
|
||||
|
||||
def split_graph(
|
||||
graph: fx.GraphModule, ops: list[str]
|
||||
graph: fx.GraphModule, resolved_ops: list[torch._ops.OpOverload]
|
||||
) -> tuple[fx.GraphModule, list[SplitItem]]:
|
||||
# split graph by ops
|
||||
subgraph_id = 0
|
||||
@@ -267,7 +293,12 @@ def split_graph(
|
||||
for node in graph.graph.nodes:
|
||||
if node.op in ("output", "placeholder"):
|
||||
continue
|
||||
if node.op == "call_function" and str(node.target) in ops:
|
||||
# Match node.target against resolved_ops
|
||||
# node.target can be OpOverloadPacket, need to check .default
|
||||
if node.op == "call_function" and (
|
||||
node.target in resolved_ops
|
||||
or (hasattr(node.target, "default") and node.target.default in resolved_ops)
|
||||
):
|
||||
subgraph_id += 1
|
||||
node_to_subgraph_id[node] = subgraph_id
|
||||
split_op_graphs.append(subgraph_id)
|
||||
@@ -615,9 +646,14 @@ class VllmBackend:
|
||||
self.graph = graph
|
||||
self.configure_post_pass()
|
||||
|
||||
self.split_gm, self.piecewise_graphs = split_graph(
|
||||
graph, self.compilation_config.splitting_ops
|
||||
)
|
||||
if self.compilation_config.use_inductor_graph_partition:
|
||||
# Let Inductor decide partitioning; avoid FX-level pre-splitting.
|
||||
fx_split_ops: list[str] = []
|
||||
else:
|
||||
fx_split_ops = self.compilation_config.splitting_ops or []
|
||||
|
||||
resolved_split_ops = resolve_defined_ops(fx_split_ops)
|
||||
self.split_gm, self.piecewise_graphs = split_graph(graph, resolved_split_ops)
|
||||
|
||||
from torch._dynamo.utils import lazy_format_graph_code
|
||||
|
||||
|
||||
Reference in New Issue
Block a user