Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
e519281920
commit
cddce79fda
@@ -15,6 +15,11 @@ import torch.fx as fx
|
||||
from torch._dispatch.python import enable_python_dispatcher
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.inductor_pass import pass_context
|
||||
from vllm.compilation.partition_rules import (
|
||||
inductor_partition_rule_context,
|
||||
resolve_defined_ops,
|
||||
)
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
@@ -76,6 +81,21 @@ class CompilerManager:
|
||||
def compute_hash(self, vllm_config: VllmConfig) -> str:
|
||||
return self.compiler.compute_hash(vllm_config)
|
||||
|
||||
@contextmanager
|
||||
def compile_context(self, runtime_shape: Optional[int] = None):
|
||||
"""Provide compilation context for the duration of compilation to set
|
||||
any torch global properties we want to scope to a single Inductor
|
||||
compilation (e.g. partition rules, pass context)."""
|
||||
with pass_context(runtime_shape):
|
||||
if self.compilation_config.use_inductor_graph_partition:
|
||||
inductor_partition_ops = resolve_defined_ops(
|
||||
self.compilation_config.splitting_ops
|
||||
)
|
||||
with inductor_partition_rule_context(inductor_partition_ops):
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
|
||||
def initialize_cache(
|
||||
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
|
||||
):
|
||||
@@ -197,9 +217,15 @@ class CompilerManager:
|
||||
maybe_key = None
|
||||
else:
|
||||
maybe_key = f"artifact_shape_{runtime_shape}_subgraph_{graph_index}"
|
||||
compiled_graph, handle = self.compiler.compile(
|
||||
graph, example_inputs, additional_inductor_config, runtime_shape, maybe_key
|
||||
)
|
||||
|
||||
with self.compile_context(runtime_shape):
|
||||
compiled_graph, handle = self.compiler.compile(
|
||||
graph,
|
||||
example_inputs,
|
||||
additional_inductor_config,
|
||||
runtime_shape,
|
||||
maybe_key,
|
||||
)
|
||||
|
||||
assert compiled_graph is not None, "Failed to compile the graph"
|
||||
|
||||
@@ -258,7 +284,7 @@ class SplitItem:
|
||||
|
||||
|
||||
def split_graph(
|
||||
graph: fx.GraphModule, ops: list[str]
|
||||
graph: fx.GraphModule, resolved_ops: list[torch._ops.OpOverload]
|
||||
) -> tuple[fx.GraphModule, list[SplitItem]]:
|
||||
# split graph by ops
|
||||
subgraph_id = 0
|
||||
@@ -267,7 +293,12 @@ def split_graph(
|
||||
for node in graph.graph.nodes:
|
||||
if node.op in ("output", "placeholder"):
|
||||
continue
|
||||
if node.op == "call_function" and str(node.target) in ops:
|
||||
# Match node.target against resolved_ops
|
||||
# node.target can be OpOverloadPacket, need to check .default
|
||||
if node.op == "call_function" and (
|
||||
node.target in resolved_ops
|
||||
or (hasattr(node.target, "default") and node.target.default in resolved_ops)
|
||||
):
|
||||
subgraph_id += 1
|
||||
node_to_subgraph_id[node] = subgraph_id
|
||||
split_op_graphs.append(subgraph_id)
|
||||
@@ -615,9 +646,14 @@ class VllmBackend:
|
||||
self.graph = graph
|
||||
self.configure_post_pass()
|
||||
|
||||
self.split_gm, self.piecewise_graphs = split_graph(
|
||||
graph, self.compilation_config.splitting_ops
|
||||
)
|
||||
if self.compilation_config.use_inductor_graph_partition:
|
||||
# Let Inductor decide partitioning; avoid FX-level pre-splitting.
|
||||
fx_split_ops: list[str] = []
|
||||
else:
|
||||
fx_split_ops = self.compilation_config.splitting_ops or []
|
||||
|
||||
resolved_split_ops = resolve_defined_ops(fx_split_ops)
|
||||
self.split_gm, self.piecewise_graphs = split_graph(graph, resolved_split_ops)
|
||||
|
||||
from torch._dynamo.utils import lazy_format_graph_code
|
||||
|
||||
|
||||
@@ -17,8 +17,6 @@ from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.utils import is_torch_equal_or_newer
|
||||
|
||||
from .inductor_pass import pass_context
|
||||
|
||||
|
||||
class CompilerInterface:
|
||||
"""
|
||||
@@ -209,13 +207,12 @@ class InductorStandaloneAdaptor(CompilerInterface):
|
||||
|
||||
from torch._inductor import standalone_compile
|
||||
|
||||
with pass_context(runtime_shape):
|
||||
compiled_graph = standalone_compile(
|
||||
graph,
|
||||
example_inputs,
|
||||
dynamic_shapes=dynamic_shapes,
|
||||
options={"config_patches": current_config},
|
||||
)
|
||||
compiled_graph = standalone_compile(
|
||||
graph,
|
||||
example_inputs,
|
||||
dynamic_shapes=dynamic_shapes,
|
||||
options={"config_patches": current_config},
|
||||
)
|
||||
|
||||
# Save the compiled artifact to disk in the specified path
|
||||
assert key is not None
|
||||
@@ -462,13 +459,12 @@ class InductorAdaptor(CompilerInterface):
|
||||
torch._functorch.config.patch(enable_remote_autograd_cache=False)
|
||||
)
|
||||
|
||||
with pass_context(runtime_shape):
|
||||
compiled_graph = compile_fx(
|
||||
graph,
|
||||
example_inputs,
|
||||
inner_compile=hijacked_compile_fx_inner,
|
||||
config_patches=current_config,
|
||||
)
|
||||
compiled_graph = compile_fx(
|
||||
graph,
|
||||
example_inputs,
|
||||
inner_compile=hijacked_compile_fx_inner,
|
||||
config_patches=current_config,
|
||||
)
|
||||
|
||||
# We treat VLLM_DISABLE_COMPILE_CACHE as the overall switch for torch
|
||||
# compilation cache. So turn off the checks if we disable the
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from torch._library.utils import lookup_op
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def resolve_defined_ops(op_names: list[str]) -> list[torch._ops.OpOverload]:
|
||||
"""Resolve operator names to OpOverload objects.
|
||||
|
||||
Skips operators that fail to resolve (e.g., operators not registered or
|
||||
model-specific operators not present in the current model).
|
||||
|
||||
Note: Users should inspect the operator graph before lowering and ensure
|
||||
the specified operators are present in the final graph. Built-in PyTorch
|
||||
operators (aten::*, torch::*) may be decomposed, fused, or transformed
|
||||
during Inductor's compilation passes, so use them with caution.
|
||||
|
||||
Args:
|
||||
op_names: List of operator names in PyTorch format
|
||||
(e.g., "vllm::unified_attention")
|
||||
|
||||
Returns:
|
||||
List of successfully resolved operator overloads
|
||||
"""
|
||||
resolved = []
|
||||
for op_name in op_names:
|
||||
try:
|
||||
resolved.append(lookup_op(op_name))
|
||||
except Exception:
|
||||
# Skip operators that don't exist (e.g., model-specific ops)
|
||||
logger.warning(
|
||||
"Failed to resolve operator for Inductor partition: %s", op_name
|
||||
)
|
||||
continue
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def inductor_partition_rule_context(overloads: list[torch._ops.OpOverload]):
|
||||
"""Context manager to temporarily register Inductor partition rules.
|
||||
|
||||
Registers custom partition rules for specified operators, forcing the
|
||||
Inductor scheduler to partition the graph at these operators. The rules
|
||||
are automatically restored to their previous state on exit.
|
||||
|
||||
Note: Callers should use resolve_defined_ops() to convert operator names
|
||||
to OpOverload objects before calling this function.
|
||||
|
||||
Args:
|
||||
overloads: List of resolved operator overload objects.
|
||||
"""
|
||||
if not overloads:
|
||||
logger.debug("No partition ops provided; skipping rule registration.")
|
||||
yield
|
||||
return
|
||||
|
||||
from torch._inductor.scheduler import ( # type: ignore
|
||||
_custom_should_partition_fns,
|
||||
register_should_partition_rule,
|
||||
)
|
||||
|
||||
def _always_partition(*_args, **_kwargs):
|
||||
return True
|
||||
|
||||
# Save current state before registering
|
||||
saved_rules = _custom_should_partition_fns.copy()
|
||||
|
||||
for overload in overloads:
|
||||
register_should_partition_rule(
|
||||
overload,
|
||||
_always_partition,
|
||||
)
|
||||
|
||||
logger.debug("Registered inductor partition rules for %d operators", len(overloads))
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# Clear and restore previous state
|
||||
_custom_should_partition_fns.clear()
|
||||
_custom_should_partition_fns.update(saved_rules)
|
||||
logger.debug("Restored previous partition rules state.")
|
||||
Reference in New Issue
Block a user