[torch.compile] Stop lazily compiling (#35472)
Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
@@ -5,7 +5,6 @@ import dataclasses
|
||||
import io
|
||||
import json
|
||||
import pickle
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pickle import Pickler
|
||||
from typing import Any
|
||||
@@ -16,7 +15,6 @@ from torch._inductor.runtime.triton_heuristics import CachingAutotuner
|
||||
from torch._logging._internal import trace_structured
|
||||
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.compilation.monitor import end_monitoring_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.utils import Range
|
||||
from vllm.logger import init_logger
|
||||
@@ -24,6 +22,55 @@ from vllm.logger import init_logger
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_fake_args_from_graph(graph: fx.GraphModule) -> list[Any]:
|
||||
"""Get fake args directly from graph placeholder nodes."""
|
||||
fake_args = []
|
||||
for node in graph.graph.nodes:
|
||||
if node.op == "placeholder":
|
||||
fake_args.append(node.meta["example_value"])
|
||||
else:
|
||||
break
|
||||
return fake_args
|
||||
|
||||
|
||||
def create_concrete_args(graph: fx.GraphModule, size: int) -> list[Any]:
|
||||
"""Create example inputs with symbolic dims replaced by a concrete size.
|
||||
|
||||
Used for single-size eager compilation where we need concrete-shaped
|
||||
inputs but don't have real runtime tensors yet.
|
||||
"""
|
||||
from torch._prims_common import compute_required_storage_length
|
||||
from torch.fx.experimental.symbolic_shapes import is_symbolic
|
||||
|
||||
def concretize(sym_val: Any) -> int:
|
||||
"""Replace all symbolic variables in a SymInt expression with size."""
|
||||
if not is_symbolic(sym_val):
|
||||
return int(sym_val)
|
||||
expr = sym_val.node.expr
|
||||
return int(expr.subs({s: size for s in expr.free_symbols}))
|
||||
|
||||
args: list[Any] = []
|
||||
for node in graph.graph.nodes:
|
||||
if node.op != "placeholder":
|
||||
break
|
||||
val = node.meta["example_value"]
|
||||
if isinstance(val, torch.SymInt):
|
||||
args.append(concretize(val))
|
||||
elif isinstance(val, torch.Tensor):
|
||||
new_shape = tuple(concretize(d) for d in val.shape)
|
||||
new_strides = tuple(concretize(s) for s in val.stride())
|
||||
new_storage_offset = concretize(val.storage_offset())
|
||||
needed_size = compute_required_storage_length(
|
||||
new_shape, new_strides, new_storage_offset
|
||||
)
|
||||
t = torch.empty(needed_size, dtype=val.dtype, device=val.device)
|
||||
t = t.as_strided(new_shape, new_strides, new_storage_offset)
|
||||
args.append(t)
|
||||
else:
|
||||
args.append(val)
|
||||
return args
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class RangeEntry:
|
||||
compile_range: Range
|
||||
@@ -109,10 +156,6 @@ class PiecewiseBackend:
|
||||
# the entries for ranges that we need to either
|
||||
self.range_entries: dict[Range, RangeEntry] = {}
|
||||
|
||||
# to_be_compiled_ranges tracks the remaining ranges to compile,
|
||||
# and updates during the compilation process, so we need to copy it
|
||||
self.to_be_compiled_ranges: set[Range] = set(self.compile_ranges)
|
||||
|
||||
# We only keep compilation management inside this class directly.
|
||||
if self.compile_sizes is not None:
|
||||
for size in self.compile_sizes:
|
||||
@@ -129,7 +172,6 @@ class PiecewiseBackend:
|
||||
self.range_entries[range] = RangeEntry(
|
||||
compile_range=range,
|
||||
)
|
||||
self.to_be_compiled_ranges.add(range)
|
||||
|
||||
for range in self.compile_ranges:
|
||||
self.range_entries[range] = RangeEntry(
|
||||
@@ -139,12 +181,10 @@ class PiecewiseBackend:
|
||||
# Track whether we've logged the graph for this subgraph (only log once)
|
||||
self._graph_logged = False
|
||||
|
||||
# get the on_compilation_complete callback from context...
|
||||
# PiecewiseBackend is created during the first call,
|
||||
# which is when the context is set (see compilation/decorators.py)
|
||||
from vllm.compilation.backends import _on_compilation_complete_callback
|
||||
|
||||
self.on_compilation_complete = _on_compilation_complete_callback.get()
|
||||
if self.graph is not None:
|
||||
self.compile_all_ranges()
|
||||
else:
|
||||
self.load_all_ranges()
|
||||
|
||||
def get_compiled_graph_wrapper(
|
||||
self, compiled_graph: Callable[..., Any]
|
||||
@@ -161,25 +201,6 @@ class PiecewiseBackend:
|
||||
|
||||
return compiled_graph_wrapper
|
||||
|
||||
def check_for_ending_compilation(self) -> None:
|
||||
if self.is_last_graph and not self.to_be_compiled_ranges:
|
||||
# no specific sizes to compile
|
||||
# save the hash of the inductor graph for the next run
|
||||
time_before_saving = time.perf_counter()
|
||||
self.vllm_backend.compiler_manager.save_to_file()
|
||||
elapsed = time.perf_counter() - time_before_saving
|
||||
if elapsed > 1:
|
||||
logger.info_once(
|
||||
"Saved compiler manager cache in %.2f seconds.",
|
||||
elapsed,
|
||||
scope="local",
|
||||
)
|
||||
|
||||
end_monitoring_torch_compile(self.vllm_config)
|
||||
# Call the completion callback (e.g., to save AOT compiled function)
|
||||
if self.on_compilation_complete is not None:
|
||||
self.on_compilation_complete()
|
||||
|
||||
def to_bytes(self) -> dict[str, bytes]:
|
||||
class StandaloneCompiledArtifactsPickler(Pickler):
|
||||
def reducer_override(self, obj: object) -> Any:
|
||||
@@ -216,27 +237,54 @@ class PiecewiseBackend:
|
||||
|
||||
return out
|
||||
|
||||
def _fakify_args(self, args: tuple[Any, ...]) -> list[Any]:
|
||||
# We need to pass fake example_inputs, otherwise torch.compile
|
||||
# will fakify the example_inputs potentially causing some non dynamic
|
||||
# dimension to be be duck shaped to other existing shapes that have hints
|
||||
# matching their values.
|
||||
# This is problem because it can lead to unintended specializations!
|
||||
# if the new wrongly dynamic dim is specialized
|
||||
# it will force specializing the whole shape
|
||||
# torch.compile probably should not accept
|
||||
# non fake tensors as example inputs!
|
||||
# See issue https://github.com/vllm-project/vllm/issues/27899
|
||||
fake_example_inputs = []
|
||||
assert self.graph is not None
|
||||
for node in self.graph.graph.nodes:
|
||||
# All place holders come first
|
||||
if node.op == "placeholder":
|
||||
fake_example_inputs.append(node.meta["example_value"])
|
||||
def compile_all_ranges(self) -> None:
|
||||
"""Compile all range entries for this piecewise subgraph up front."""
|
||||
assert self.graph is not None, (
|
||||
"Cannot compile without a graph. "
|
||||
"When loading from cache/AOT artifacts, "
|
||||
"compile_all_ranges should not be called."
|
||||
)
|
||||
|
||||
for range_entry in self.range_entries.values():
|
||||
if range_entry.compiled:
|
||||
continue
|
||||
|
||||
self._log_compile_start(range_entry.compile_range)
|
||||
|
||||
if range_entry.compile_range.is_single_size():
|
||||
args_list = create_concrete_args(
|
||||
self.graph, range_entry.compile_range.start
|
||||
)
|
||||
else:
|
||||
break
|
||||
assert len(fake_example_inputs) == len(args)
|
||||
return fake_example_inputs
|
||||
args_list = get_fake_args_from_graph(self.graph)
|
||||
|
||||
# TODO(https://github.com/vllm-project/vllm/issues/35766)
|
||||
# Can we remove strict_autograd_cache and
|
||||
# force_non_lazy_backward_lowering overrides?
|
||||
# I added them explicitly because this is what they are
|
||||
# set to before the refactor
|
||||
# (https://github.com/vllm-project/vllm/pull/35472).
|
||||
# They affect the aotautograd cache key computation
|
||||
# but they shouldn't have any effect on the actual
|
||||
# compilation.
|
||||
config_patches = dict(
|
||||
bundled_autograd_cache=True,
|
||||
strict_autograd_cache=False,
|
||||
)
|
||||
if hasattr(torch._functorch.config, "force_non_lazy_backward_lowering"):
|
||||
config_patches["force_non_lazy_backward_lowering"] = False
|
||||
with torch._functorch.config.patch(**config_patches):
|
||||
range_entry.runnable = self.vllm_backend.compiler_manager.compile(
|
||||
self.graph,
|
||||
args_list,
|
||||
self.vllm_backend.inductor_config,
|
||||
self.compilation_config,
|
||||
compile_range=range_entry.compile_range,
|
||||
graph_index=self.piecewise_compile_index,
|
||||
num_graphs=self.total_piecewise_compiles,
|
||||
)
|
||||
|
||||
range_entry.compiled = True
|
||||
|
||||
def _log_compile_start(self, compile_range: Range):
|
||||
"""Log compilation event for TORCH_TRACE/tlparse."""
|
||||
@@ -277,44 +325,29 @@ class PiecewiseBackend:
|
||||
payload_fn=lambda: self.graph.print_readable(print_output=False),
|
||||
)
|
||||
|
||||
def _maybe_compile_for_range_entry(
|
||||
self, range_entry: RangeEntry, args: tuple[Any, ...]
|
||||
) -> Any:
|
||||
if not range_entry.compiled:
|
||||
if self.compiled_runnables is not None:
|
||||
range_entry.runnable = self.get_compiled_graph_wrapper(
|
||||
self.compiled_runnables[str(range_entry.compile_range)]
|
||||
)
|
||||
else:
|
||||
self._log_compile_start(range_entry.compile_range)
|
||||
|
||||
# args are real arguments
|
||||
# fakify for range, real args for concrete size.
|
||||
# For concrete size, we clear the shape env in
|
||||
# compiler_manager.compile() so no need to fakify.
|
||||
args_list = (
|
||||
self._fakify_args(args)
|
||||
if not range_entry.compile_range.is_single_size()
|
||||
else list(args)
|
||||
)
|
||||
|
||||
with (
|
||||
torch._functorch.config.patch("bundled_autograd_cache", True),
|
||||
):
|
||||
range_entry.runnable = self.vllm_backend.compiler_manager.compile(
|
||||
self.graph,
|
||||
args_list,
|
||||
self.vllm_backend.inductor_config,
|
||||
self.compilation_config,
|
||||
compile_range=range_entry.compile_range,
|
||||
graph_index=self.piecewise_compile_index,
|
||||
num_graphs=self.total_piecewise_compiles,
|
||||
)
|
||||
def load_all_ranges(self) -> None:
|
||||
"""Load all pre-compiled runnables for this piecewise subgraph.
|
||||
|
||||
Called during warm start to wrap all cached compiled_runnables
|
||||
into range_entry.runnable up front, analogous to compile_all_ranges()
|
||||
for the cold start path.
|
||||
"""
|
||||
assert self.compiled_runnables is not None, (
|
||||
"load_all_ranges should only be called when compiled_runnables "
|
||||
"is set (warm start / cache loading path)."
|
||||
)
|
||||
for range_entry in self.range_entries.values():
|
||||
if range_entry.compiled:
|
||||
continue
|
||||
key = str(range_entry.compile_range)
|
||||
assert key in self.compiled_runnables, (
|
||||
f"Missing compiled runnable for range {range_entry.compile_range}. "
|
||||
f"Available keys: {list(self.compiled_runnables.keys())}"
|
||||
)
|
||||
range_entry.runnable = self.get_compiled_graph_wrapper(
|
||||
self.compiled_runnables[key]
|
||||
)
|
||||
range_entry.compiled = True
|
||||
self.to_be_compiled_ranges.remove(range_entry.compile_range)
|
||||
|
||||
self.check_for_ending_compilation()
|
||||
|
||||
def _find_range_for_shape(self, runtime_shape: int) -> RangeEntry | None:
|
||||
# First we try to find the range entry for the concrete compile size
|
||||
@@ -338,6 +371,9 @@ class PiecewiseBackend:
|
||||
assert range_entry is not None, (
|
||||
f"Shape: {runtime_shape} out of considered ranges: {self.compile_ranges}"
|
||||
)
|
||||
|
||||
self._maybe_compile_for_range_entry(range_entry, args)
|
||||
assert range_entry.compiled, (
|
||||
"All ranges should be compiled or loaded up front in "
|
||||
"PiecewiseBackend.__init__. "
|
||||
f"range_entry={range_entry.compile_range}"
|
||||
)
|
||||
return range_entry.runnable(*args)
|
||||
|
||||
Reference in New Issue
Block a user