[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)

Signed-off-by: fangyao fan <2281693949@qq.com>
Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taimys
2026-10-13 19:17:29 +00:00
committed by GitHub
co-authored by Taimys Harry Mellor
parent 3f550129e8
commit c068db0e1d
9 changed files with 387 additions and 54 deletions
+177
View File
@@ -1,9 +1,13 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import hashlib import hashlib
import json
import os import os
import pickle import pickle
import subprocess
import sys
import tempfile import tempfile
from contextlib import contextmanager from contextlib import contextmanager
from pathlib import Path from pathlib import Path
@@ -890,3 +894,176 @@ def test_disable_compile_cache_skips_aot_load(
mod(*args) mod(*args)
assert not mod.was_aot_compile_fn_loaded_from_disk assert not mod.was_aot_compile_fn_loaded_from_disk
_CACHE_OFFSET = int(os.environ.get("VLLM_TEST_CACHE_OFFSET", "0"))
def _capture_cache_offset(fn):
offset = _CACHE_OFFSET
@functools.wraps(fn)
def wrapped(self, x):
return fn(self, x) + offset
return wrapped
def _run_graph_cache_probe():
"""Fresh-process probe; only the vLLM cache root survives between runs."""
from torch._inductor.runtime.cache_dir_utils import cache_dir
@support_torch_compile(dynamic_arg_dims={"x": 0}, use_aot_compile=False)
class GraphCacheMod(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
@_capture_cache_offset
def forward(self, x):
return x * 2 + 1
cache_dir() # Exercise a tmp default populated before backend initialization.
config = make_vllm_config()
x = torch.arange(64, device=current_platform.device_type).reshape(8, 8).float()
expected = x * 2 + 1 + int(os.environ["VLLM_TEST_CACHE_OFFSET"])
with use_vllm_config(config):
mod = GraphCacheMod(vllm_config=config)
assert torch.equal(mod(x), expected)
traced = torch._dynamo.utils.counters["stats"]["unique_graphs"]
assert traced > 0
aot_compiles = compilation_counter.num_aot_compiles
assert torch.equal(mod(x), expected)
assert torch._dynamo.utils.counters["stats"]["unique_graphs"] == traced
assert compilation_counter.num_aot_compiles == aot_compiles == 0
for name in (
"num_aot_artifacts_saved",
"num_aot_artifacts_loaded",
"num_compiled_artifacts_saved",
"num_compiled_artifacts_loaded",
):
assert getattr(compilation_counter, name) == 0
assert Path(cache_dir()).is_relative_to(config.compilation_config.cache_dir)
assert Path(os.environ["TRITON_CACHE_DIR"]).is_relative_to(
config.compilation_config.cache_dir
)
result = {
"inductor_dir": cache_dir(),
"cache_dir": config.compilation_config.cache_dir,
"fx_hits": torch._dynamo.utils.counters["inductor"]["fxgraph_cache_hit"],
"autograd_hits": torch._dynamo.utils.counters["aot_autograd"][
"autograd_cache_hit"
],
}
native = os.environ.get("VLLM_TEST_NATIVE_AOT")
if native:
if native == "warm":
os.environ["VLLM_FORCE_AOT_LOAD"] = "1"
disable_envs_cache()
with use_vllm_config(config):
native_mod = CompiledMod(vllm_config=config)
assert torch.equal(native_mod(x), reference_fn(x))
assert compilation_counter.num_aot_artifacts_loaded == int(native == "warm")
assert compilation_counter.num_aot_artifacts_saved == int(native == "cold")
assert compilation_counter.num_aot_compiles == int(native == "cold")
Path(os.environ["VLLM_TEST_CACHE_RESULT"]).write_text(json.dumps(result))
@pytest.fixture
def graph_cache_probe(tmp_path):
def run(offset, *, standalone="1", native=""):
private = Path(tempfile.mkdtemp(dir=tmp_path))
env = os.environ.copy()
for key in ("TORCHINDUCTOR_CACHE_DIR", "TRITON_CACHE_DIR"):
env.pop(key, None)
for key in ("HOME", "TMPDIR", "XDG_CACHE_HOME"):
directory = private / key
directory.mkdir()
env[key] = str(directory)
result = private / "result.json"
env.update(
VLLM_CACHE_ROOT=str(tmp_path / "shared"),
VLLM_USE_AOT_COMPILE="1",
VLLM_USE_MEGA_AOT_ARTIFACT="1",
VLLM_USE_STANDALONE_COMPILE=standalone,
VLLM_FORCE_AOT_LOAD="0",
VLLM_DISABLE_COMPILE_CACHE="0",
VLLM_USE_BYTECODE_HOOK="0",
VLLM_TEST_CACHE_OFFSET=str(offset),
VLLM_TEST_CACHE_RESULT=str(result),
VLLM_TEST_NATIVE_AOT=native,
TORCHINDUCTOR_FX_GRAPH_REMOTE_CACHE="0",
TORCHINDUCTOR_AUTOGRAD_REMOTE_CACHE="0",
)
proc = subprocess.run(
[
sys.executable,
"-c",
"from tests.compile.test_aot_compile import "
"_run_graph_cache_probe; _run_graph_cache_probe()",
],
cwd=Path(__file__).resolve().parents[2],
env=env,
capture_output=True,
text=True,
timeout=180,
)
assert proc.returncode == 0, proc.stdout + proc.stderr
return json.loads(result.read_text())
return run
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
@pytest.mark.parametrize("standalone", ["0", "1"])
def test_compile_opt_out_retraces_changed_closure(graph_cache_probe, standalone):
"""A warm graph is reused, but a changed closure never loads a stale handle."""
results = [
graph_cache_probe(offset, standalone=standalone) for offset in (0, 0, 7, 7)
]
assert len({r["cache_dir"] for r in results}) == 1
assert len({r["inductor_dir"] for r in results}) == 1
for warm in (results[1], results[3]):
assert warm["fx_hits"] > 0
if standalone == "1":
assert warm["autograd_hits"] > 0
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_compile_opt_out_preserves_native_aot_roundtrip(graph_cache_probe):
"""Native AOT survives an opt-out module compiled first under the same config."""
graph_cache_probe(0, native="cold")
graph_cache_probe(0, native="warm")
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_compile_opt_out_respected_in_dynamo_trace_once(monkeypatch, fresh_vllm_cache):
"""Opt-out skips model AOT without disabling tracing or causing retracing."""
monkeypatch.setenv("VLLM_USE_AOT_COMPILE", "1")
monkeypatch.setenv("VLLM_FORCE_AOT_LOAD", "0")
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "0")
disable_envs_cache()
@support_torch_compile(dynamic_arg_dims={"x": 0}, use_aot_compile=False)
class OptOutMod(torch.nn.Module):
def forward(self, x):
return x * 2 + 1
config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.DYNAMO_TRACE_ONCE, backend="inductor"
)
)
x = torch.arange(64, device=current_platform.device_type).reshape(8, 8).float()
counters = torch._dynamo.utils.counters["stats"]
with (
use_vllm_config(config),
patch.object(OptOutMod, "aot_compile", side_effect=AssertionError("Model AOT")),
):
assert envs.VLLM_USE_AOT_COMPILE
mod = OptOutMod(vllm_config=config)
before = counters["unique_graphs"]
assert torch.equal(mod(x), x * 2 + 1)
traced = counters["unique_graphs"]
assert traced > before
assert torch.equal(mod(x + 10), (x + 10) * 2 + 1)
assert counters["unique_graphs"] == traced
+84 -1
View File
@@ -18,7 +18,7 @@ def test_inductor_standalone_load_uses_current_cache_dir(
new_cache_dir = tmp_path / "new" new_cache_dir = tmp_path / "new"
key = "artifact_shape_None_subgraph_0" key = "artifact_shape_None_subgraph_0"
adaptor = InductorStandaloneAdaptor(save_format=save_format) adaptor = InductorStandaloneAdaptor(use_aot_compile=True, save_format=save_format)
adaptor.initialize_cache(str(new_cache_dir)) adaptor.initialize_cache(str(new_cache_dir))
# Simulate a handle persisted before the cache directory was relocated. # Simulate a handle persisted before the cache directory was relocated.
@@ -46,3 +46,86 @@ def test_inductor_standalone_load_uses_current_cache_dir(
path=str(new_cache_dir / key), path=str(new_cache_dir / key),
format=save_format, format=save_format,
) )
@pytest.mark.parametrize("raises", [False, True])
def test_opt_out_functorch_config_is_scoped(monkeypatch, raises):
from contextlib import nullcontext
import torch
from vllm.compilation.compiler_interface import (
get_inductor_factors,
set_functorch_config,
)
config = torch._functorch.config
monkeypatch.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
from vllm.envs import disable_envs_cache
disable_envs_cache()
with config.patch(bundled_autograd_cache=True):
error = pytest.raises(RuntimeError, match="probe") if raises else nullcontext()
with error, set_functorch_config(use_aot_compile=False):
assert config.bundled_autograd_cache is False
if raises:
raise RuntimeError("probe")
assert config.bundled_autograd_cache is True
original = type(config).save_config_portable
observed = []
def observe(module):
if module is config:
observed.append(config.bundled_autograd_cache)
return original(module)
with patch.object(type(config), "save_config_portable", new=observe):
get_inductor_factors(use_aot_compile=False)
assert observed == [False]
assert config.bundled_autograd_cache is True
def test_opt_out_ignores_existing_direct_cache_index(tmp_path, monkeypatch):
import builtins
from vllm.compilation.backends import CompilerManager
from vllm.config import CompilationConfig, CompilationMode
from vllm.config.utils import Range
from vllm.envs import disable_envs_cache
monkeypatch.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
disable_envs_cache()
cache = tmp_path / "rank_0_0" / "decoder"
cache.mkdir(parents=True)
index = cache / "vllm_compile_cache.py"
index.write_text("old index must not be read")
manager = CompilerManager(
CompilationConfig(mode=CompilationMode.VLLM_COMPILE), use_aot_compile=False
)
real_open = builtins.open
reads = []
def check_open(path, *args, **kwargs):
if str(path) == str(index):
reads.append(path)
raise AssertionError("read old index")
return real_open(path, *args, **kwargs)
with (
patch.object(manager.compiler, "initialize_cache"),
patch("builtins.open", side_effect=check_open),
):
manager.initialize_cache(str(cache), prefix="decoder")
assert Path(manager.cache_file_path) == index
assert reads == []
compile_range = Range(1, 8)
manager.cache[(compile_range, 0, manager.compiler.name)] = {
"graph_handle": ("old", "unused"),
"cache_key": "old",
}
with patch.object(
manager.compiler, "load", side_effect=AssertionError("direct load")
) as load:
assert manager.load(MagicMock(), [], 0, compile_range) is None
load.assert_not_called()
assert index.read_text() == "old index must not be read"
@@ -206,7 +206,9 @@ def test_segmented_triton_cache_survives_standalone_compilation(
RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto") RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
assert os.environ["TRITON_CACHE_DIR"] == str(expected) assert os.environ["TRITON_CACHE_DIR"] == str(expected)
assert expected.is_dir() assert expected.is_dir()
InductorStandaloneAdaptor("binary").initialize_cache(str(tmp_path / model)) InductorStandaloneAdaptor(
use_aot_compile=True, save_format="binary"
).initialize_cache(str(tmp_path / model))
assert os.environ["TRITON_CACHE_DIR"] == str(expected) assert os.environ["TRITON_CACHE_DIR"] == str(expected)
+36 -17
View File
@@ -94,10 +94,13 @@ def make_copy_and_call(
return copy_and_call return copy_and_call
def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface: def make_compiler(
assert not envs.VLLM_USE_MEGA_AOT_ARTIFACT or envs.VLLM_USE_STANDALONE_COMPILE, ( compilation_config: CompilationConfig, *, use_aot_compile: bool
"VLLM_USE_MEGA_AOT_ARTIFACT=1 requires VLLM_USE_STANDALONE_COMPILE=1" ) -> CompilerInterface:
) assert (
not (use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT)
or envs.VLLM_USE_STANDALONE_COMPILE
), "VLLM_USE_MEGA_AOT_ARTIFACT=1 requires VLLM_USE_STANDALONE_COMPILE=1"
if compilation_config.backend == "inductor": if compilation_config.backend == "inductor":
# Use standalone compile only if requested, version is new enough, # Use standalone compile only if requested, version is new enough,
@@ -107,11 +110,12 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
): ):
logger.debug("Using InductorStandaloneAdaptor") logger.debug("Using InductorStandaloneAdaptor")
return InductorStandaloneAdaptor( return InductorStandaloneAdaptor(
compilation_config.compile_cache_save_format compilation_config.compile_cache_save_format,
use_aot_compile=use_aot_compile,
) )
else: else:
logger.debug("Using InductorAdaptor") logger.debug("Using InductorAdaptor")
return InductorAdaptor() return InductorAdaptor(use_aot_compile=use_aot_compile)
elif compilation_config.backend == "eager": elif compilation_config.backend == "eager":
logger.debug("Using EagerAdaptor") logger.debug("Using EagerAdaptor")
return EagerAdaptor() return EagerAdaptor()
@@ -136,11 +140,16 @@ class CompilerManager:
support int as key. support int as key.
""" """
def __init__(self, compilation_config: CompilationConfig) -> None: def __init__(
self, compilation_config: CompilationConfig, *, use_aot_compile: bool
) -> None:
self.cache: dict[tuple[Range, int, str], Any] = dict() self.cache: dict[tuple[Range, int, str], Any] = dict()
self.is_cache_updated = False self.is_cache_updated = False
self.compilation_config = compilation_config self.compilation_config = compilation_config
self.compiler = make_compiler(compilation_config) self.use_aot_compile = use_aot_compile
self.compiler = make_compiler(
compilation_config, use_aot_compile=use_aot_compile
)
self.loaded_artifacts: dict[str, Any] = {} self.loaded_artifacts: dict[str, Any] = {}
self.prefix: str = "" self.prefix: str = ""
@@ -185,7 +194,11 @@ class CompilerManager:
self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py") self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
self.prefix = prefix self.prefix = prefix
if not disable_cache and os.path.exists(self.cache_file_path): if (
self.use_aot_compile
and not disable_cache
and os.path.exists(self.cache_file_path)
):
# load the cache from the file # load the cache from the file
with open(self.cache_file_path) as f: with open(self.cache_file_path) as f:
# we use ast.literal_eval to parse the data # we use ast.literal_eval to parse the data
@@ -216,7 +229,7 @@ class CompilerManager:
) )
def save_to_file(self) -> None: def save_to_file(self) -> None:
if self.disable_cache or not self.is_cache_updated: if not self.use_aot_compile or self.disable_cache or not self.is_cache_updated:
return return
printer = pprint.PrettyPrinter(indent=4) printer = pprint.PrettyPrinter(indent=4)
data = printer.pformat(self.cache) data = printer.pformat(self.cache)
@@ -230,6 +243,8 @@ class CompilerManager:
graph_index: int, graph_index: int,
compile_range: Range, compile_range: Range,
) -> Callable[..., Any] | None: ) -> Callable[..., Any] | None:
if not self.use_aot_compile:
return None
if (compile_range, graph_index, self.compiler.name) not in self.cache: if (compile_range, graph_index, self.compiler.name) not in self.cache:
return None return None
@@ -370,7 +385,11 @@ class CompilerManager:
assert compiled_graph is not None, "Failed to compile the graph" assert compiled_graph is not None, "Failed to compile the graph"
# store the artifact in the cache # store the artifact in the cache
if is_compile_cache_enabled(additional_inductor_config) and handle is not None: if (
self.use_aot_compile
and is_compile_cache_enabled(additional_inductor_config)
and handle is not None
):
self.cache[(compile_range, graph_index, self.compiler.name)] = { self.cache[(compile_range, graph_index, self.compiler.name)] = {
"graph_handle": handle, "graph_handle": handle,
"cache_key": cache_key, "cache_key": cache_key,
@@ -839,6 +858,7 @@ class VllmBackend:
vllm_config: VllmConfig, vllm_config: VllmConfig,
prefix: str = "", prefix: str = "",
is_encoder: bool = False, is_encoder: bool = False,
use_aot_compile: bool = True,
) -> None: ) -> None:
# if the model is initialized with a non-empty prefix, # if the model is initialized with a non-empty prefix,
# then usually it's enough to use that prefix, # then usually it's enough to use that prefix,
@@ -859,9 +879,10 @@ class VllmBackend:
self.vllm_config = vllm_config self.vllm_config = vllm_config
self.compilation_config = vllm_config.compilation_config self.compilation_config = vllm_config.compilation_config
self.use_mega_artifact = use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT
self.compiler_manager: CompilerManager = CompilerManager( self.compiler_manager: CompilerManager = CompilerManager(
self.compilation_config self.compilation_config, use_aot_compile=use_aot_compile
) )
# Deepcopy the inductor config to detach the post-grad custom pass # Deepcopy the inductor config to detach the post-grad custom pass
@@ -892,7 +913,7 @@ class VllmBackend:
returns_tuple returns_tuple
""" """
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT: if not self.use_mega_artifact:
return None, None, None return None, None, None
from .caching import StandaloneCompiledArtifacts from .caching import StandaloneCompiledArtifacts
@@ -1193,7 +1214,7 @@ class VllmBackend:
# keep a split_gm copy from BEFORE the interpreter replaces # keep a split_gm copy from BEFORE the interpreter replaces
# submodules with PiecewiseBackend -- used for serialization # submodules with PiecewiseBackend -- used for serialization
original_split_gm = None original_split_gm = None
if envs.VLLM_USE_MEGA_AOT_ARTIFACT: if self.use_mega_artifact:
original_split_gm = deepcopy(self.split_gm) original_split_gm = deepcopy(self.split_gm)
from torch._dynamo.utils import lazy_format_graph_code from torch._dynamo.utils import lazy_format_graph_code
@@ -1285,9 +1306,7 @@ class VllmBackend:
logger.debug_once("Computation graph saved to %s", graph_path) logger.debug_once("Computation graph saved to %s", graph_path)
self._called = True self._called = True
graph_to_serialize = ( graph_to_serialize = original_split_gm if self.use_mega_artifact else self.graph
original_split_gm if envs.VLLM_USE_MEGA_AOT_ARTIFACT else self.graph
)
execution_code, submod_names, consts = generate_execution_code(self.split_gm) execution_code, submod_names, consts = generate_execution_code(self.split_gm)
# Use getattr to get correct callables: __dict__ has PiecewiseBackend # Use getattr to get correct callables: __dict__ has PiecewiseBackend
+59 -29
View File
@@ -3,7 +3,7 @@
import contextlib import contextlib
import copy import copy
import os import os
from collections.abc import Callable from collections.abc import Callable, Generator
from contextlib import ExitStack from contextlib import ExitStack
from typing import Any, Literal from typing import Any, Literal
from unittest.mock import patch from unittest.mock import patch
@@ -26,6 +26,22 @@ logger = init_logger(__name__)
_last_inductor_triton_cache_dir: str | None = None _last_inductor_triton_cache_dir: str | None = None
def initialize_inductor_cache(base_cache_dir: str) -> None:
global _last_inductor_triton_cache_dir
# Keep artifacts together unless the caller supplies a Triton cache.
inductor_cache = os.path.join(base_cache_dir, "inductor_cache")
os.makedirs(inductor_cache, exist_ok=True)
os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
triton_cache = os.environ.get("TRITON_CACHE_DIR")
# A directory assigned by an earlier adaptor is still a model-local
# default, rather than a caller override for subsequent models.
if triton_cache is None or triton_cache == _last_inductor_triton_cache_dir:
triton_cache = os.path.join(base_cache_dir, "triton_cache")
_last_inductor_triton_cache_dir = triton_cache
os.makedirs(triton_cache, exist_ok=True)
os.environ["TRITON_CACHE_DIR"] = triton_cache
class CompilerInterface: class CompilerInterface:
"""The interface for a compiler that can be used by vLLM.""" """The interface for a compiler that can be used by vLLM."""
@@ -151,18 +167,18 @@ class AlwaysHitShapeEnv:
return "" return ""
def _get_vllm_functorch_config() -> dict[str, Any]: def _get_vllm_functorch_config(use_aot_compile: bool = True) -> dict[str, Any]:
"""Return the functorch config overrides that vLLM applies at compile time. """Return the functorch config overrides that vLLM applies at compile time.
Used by both set_functorch_config() and get_inductor_factors() to ensure Used by both set_functorch_config() and get_inductor_factors() to ensure
the compile-time config and cache key are always consistent.""" the compile-time config and cache key are always consistent."""
cfg: dict[str, Any] = {} cfg: dict[str, Any] = {}
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT: if not (use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT):
cfg["bundled_autograd_cache"] = False cfg["bundled_autograd_cache"] = False
return cfg return cfg
def get_inductor_factors() -> list[Any]: def get_inductor_factors(use_aot_compile: bool = True) -> list[Any]:
factors: list[Any] = [] factors: list[Any] = []
# summarize system state # summarize system state
from torch._inductor.codecache import CacheBase from torch._inductor.codecache import CacheBase
@@ -180,7 +196,7 @@ def get_inductor_factors() -> list[Any]:
from torch._inductor import config as inductor_config from torch._inductor import config as inductor_config
factors.append(inductor_config.save_config_portable()) factors.append(inductor_config.save_config_portable())
with functorch_config.patch(_get_vllm_functorch_config()): with functorch_config.patch(_get_vllm_functorch_config(use_aot_compile)):
factors.append(functorch_config.save_config_portable()) factors.append(functorch_config.save_config_portable())
return factors return factors
@@ -254,13 +270,16 @@ class InductorStandaloneAdaptor(CompilerInterface):
name = "inductor_standalone" name = "inductor_standalone"
def __init__(self, save_format: Literal["binary", "unpacked"]) -> None: def __init__(
self, save_format: Literal["binary", "unpacked"], *, use_aot_compile: bool
) -> None:
if not is_torch_equal_or_newer("2.10.0"): if not is_torch_equal_or_newer("2.10.0"):
_patch_standalone_compile_atomic_save() _patch_standalone_compile_atomic_save()
self.save_format = save_format self.save_format = save_format
self.use_aot_compile = use_aot_compile
def compute_hash(self, vllm_config: VllmConfig) -> str: def compute_hash(self, vllm_config: VllmConfig) -> str:
factors = get_inductor_factors() factors = get_inductor_factors(self.use_aot_compile)
hash_str: str = safe_hash( hash_str: str = safe_hash(
str(factors).encode(), usedforsecurity=False str(factors).encode(), usedforsecurity=False
).hexdigest()[:10] ).hexdigest()[:10]
@@ -270,6 +289,10 @@ class InductorStandaloneAdaptor(CompilerInterface):
self, cache_dir: str, disable_cache: bool = False, prefix: str = "" self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None: ) -> None:
self.cache_dir = cache_dir self.cache_dir = cache_dir
if not self.use_aot_compile and not disable_cache:
initialize_inductor_cache(
cache_dir[: -len(prefix)] if prefix else cache_dir
)
def compile( def compile(
self, self,
@@ -285,7 +308,6 @@ class InductorStandaloneAdaptor(CompilerInterface):
if compiler_config is not None: if compiler_config is not None:
current_config.update(compiler_config) current_config.update(compiler_config)
set_inductor_config(current_config, compile_range) set_inductor_config(current_config, compile_range)
set_functorch_config()
if compile_range.is_single_size(): if compile_range.is_single_size():
dynamic_shapes = "from_example_inputs" dynamic_shapes = "from_example_inputs"
@@ -296,7 +318,11 @@ class InductorStandaloneAdaptor(CompilerInterface):
supports_aot = is_torch_equal_or_newer("2.10.0") supports_aot = is_torch_equal_or_newer("2.10.0")
if not supports_aot and envs.VLLM_USE_MEGA_AOT_ARTIFACT: if (
not supports_aot
and self.use_aot_compile
and envs.VLLM_USE_MEGA_AOT_ARTIFACT
):
logger.error( logger.error(
"CRITICAL: VLLM_USE_MEGA_AOT_ARTIFACT " "CRITICAL: VLLM_USE_MEGA_AOT_ARTIFACT "
"is enabled but PyTorch version does not support 'aot' " "is enabled but PyTorch version does not support 'aot' "
@@ -314,7 +340,9 @@ class InductorStandaloneAdaptor(CompilerInterface):
if is_torch_equal_or_newer("2.13.0.dev"): if is_torch_equal_or_newer("2.13.0.dev"):
compile_kwargs["donate_graph_module"] = True # type: ignore[assignment] compile_kwargs["donate_graph_module"] = True # type: ignore[assignment]
use_aot: bool = supports_aot and envs.VLLM_USE_MEGA_AOT_ARTIFACT use_aot: bool = (
supports_aot and self.use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT
)
# only add 'aot' parameter if both supported and enabled... # only add 'aot' parameter if both supported and enabled...
# this will set bundled_autograd_cache # this will set bundled_autograd_cache
# https://github.com/pytorch/pytorch/blob/9bbc5b2905c260adf41bc866a732f9c121a2828a/torch/_inductor/standalone_compile.py#L359 # noqa # https://github.com/pytorch/pytorch/blob/9bbc5b2905c260adf41bc866a732f9c121a2828a/torch/_inductor/standalone_compile.py#L359 # noqa
@@ -366,9 +394,12 @@ class InductorStandaloneAdaptor(CompilerInterface):
else: else:
fake_mode_ctx = contextlib.nullcontext() fake_mode_ctx = contextlib.nullcontext()
with pregrad_ctx, fake_mode_ctx: with pregrad_ctx, fake_mode_ctx, set_functorch_config(self.use_aot_compile):
compiled_graph = standalone_compile(graph, example_inputs, **compile_kwargs) compiled_graph = standalone_compile(graph, example_inputs, **compile_kwargs)
if not self.use_aot_compile:
return compiled_graph, None
if use_aot: if use_aot:
from torch._inductor.standalone_compile import AOTCompiledArtifact from torch._inductor.standalone_compile import AOTCompiledArtifact
@@ -445,8 +476,11 @@ class InductorAdaptor(CompilerInterface):
name = "inductor" name = "inductor"
def __init__(self, *, use_aot_compile: bool) -> None:
self.use_aot_compile = use_aot_compile
def compute_hash(self, vllm_config: VllmConfig) -> str: def compute_hash(self, vllm_config: VllmConfig) -> str:
factors = get_inductor_factors() factors = get_inductor_factors(self.use_aot_compile)
hash_str: str = safe_hash( hash_str: str = safe_hash(
str(factors).encode(), usedforsecurity=False str(factors).encode(), usedforsecurity=False
).hexdigest()[:10] ).hexdigest()[:10]
@@ -455,24 +489,12 @@ class InductorAdaptor(CompilerInterface):
def initialize_cache( def initialize_cache(
self, cache_dir: str, disable_cache: bool = False, prefix: str = "" self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None: ) -> None:
global _last_inductor_triton_cache_dir
self.cache_dir = cache_dir self.cache_dir = cache_dir
self.prefix = prefix self.prefix = prefix
self.base_cache_dir = cache_dir[: -len(prefix)] if prefix else cache_dir self.base_cache_dir = cache_dir[: -len(prefix)] if prefix else cache_dir
if disable_cache: if disable_cache:
return return
# Keep artifacts together unless the caller supplies a Triton cache. initialize_inductor_cache(self.base_cache_dir)
inductor_cache = os.path.join(self.base_cache_dir, "inductor_cache")
os.makedirs(inductor_cache, exist_ok=True)
os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
triton_cache = os.environ.get("TRITON_CACHE_DIR")
# A directory assigned by an earlier adaptor is still a model-local
# default, rather than a caller override for subsequent models.
if triton_cache is None or triton_cache == _last_inductor_triton_cache_dir:
triton_cache = os.path.join(self.base_cache_dir, "triton_cache")
_last_inductor_triton_cache_dir = triton_cache
os.makedirs(triton_cache, exist_ok=True)
os.environ["TRITON_CACHE_DIR"] = triton_cache
def compile( def compile(
self, self,
@@ -495,7 +517,6 @@ class InductorAdaptor(CompilerInterface):
current_config["fx_graph_remote_cache"] = False current_config["fx_graph_remote_cache"] = False
set_inductor_config(current_config, compile_range) set_inductor_config(current_config, compile_range)
set_functorch_config()
# inductor can inplace modify the graph, so we need to copy it # inductor can inplace modify the graph, so we need to copy it
# see https://github.com/pytorch/pytorch/issues/138980 # see https://github.com/pytorch/pytorch/issues/138980
@@ -554,6 +575,7 @@ class InductorAdaptor(CompilerInterface):
return AlwaysHitShapeEnv() return AlwaysHitShapeEnv()
with ExitStack() as stack: with ExitStack() as stack:
stack.enter_context(set_functorch_config(self.use_aot_compile))
# for hijacking the hash of the compiled graph # for hijacking the hash of the compiled graph
stack.enter_context( stack.enter_context(
patch( patch(
@@ -754,9 +776,17 @@ def set_inductor_config(config: dict[str, Any], compile_range: Range) -> None:
) )
def set_functorch_config() -> None: @contextlib.contextmanager
for k, v in _get_vllm_functorch_config().items(): def set_functorch_config(use_aot_compile: bool) -> Generator[None, None, None]:
setattr(torch._functorch.config, k, v) overrides = _get_vllm_functorch_config(use_aot_compile)
if use_aot_compile:
# Preserve the existing native configuration lifetime.
for key, value in overrides.items():
setattr(torch._functorch.config, key, value)
yield
else:
with torch._functorch.config.patch(overrides):
yield
def trigger_inductor_lazy_init(device: torch.device | None = None) -> None: def trigger_inductor_lazy_init(device: torch.device | None = None) -> None:
+12 -3
View File
@@ -110,6 +110,7 @@ def support_torch_compile(
mark_unbacked_dims: dict[str, int | list[int]] | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False, is_encoder: bool = False,
use_aot_compile: bool = True,
) -> Callable[[type[_T]], type[_T]]: ... ) -> Callable[[type[_T]], type[_T]]: ...
@@ -124,6 +125,7 @@ def support_torch_compile(
mark_unbacked_dims: dict[str, int | list[int]] | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False, is_encoder: bool = False,
use_aot_compile: bool = True,
) -> Callable[[type[_T]], type[_T]] | type[_T]: ) -> Callable[[type[_T]], type[_T]] | type[_T]:
"""A decorator to add support for compiling the forward method of a class. """A decorator to add support for compiling the forward method of a class.
@@ -175,6 +177,10 @@ def support_torch_compile(
the lifetime of the model, otherwise, it cannot be captured as a single the lifetime of the model, otherwise, it cannot be captured as a single
computation graph. computation graph.
`use_aot_compile=False` disables model AOT in DYNAMO_TRACE_ONCE and
VLLM_COMPILE modes. In VLLM_COMPILE mode, it also skips mega and direct
compiled artifacts while retaining the underlying PyTorch graph caches.
`enable_if` is a function that takes a `VllmConfig` object as input and `enable_if` is a function that takes a `VllmConfig` object as input and
returns a boolean value indicating whether to compile the model or not. returns a boolean value indicating whether to compile the model or not.
This is useful if you want to compile the model only when certain This is useful if you want to compile the model only when certain
@@ -243,6 +249,7 @@ def support_torch_compile(
mark_unbacked_dims, mark_unbacked_dims,
enable_if, enable_if,
is_encoder, is_encoder,
use_aot_compile,
) )
if cls is not None: if cls is not None:
@@ -347,6 +354,7 @@ def _support_torch_compile(
mark_unbacked_dims: dict[str, int | list[int]] | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False, is_encoder: bool = False,
use_aot_compile: bool = True,
) -> type[_T]: ) -> type[_T]:
"""Internal implementation of support_torch_compile decorator.""" """Internal implementation of support_torch_compile decorator."""
if TorchCompileWithNoGuardsWrapper in cls.__bases__: if TorchCompileWithNoGuardsWrapper in cls.__bases__:
@@ -396,6 +404,7 @@ def _support_torch_compile(
self.vllm_config = vllm_config self.vllm_config = vllm_config
self.compilation_config = self.vllm_config.compilation_config self.compilation_config = self.vllm_config.compilation_config
self._allow_aot_compile = use_aot_compile
enable_compile = enable_if is None or enable_if(vllm_config) enable_compile = enable_if is None or enable_if(vllm_config)
# for CompilationMode.STOCK_TORCH_COMPILE , the upper level model runner # for CompilationMode.STOCK_TORCH_COMPILE , the upper level model runner
# will handle the compilation, so we don't need to do anything here. # will handle the compilation, so we don't need to do anything here.
@@ -544,7 +553,7 @@ def _support_torch_compile(
ds_type = self.compilation_config.dynamic_shapes_config.type ds_type = self.compilation_config.dynamic_shapes_config.type
cache_dir = None cache_dir = None
aot_compilation_path = None aot_compilation_path = None
if envs.VLLM_USE_AOT_COMPILE: if self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE:
""" """
When using torch.compile in AOT mode, we store the cache artifacts When using torch.compile in AOT mode, we store the cache artifacts
under VLLM_CACHE_ROOT/torch_compile_cache/torch_aot_compile/{hash} under VLLM_CACHE_ROOT/torch_compile_cache/torch_aot_compile/{hash}
@@ -597,7 +606,7 @@ def _support_torch_compile(
if self.compiled: if self.compiled:
assert ( assert (
not envs.VLLM_USE_AOT_COMPILE not (self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE)
or self.vllm_config.compilation_config.backend == "eager" or self.vllm_config.compilation_config.backend == "eager"
) )
return TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs) # type: ignore[arg-type] return TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs) # type: ignore[arg-type]
@@ -680,7 +689,7 @@ def _support_torch_compile(
torch.fx.experimental._config.patch(**fx_config_patches), torch.fx.experimental._config.patch(**fx_config_patches),
torch._inductor.config.patch(**inductor_config_patches), torch._inductor.config.patch(**inductor_config_patches),
): ):
use_aot_compile = envs.VLLM_USE_AOT_COMPILE use_aot_compile = self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE
if self.vllm_config.compilation_config.backend == "eager": if self.vllm_config.compilation_config.backend == "eager":
logger.warning("Detected eager backend, disabling AOT compile.") logger.warning("Detected eager backend, disabling AOT compile.")
use_aot_compile = False use_aot_compile = False
+7 -2
View File
@@ -85,6 +85,8 @@ class TorchCompileWithNoGuardsWrapper:
return ctx.result return ctx.result
return callable_fn(*args, **kwargs) return callable_fn(*args, **kwargs)
_allow_aot_compile: bool = True
def __init__( def __init__(
self, self,
compile_prefix: str = "", compile_prefix: str = "",
@@ -104,7 +106,10 @@ class TorchCompileWithNoGuardsWrapper:
raise RuntimeError("Compilation mode cannot be NO_COMPILATION") raise RuntimeError("Compilation mode cannot be NO_COMPILATION")
backend = vllm_config.compilation_config.init_backend( backend = vllm_config.compilation_config.init_backend(
vllm_config, prefix=compile_prefix, is_encoder=is_encoder vllm_config,
prefix=compile_prefix,
is_encoder=is_encoder,
use_aot_compile=self._allow_aot_compile,
) )
options = {} options = {}
@@ -151,7 +156,7 @@ class TorchCompileWithNoGuardsWrapper:
_apply_constrain_to_fx_strides_patch() _apply_constrain_to_fx_strides_patch()
aot_context = nullcontext() aot_context = nullcontext()
if envs.VLLM_USE_AOT_COMPILE: if self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE:
if hasattr(torch._dynamo.config, "enable_aot_compile"): if hasattr(torch._dynamo.config, "enable_aot_compile"):
aot_context = torch._dynamo.config.patch(enable_aot_compile=True) aot_context = torch._dynamo.config.patch(enable_aot_compile=True)
else: else:
+8 -1
View File
@@ -1085,6 +1085,7 @@ class CompilationConfig:
vllm_config: "VllmConfig", vllm_config: "VllmConfig",
prefix: str = "", prefix: str = "",
is_encoder: bool = False, is_encoder: bool = False,
use_aot_compile: bool = True,
) -> str | Callable: ) -> str | Callable:
"""Initialize the backend for the compilation config from a vllm config. """Initialize the backend for the compilation config from a vllm config.
@@ -1093,6 +1094,7 @@ class CompilationConfig:
prefix: Cache directory prefix for this compiled module. prefix: Cache directory prefix for this compiled module.
is_encoder: Whether this module is used in an encoder (as is_encoder: Whether this module is used in an encoder (as
opposed to a text backbone). opposed to a text backbone).
use_aot_compile: Whether the module permits AOT and direct artifacts.
Returns: Returns:
The backend for the compilation config. The backend for the compilation config.
@@ -1124,7 +1126,12 @@ class CompilationConfig:
from vllm.compilation.backends import VllmBackend from vllm.compilation.backends import VllmBackend
return VllmBackend(vllm_config, prefix=prefix, is_encoder=is_encoder) return VllmBackend(
vllm_config,
prefix=prefix,
is_encoder=is_encoder,
use_aot_compile=use_aot_compile,
)
def post_init_cudagraph_sizes(self) -> None: def post_init_cudagraph_sizes(self) -> None:
"""To complete the initialization after cudagraph related """To complete the initialization after cudagraph related
@@ -282,6 +282,7 @@ class Base(
dynamic_arg_dims=dynamic_arg_dims, dynamic_arg_dims=dynamic_arg_dims,
enable_if=enable_if, enable_if=enable_if,
is_encoder=is_encoder, is_encoder=is_encoder,
use_aot_compile=False,
)(cls) )(cls)
def _decorate_for_torch_compile(self): def _decorate_for_torch_compile(self):