[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)

Signed-off-by: fangyao fan <2281693949@qq.com>
Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taimys
2026-10-13 19:17:29 +00:00
committed by GitHub
co-authored by Taimys Harry Mellor
parent 3f550129e8
commit c068db0e1d
9 changed files with 387 additions and 54 deletions
+177
View File
@@ -1,9 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import hashlib
import json
import os
import pickle
import subprocess
import sys
import tempfile
from contextlib import contextmanager
from pathlib import Path
@@ -890,3 +894,176 @@ def test_disable_compile_cache_skips_aot_load(
mod(*args)
assert not mod.was_aot_compile_fn_loaded_from_disk
_CACHE_OFFSET = int(os.environ.get("VLLM_TEST_CACHE_OFFSET", "0"))
def _capture_cache_offset(fn):
offset = _CACHE_OFFSET
@functools.wraps(fn)
def wrapped(self, x):
return fn(self, x) + offset
return wrapped
def _run_graph_cache_probe():
"""Fresh-process probe; only the vLLM cache root survives between runs."""
from torch._inductor.runtime.cache_dir_utils import cache_dir
@support_torch_compile(dynamic_arg_dims={"x": 0}, use_aot_compile=False)
class GraphCacheMod(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
@_capture_cache_offset
def forward(self, x):
return x * 2 + 1
cache_dir() # Exercise a tmp default populated before backend initialization.
config = make_vllm_config()
x = torch.arange(64, device=current_platform.device_type).reshape(8, 8).float()
expected = x * 2 + 1 + int(os.environ["VLLM_TEST_CACHE_OFFSET"])
with use_vllm_config(config):
mod = GraphCacheMod(vllm_config=config)
assert torch.equal(mod(x), expected)
traced = torch._dynamo.utils.counters["stats"]["unique_graphs"]
assert traced > 0
aot_compiles = compilation_counter.num_aot_compiles
assert torch.equal(mod(x), expected)
assert torch._dynamo.utils.counters["stats"]["unique_graphs"] == traced
assert compilation_counter.num_aot_compiles == aot_compiles == 0
for name in (
"num_aot_artifacts_saved",
"num_aot_artifacts_loaded",
"num_compiled_artifacts_saved",
"num_compiled_artifacts_loaded",
):
assert getattr(compilation_counter, name) == 0
assert Path(cache_dir()).is_relative_to(config.compilation_config.cache_dir)
assert Path(os.environ["TRITON_CACHE_DIR"]).is_relative_to(
config.compilation_config.cache_dir
)
result = {
"inductor_dir": cache_dir(),
"cache_dir": config.compilation_config.cache_dir,
"fx_hits": torch._dynamo.utils.counters["inductor"]["fxgraph_cache_hit"],
"autograd_hits": torch._dynamo.utils.counters["aot_autograd"][
"autograd_cache_hit"
],
}
native = os.environ.get("VLLM_TEST_NATIVE_AOT")
if native:
if native == "warm":
os.environ["VLLM_FORCE_AOT_LOAD"] = "1"
disable_envs_cache()
with use_vllm_config(config):
native_mod = CompiledMod(vllm_config=config)
assert torch.equal(native_mod(x), reference_fn(x))
assert compilation_counter.num_aot_artifacts_loaded == int(native == "warm")
assert compilation_counter.num_aot_artifacts_saved == int(native == "cold")
assert compilation_counter.num_aot_compiles == int(native == "cold")
Path(os.environ["VLLM_TEST_CACHE_RESULT"]).write_text(json.dumps(result))
@pytest.fixture
def graph_cache_probe(tmp_path):
def run(offset, *, standalone="1", native=""):
private = Path(tempfile.mkdtemp(dir=tmp_path))
env = os.environ.copy()
for key in ("TORCHINDUCTOR_CACHE_DIR", "TRITON_CACHE_DIR"):
env.pop(key, None)
for key in ("HOME", "TMPDIR", "XDG_CACHE_HOME"):
directory = private / key
directory.mkdir()
env[key] = str(directory)
result = private / "result.json"
env.update(
VLLM_CACHE_ROOT=str(tmp_path / "shared"),
VLLM_USE_AOT_COMPILE="1",
VLLM_USE_MEGA_AOT_ARTIFACT="1",
VLLM_USE_STANDALONE_COMPILE=standalone,
VLLM_FORCE_AOT_LOAD="0",
VLLM_DISABLE_COMPILE_CACHE="0",
VLLM_USE_BYTECODE_HOOK="0",
VLLM_TEST_CACHE_OFFSET=str(offset),
VLLM_TEST_CACHE_RESULT=str(result),
VLLM_TEST_NATIVE_AOT=native,
TORCHINDUCTOR_FX_GRAPH_REMOTE_CACHE="0",
TORCHINDUCTOR_AUTOGRAD_REMOTE_CACHE="0",
)
proc = subprocess.run(
[
sys.executable,
"-c",
"from tests.compile.test_aot_compile import "
"_run_graph_cache_probe; _run_graph_cache_probe()",
],
cwd=Path(__file__).resolve().parents[2],
env=env,
capture_output=True,
text=True,
timeout=180,
)
assert proc.returncode == 0, proc.stdout + proc.stderr
return json.loads(result.read_text())
return run
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
@pytest.mark.parametrize("standalone", ["0", "1"])
def test_compile_opt_out_retraces_changed_closure(graph_cache_probe, standalone):
"""A warm graph is reused, but a changed closure never loads a stale handle."""
results = [
graph_cache_probe(offset, standalone=standalone) for offset in (0, 0, 7, 7)
]
assert len({r["cache_dir"] for r in results}) == 1
assert len({r["inductor_dir"] for r in results}) == 1
for warm in (results[1], results[3]):
assert warm["fx_hits"] > 0
if standalone == "1":
assert warm["autograd_hits"] > 0
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_compile_opt_out_preserves_native_aot_roundtrip(graph_cache_probe):
"""Native AOT survives an opt-out module compiled first under the same config."""
graph_cache_probe(0, native="cold")
graph_cache_probe(0, native="warm")
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_compile_opt_out_respected_in_dynamo_trace_once(monkeypatch, fresh_vllm_cache):
"""Opt-out skips model AOT without disabling tracing or causing retracing."""
monkeypatch.setenv("VLLM_USE_AOT_COMPILE", "1")
monkeypatch.setenv("VLLM_FORCE_AOT_LOAD", "0")
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "0")
disable_envs_cache()
@support_torch_compile(dynamic_arg_dims={"x": 0}, use_aot_compile=False)
class OptOutMod(torch.nn.Module):
def forward(self, x):
return x * 2 + 1
config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.DYNAMO_TRACE_ONCE, backend="inductor"
)
)
x = torch.arange(64, device=current_platform.device_type).reshape(8, 8).float()
counters = torch._dynamo.utils.counters["stats"]
with (
use_vllm_config(config),
patch.object(OptOutMod, "aot_compile", side_effect=AssertionError("Model AOT")),
):
assert envs.VLLM_USE_AOT_COMPILE
mod = OptOutMod(vllm_config=config)
before = counters["unique_graphs"]
assert torch.equal(mod(x), x * 2 + 1)
traced = counters["unique_graphs"]
assert traced > before
assert torch.equal(mod(x + 10), (x + 10) * 2 + 1)
assert counters["unique_graphs"] == traced
+84 -1
View File
@@ -18,7 +18,7 @@ def test_inductor_standalone_load_uses_current_cache_dir(
new_cache_dir = tmp_path / "new"
key = "artifact_shape_None_subgraph_0"
adaptor = InductorStandaloneAdaptor(save_format=save_format)
adaptor = InductorStandaloneAdaptor(use_aot_compile=True, save_format=save_format)
adaptor.initialize_cache(str(new_cache_dir))
# Simulate a handle persisted before the cache directory was relocated.
@@ -46,3 +46,86 @@ def test_inductor_standalone_load_uses_current_cache_dir(
path=str(new_cache_dir / key),
format=save_format,
)
@pytest.mark.parametrize("raises", [False, True])
def test_opt_out_functorch_config_is_scoped(monkeypatch, raises):
from contextlib import nullcontext
import torch
from vllm.compilation.compiler_interface import (
get_inductor_factors,
set_functorch_config,
)
config = torch._functorch.config
monkeypatch.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
from vllm.envs import disable_envs_cache
disable_envs_cache()
with config.patch(bundled_autograd_cache=True):
error = pytest.raises(RuntimeError, match="probe") if raises else nullcontext()
with error, set_functorch_config(use_aot_compile=False):
assert config.bundled_autograd_cache is False
if raises:
raise RuntimeError("probe")
assert config.bundled_autograd_cache is True
original = type(config).save_config_portable
observed = []
def observe(module):
if module is config:
observed.append(config.bundled_autograd_cache)
return original(module)
with patch.object(type(config), "save_config_portable", new=observe):
get_inductor_factors(use_aot_compile=False)
assert observed == [False]
assert config.bundled_autograd_cache is True
def test_opt_out_ignores_existing_direct_cache_index(tmp_path, monkeypatch):
import builtins
from vllm.compilation.backends import CompilerManager
from vllm.config import CompilationConfig, CompilationMode
from vllm.config.utils import Range
from vllm.envs import disable_envs_cache
monkeypatch.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
disable_envs_cache()
cache = tmp_path / "rank_0_0" / "decoder"
cache.mkdir(parents=True)
index = cache / "vllm_compile_cache.py"
index.write_text("old index must not be read")
manager = CompilerManager(
CompilationConfig(mode=CompilationMode.VLLM_COMPILE), use_aot_compile=False
)
real_open = builtins.open
reads = []
def check_open(path, *args, **kwargs):
if str(path) == str(index):
reads.append(path)
raise AssertionError("read old index")
return real_open(path, *args, **kwargs)
with (
patch.object(manager.compiler, "initialize_cache"),
patch("builtins.open", side_effect=check_open),
):
manager.initialize_cache(str(cache), prefix="decoder")
assert Path(manager.cache_file_path) == index
assert reads == []
compile_range = Range(1, 8)
manager.cache[(compile_range, 0, manager.compiler.name)] = {
"graph_handle": ("old", "unused"),
"cache_key": "old",
}
with patch.object(
manager.compiler, "load", side_effect=AssertionError("direct load")
) as load:
assert manager.load(MagicMock(), [], 0, compile_range) is None
load.assert_not_called()
assert index.read_text() == "old index must not be read"
@@ -206,7 +206,9 @@ def test_segmented_triton_cache_survives_standalone_compilation(
RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
assert expected.is_dir()
InductorStandaloneAdaptor("binary").initialize_cache(str(tmp_path / model))
InductorStandaloneAdaptor(
use_aot_compile=True, save_format="binary"
).initialize_cache(str(tmp_path / model))
assert os.environ["TRITON_CACHE_DIR"] == str(expected)