[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)
Signed-off-by: fangyao fan <2281693949@qq.com> Signed-off-by: Taimys <ffy2281693949@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Taimys <ffy2281693949@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Taimys
Harry Mellor
parent
3f550129e8
commit
c068db0e1d
@@ -1,9 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
@@ -890,3 +894,176 @@ def test_disable_compile_cache_skips_aot_load(
|
||||
mod(*args)
|
||||
|
||||
assert not mod.was_aot_compile_fn_loaded_from_disk
|
||||
|
||||
|
||||
_CACHE_OFFSET = int(os.environ.get("VLLM_TEST_CACHE_OFFSET", "0"))
|
||||
|
||||
|
||||
def _capture_cache_offset(fn):
|
||||
offset = _CACHE_OFFSET
|
||||
|
||||
@functools.wraps(fn)
|
||||
def wrapped(self, x):
|
||||
return fn(self, x) + offset
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
def _run_graph_cache_probe():
|
||||
"""Fresh-process probe; only the vLLM cache root survives between runs."""
|
||||
from torch._inductor.runtime.cache_dir_utils import cache_dir
|
||||
|
||||
@support_torch_compile(dynamic_arg_dims={"x": 0}, use_aot_compile=False)
|
||||
class GraphCacheMod(torch.nn.Module):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
@_capture_cache_offset
|
||||
def forward(self, x):
|
||||
return x * 2 + 1
|
||||
|
||||
cache_dir() # Exercise a tmp default populated before backend initialization.
|
||||
config = make_vllm_config()
|
||||
x = torch.arange(64, device=current_platform.device_type).reshape(8, 8).float()
|
||||
expected = x * 2 + 1 + int(os.environ["VLLM_TEST_CACHE_OFFSET"])
|
||||
with use_vllm_config(config):
|
||||
mod = GraphCacheMod(vllm_config=config)
|
||||
assert torch.equal(mod(x), expected)
|
||||
traced = torch._dynamo.utils.counters["stats"]["unique_graphs"]
|
||||
assert traced > 0
|
||||
aot_compiles = compilation_counter.num_aot_compiles
|
||||
assert torch.equal(mod(x), expected)
|
||||
assert torch._dynamo.utils.counters["stats"]["unique_graphs"] == traced
|
||||
assert compilation_counter.num_aot_compiles == aot_compiles == 0
|
||||
for name in (
|
||||
"num_aot_artifacts_saved",
|
||||
"num_aot_artifacts_loaded",
|
||||
"num_compiled_artifacts_saved",
|
||||
"num_compiled_artifacts_loaded",
|
||||
):
|
||||
assert getattr(compilation_counter, name) == 0
|
||||
assert Path(cache_dir()).is_relative_to(config.compilation_config.cache_dir)
|
||||
assert Path(os.environ["TRITON_CACHE_DIR"]).is_relative_to(
|
||||
config.compilation_config.cache_dir
|
||||
)
|
||||
result = {
|
||||
"inductor_dir": cache_dir(),
|
||||
"cache_dir": config.compilation_config.cache_dir,
|
||||
"fx_hits": torch._dynamo.utils.counters["inductor"]["fxgraph_cache_hit"],
|
||||
"autograd_hits": torch._dynamo.utils.counters["aot_autograd"][
|
||||
"autograd_cache_hit"
|
||||
],
|
||||
}
|
||||
native = os.environ.get("VLLM_TEST_NATIVE_AOT")
|
||||
if native:
|
||||
if native == "warm":
|
||||
os.environ["VLLM_FORCE_AOT_LOAD"] = "1"
|
||||
disable_envs_cache()
|
||||
with use_vllm_config(config):
|
||||
native_mod = CompiledMod(vllm_config=config)
|
||||
assert torch.equal(native_mod(x), reference_fn(x))
|
||||
assert compilation_counter.num_aot_artifacts_loaded == int(native == "warm")
|
||||
assert compilation_counter.num_aot_artifacts_saved == int(native == "cold")
|
||||
assert compilation_counter.num_aot_compiles == int(native == "cold")
|
||||
Path(os.environ["VLLM_TEST_CACHE_RESULT"]).write_text(json.dumps(result))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def graph_cache_probe(tmp_path):
|
||||
def run(offset, *, standalone="1", native=""):
|
||||
private = Path(tempfile.mkdtemp(dir=tmp_path))
|
||||
env = os.environ.copy()
|
||||
for key in ("TORCHINDUCTOR_CACHE_DIR", "TRITON_CACHE_DIR"):
|
||||
env.pop(key, None)
|
||||
for key in ("HOME", "TMPDIR", "XDG_CACHE_HOME"):
|
||||
directory = private / key
|
||||
directory.mkdir()
|
||||
env[key] = str(directory)
|
||||
result = private / "result.json"
|
||||
env.update(
|
||||
VLLM_CACHE_ROOT=str(tmp_path / "shared"),
|
||||
VLLM_USE_AOT_COMPILE="1",
|
||||
VLLM_USE_MEGA_AOT_ARTIFACT="1",
|
||||
VLLM_USE_STANDALONE_COMPILE=standalone,
|
||||
VLLM_FORCE_AOT_LOAD="0",
|
||||
VLLM_DISABLE_COMPILE_CACHE="0",
|
||||
VLLM_USE_BYTECODE_HOOK="0",
|
||||
VLLM_TEST_CACHE_OFFSET=str(offset),
|
||||
VLLM_TEST_CACHE_RESULT=str(result),
|
||||
VLLM_TEST_NATIVE_AOT=native,
|
||||
TORCHINDUCTOR_FX_GRAPH_REMOTE_CACHE="0",
|
||||
TORCHINDUCTOR_AUTOGRAD_REMOTE_CACHE="0",
|
||||
)
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"from tests.compile.test_aot_compile import "
|
||||
"_run_graph_cache_probe; _run_graph_cache_probe()",
|
||||
],
|
||||
cwd=Path(__file__).resolve().parents[2],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=180,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stdout + proc.stderr
|
||||
return json.loads(result.read_text())
|
||||
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
@pytest.mark.parametrize("standalone", ["0", "1"])
|
||||
def test_compile_opt_out_retraces_changed_closure(graph_cache_probe, standalone):
|
||||
"""A warm graph is reused, but a changed closure never loads a stale handle."""
|
||||
results = [
|
||||
graph_cache_probe(offset, standalone=standalone) for offset in (0, 0, 7, 7)
|
||||
]
|
||||
assert len({r["cache_dir"] for r in results}) == 1
|
||||
assert len({r["inductor_dir"] for r in results}) == 1
|
||||
for warm in (results[1], results[3]):
|
||||
assert warm["fx_hits"] > 0
|
||||
if standalone == "1":
|
||||
assert warm["autograd_hits"] > 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_compile_opt_out_preserves_native_aot_roundtrip(graph_cache_probe):
|
||||
"""Native AOT survives an opt-out module compiled first under the same config."""
|
||||
graph_cache_probe(0, native="cold")
|
||||
graph_cache_probe(0, native="warm")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_compile_opt_out_respected_in_dynamo_trace_once(monkeypatch, fresh_vllm_cache):
|
||||
"""Opt-out skips model AOT without disabling tracing or causing retracing."""
|
||||
monkeypatch.setenv("VLLM_USE_AOT_COMPILE", "1")
|
||||
monkeypatch.setenv("VLLM_FORCE_AOT_LOAD", "0")
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "0")
|
||||
disable_envs_cache()
|
||||
|
||||
@support_torch_compile(dynamic_arg_dims={"x": 0}, use_aot_compile=False)
|
||||
class OptOutMod(torch.nn.Module):
|
||||
def forward(self, x):
|
||||
return x * 2 + 1
|
||||
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.DYNAMO_TRACE_ONCE, backend="inductor"
|
||||
)
|
||||
)
|
||||
x = torch.arange(64, device=current_platform.device_type).reshape(8, 8).float()
|
||||
counters = torch._dynamo.utils.counters["stats"]
|
||||
with (
|
||||
use_vllm_config(config),
|
||||
patch.object(OptOutMod, "aot_compile", side_effect=AssertionError("Model AOT")),
|
||||
):
|
||||
assert envs.VLLM_USE_AOT_COMPILE
|
||||
mod = OptOutMod(vllm_config=config)
|
||||
before = counters["unique_graphs"]
|
||||
assert torch.equal(mod(x), x * 2 + 1)
|
||||
traced = counters["unique_graphs"]
|
||||
assert traced > before
|
||||
assert torch.equal(mod(x + 10), (x + 10) * 2 + 1)
|
||||
assert counters["unique_graphs"] == traced
|
||||
|
||||
@@ -18,7 +18,7 @@ def test_inductor_standalone_load_uses_current_cache_dir(
|
||||
new_cache_dir = tmp_path / "new"
|
||||
key = "artifact_shape_None_subgraph_0"
|
||||
|
||||
adaptor = InductorStandaloneAdaptor(save_format=save_format)
|
||||
adaptor = InductorStandaloneAdaptor(use_aot_compile=True, save_format=save_format)
|
||||
adaptor.initialize_cache(str(new_cache_dir))
|
||||
|
||||
# Simulate a handle persisted before the cache directory was relocated.
|
||||
@@ -46,3 +46,86 @@ def test_inductor_standalone_load_uses_current_cache_dir(
|
||||
path=str(new_cache_dir / key),
|
||||
format=save_format,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raises", [False, True])
|
||||
def test_opt_out_functorch_config_is_scoped(monkeypatch, raises):
|
||||
from contextlib import nullcontext
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.compiler_interface import (
|
||||
get_inductor_factors,
|
||||
set_functorch_config,
|
||||
)
|
||||
|
||||
config = torch._functorch.config
|
||||
monkeypatch.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
|
||||
from vllm.envs import disable_envs_cache
|
||||
|
||||
disable_envs_cache()
|
||||
with config.patch(bundled_autograd_cache=True):
|
||||
error = pytest.raises(RuntimeError, match="probe") if raises else nullcontext()
|
||||
with error, set_functorch_config(use_aot_compile=False):
|
||||
assert config.bundled_autograd_cache is False
|
||||
if raises:
|
||||
raise RuntimeError("probe")
|
||||
assert config.bundled_autograd_cache is True
|
||||
original = type(config).save_config_portable
|
||||
observed = []
|
||||
|
||||
def observe(module):
|
||||
if module is config:
|
||||
observed.append(config.bundled_autograd_cache)
|
||||
return original(module)
|
||||
|
||||
with patch.object(type(config), "save_config_portable", new=observe):
|
||||
get_inductor_factors(use_aot_compile=False)
|
||||
assert observed == [False]
|
||||
assert config.bundled_autograd_cache is True
|
||||
|
||||
|
||||
def test_opt_out_ignores_existing_direct_cache_index(tmp_path, monkeypatch):
|
||||
import builtins
|
||||
|
||||
from vllm.compilation.backends import CompilerManager
|
||||
from vllm.config import CompilationConfig, CompilationMode
|
||||
from vllm.config.utils import Range
|
||||
from vllm.envs import disable_envs_cache
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
|
||||
disable_envs_cache()
|
||||
cache = tmp_path / "rank_0_0" / "decoder"
|
||||
cache.mkdir(parents=True)
|
||||
index = cache / "vllm_compile_cache.py"
|
||||
index.write_text("old index must not be read")
|
||||
manager = CompilerManager(
|
||||
CompilationConfig(mode=CompilationMode.VLLM_COMPILE), use_aot_compile=False
|
||||
)
|
||||
real_open = builtins.open
|
||||
reads = []
|
||||
|
||||
def check_open(path, *args, **kwargs):
|
||||
if str(path) == str(index):
|
||||
reads.append(path)
|
||||
raise AssertionError("read old index")
|
||||
return real_open(path, *args, **kwargs)
|
||||
|
||||
with (
|
||||
patch.object(manager.compiler, "initialize_cache"),
|
||||
patch("builtins.open", side_effect=check_open),
|
||||
):
|
||||
manager.initialize_cache(str(cache), prefix="decoder")
|
||||
assert Path(manager.cache_file_path) == index
|
||||
assert reads == []
|
||||
compile_range = Range(1, 8)
|
||||
manager.cache[(compile_range, 0, manager.compiler.name)] = {
|
||||
"graph_handle": ("old", "unused"),
|
||||
"cache_key": "old",
|
||||
}
|
||||
with patch.object(
|
||||
manager.compiler, "load", side_effect=AssertionError("direct load")
|
||||
) as load:
|
||||
assert manager.load(MagicMock(), [], 0, compile_range) is None
|
||||
load.assert_not_called()
|
||||
assert index.read_text() == "old index must not be read"
|
||||
|
||||
@@ -206,7 +206,9 @@ def test_segmented_triton_cache_survives_standalone_compilation(
|
||||
RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
|
||||
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
|
||||
assert expected.is_dir()
|
||||
InductorStandaloneAdaptor("binary").initialize_cache(str(tmp_path / model))
|
||||
InductorStandaloneAdaptor(
|
||||
use_aot_compile=True, save_format="binary"
|
||||
).initialize_cache(str(tmp_path / model))
|
||||
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user