[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)

Signed-off-by: fangyao fan <2281693949@qq.com>
Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taimys
2026-10-13 19:17:29 +00:00
committed by GitHub
co-authored by Taimys Harry Mellor
parent 3f550129e8
commit c068db0e1d
9 changed files with 387 additions and 54 deletions
+59 -29
View File
@@ -3,7 +3,7 @@
import contextlib
import copy
import os
from collections.abc import Callable
from collections.abc import Callable, Generator
from contextlib import ExitStack
from typing import Any, Literal
from unittest.mock import patch
@@ -26,6 +26,22 @@ logger = init_logger(__name__)
_last_inductor_triton_cache_dir: str | None = None
def initialize_inductor_cache(base_cache_dir: str) -> None:
global _last_inductor_triton_cache_dir
# Keep artifacts together unless the caller supplies a Triton cache.
inductor_cache = os.path.join(base_cache_dir, "inductor_cache")
os.makedirs(inductor_cache, exist_ok=True)
os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
triton_cache = os.environ.get("TRITON_CACHE_DIR")
# A directory assigned by an earlier adaptor is still a model-local
# default, rather than a caller override for subsequent models.
if triton_cache is None or triton_cache == _last_inductor_triton_cache_dir:
triton_cache = os.path.join(base_cache_dir, "triton_cache")
_last_inductor_triton_cache_dir = triton_cache
os.makedirs(triton_cache, exist_ok=True)
os.environ["TRITON_CACHE_DIR"] = triton_cache
class CompilerInterface:
"""The interface for a compiler that can be used by vLLM."""
@@ -151,18 +167,18 @@ class AlwaysHitShapeEnv:
return ""
def _get_vllm_functorch_config() -> dict[str, Any]:
def _get_vllm_functorch_config(use_aot_compile: bool = True) -> dict[str, Any]:
"""Return the functorch config overrides that vLLM applies at compile time.
Used by both set_functorch_config() and get_inductor_factors() to ensure
the compile-time config and cache key are always consistent."""
cfg: dict[str, Any] = {}
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
if not (use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT):
cfg["bundled_autograd_cache"] = False
return cfg
def get_inductor_factors() -> list[Any]:
def get_inductor_factors(use_aot_compile: bool = True) -> list[Any]:
factors: list[Any] = []
# summarize system state
from torch._inductor.codecache import CacheBase
@@ -180,7 +196,7 @@ def get_inductor_factors() -> list[Any]:
from torch._inductor import config as inductor_config
factors.append(inductor_config.save_config_portable())
with functorch_config.patch(_get_vllm_functorch_config()):
with functorch_config.patch(_get_vllm_functorch_config(use_aot_compile)):
factors.append(functorch_config.save_config_portable())
return factors
@@ -254,13 +270,16 @@ class InductorStandaloneAdaptor(CompilerInterface):
name = "inductor_standalone"
def __init__(self, save_format: Literal["binary", "unpacked"]) -> None:
def __init__(
self, save_format: Literal["binary", "unpacked"], *, use_aot_compile: bool
) -> None:
if not is_torch_equal_or_newer("2.10.0"):
_patch_standalone_compile_atomic_save()
self.save_format = save_format
self.use_aot_compile = use_aot_compile
def compute_hash(self, vllm_config: VllmConfig) -> str:
factors = get_inductor_factors()
factors = get_inductor_factors(self.use_aot_compile)
hash_str: str = safe_hash(
str(factors).encode(), usedforsecurity=False
).hexdigest()[:10]
@@ -270,6 +289,10 @@ class InductorStandaloneAdaptor(CompilerInterface):
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None:
self.cache_dir = cache_dir
if not self.use_aot_compile and not disable_cache:
initialize_inductor_cache(
cache_dir[: -len(prefix)] if prefix else cache_dir
)
def compile(
self,
@@ -285,7 +308,6 @@ class InductorStandaloneAdaptor(CompilerInterface):
if compiler_config is not None:
current_config.update(compiler_config)
set_inductor_config(current_config, compile_range)
set_functorch_config()
if compile_range.is_single_size():
dynamic_shapes = "from_example_inputs"
@@ -296,7 +318,11 @@ class InductorStandaloneAdaptor(CompilerInterface):
supports_aot = is_torch_equal_or_newer("2.10.0")
if not supports_aot and envs.VLLM_USE_MEGA_AOT_ARTIFACT:
if (
not supports_aot
and self.use_aot_compile
and envs.VLLM_USE_MEGA_AOT_ARTIFACT
):
logger.error(
"CRITICAL: VLLM_USE_MEGA_AOT_ARTIFACT "
"is enabled but PyTorch version does not support 'aot' "
@@ -314,7 +340,9 @@ class InductorStandaloneAdaptor(CompilerInterface):
if is_torch_equal_or_newer("2.13.0.dev"):
compile_kwargs["donate_graph_module"] = True # type: ignore[assignment]
use_aot: bool = supports_aot and envs.VLLM_USE_MEGA_AOT_ARTIFACT
use_aot: bool = (
supports_aot and self.use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT
)
# only add 'aot' parameter if both supported and enabled...
# this will set bundled_autograd_cache
# https://github.com/pytorch/pytorch/blob/9bbc5b2905c260adf41bc866a732f9c121a2828a/torch/_inductor/standalone_compile.py#L359 # noqa
@@ -366,9 +394,12 @@ class InductorStandaloneAdaptor(CompilerInterface):
else:
fake_mode_ctx = contextlib.nullcontext()
with pregrad_ctx, fake_mode_ctx:
with pregrad_ctx, fake_mode_ctx, set_functorch_config(self.use_aot_compile):
compiled_graph = standalone_compile(graph, example_inputs, **compile_kwargs)
if not self.use_aot_compile:
return compiled_graph, None
if use_aot:
from torch._inductor.standalone_compile import AOTCompiledArtifact
@@ -445,8 +476,11 @@ class InductorAdaptor(CompilerInterface):
name = "inductor"
def __init__(self, *, use_aot_compile: bool) -> None:
self.use_aot_compile = use_aot_compile
def compute_hash(self, vllm_config: VllmConfig) -> str:
factors = get_inductor_factors()
factors = get_inductor_factors(self.use_aot_compile)
hash_str: str = safe_hash(
str(factors).encode(), usedforsecurity=False
).hexdigest()[:10]
@@ -455,24 +489,12 @@ class InductorAdaptor(CompilerInterface):
def initialize_cache(
self, cache_dir: str, disable_cache: bool = False, prefix: str = ""
) -> None:
global _last_inductor_triton_cache_dir
self.cache_dir = cache_dir
self.prefix = prefix
self.base_cache_dir = cache_dir[: -len(prefix)] if prefix else cache_dir
if disable_cache:
return
# Keep artifacts together unless the caller supplies a Triton cache.
inductor_cache = os.path.join(self.base_cache_dir, "inductor_cache")
os.makedirs(inductor_cache, exist_ok=True)
os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
triton_cache = os.environ.get("TRITON_CACHE_DIR")
# A directory assigned by an earlier adaptor is still a model-local
# default, rather than a caller override for subsequent models.
if triton_cache is None or triton_cache == _last_inductor_triton_cache_dir:
triton_cache = os.path.join(self.base_cache_dir, "triton_cache")
_last_inductor_triton_cache_dir = triton_cache
os.makedirs(triton_cache, exist_ok=True)
os.environ["TRITON_CACHE_DIR"] = triton_cache
initialize_inductor_cache(self.base_cache_dir)
def compile(
self,
@@ -495,7 +517,6 @@ class InductorAdaptor(CompilerInterface):
current_config["fx_graph_remote_cache"] = False
set_inductor_config(current_config, compile_range)
set_functorch_config()
# inductor can inplace modify the graph, so we need to copy it
# see https://github.com/pytorch/pytorch/issues/138980
@@ -554,6 +575,7 @@ class InductorAdaptor(CompilerInterface):
return AlwaysHitShapeEnv()
with ExitStack() as stack:
stack.enter_context(set_functorch_config(self.use_aot_compile))
# for hijacking the hash of the compiled graph
stack.enter_context(
patch(
@@ -754,9 +776,17 @@ def set_inductor_config(config: dict[str, Any], compile_range: Range) -> None:
)
def set_functorch_config() -> None:
for k, v in _get_vllm_functorch_config().items():
setattr(torch._functorch.config, k, v)
@contextlib.contextmanager
def set_functorch_config(use_aot_compile: bool) -> Generator[None, None, None]:
overrides = _get_vllm_functorch_config(use_aot_compile)
if use_aot_compile:
# Preserve the existing native configuration lifetime.
for key, value in overrides.items():
setattr(torch._functorch.config, key, value)
yield
else:
with torch._functorch.config.patch(overrides):
yield
def trigger_inductor_lazy_init(device: torch.device | None = None) -> None: