[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)

Signed-off-by: fangyao fan <2281693949@qq.com>
Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taimys
2026-10-13 19:17:29 +00:00
committed by GitHub
co-authored by Taimys Harry Mellor
parent 3f550129e8
commit c068db0e1d
9 changed files with 387 additions and 54 deletions
+36 -17
View File
@@ -94,10 +94,13 @@ def make_copy_and_call(
return copy_and_call
def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
assert not envs.VLLM_USE_MEGA_AOT_ARTIFACT or envs.VLLM_USE_STANDALONE_COMPILE, (
"VLLM_USE_MEGA_AOT_ARTIFACT=1 requires VLLM_USE_STANDALONE_COMPILE=1"
)
def make_compiler(
compilation_config: CompilationConfig, *, use_aot_compile: bool
) -> CompilerInterface:
assert (
not (use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT)
or envs.VLLM_USE_STANDALONE_COMPILE
), "VLLM_USE_MEGA_AOT_ARTIFACT=1 requires VLLM_USE_STANDALONE_COMPILE=1"
if compilation_config.backend == "inductor":
# Use standalone compile only if requested, version is new enough,
@@ -107,11 +110,12 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
):
logger.debug("Using InductorStandaloneAdaptor")
return InductorStandaloneAdaptor(
compilation_config.compile_cache_save_format
compilation_config.compile_cache_save_format,
use_aot_compile=use_aot_compile,
)
else:
logger.debug("Using InductorAdaptor")
return InductorAdaptor()
return InductorAdaptor(use_aot_compile=use_aot_compile)
elif compilation_config.backend == "eager":
logger.debug("Using EagerAdaptor")
return EagerAdaptor()
@@ -136,11 +140,16 @@ class CompilerManager:
support int as key.
"""
def __init__(self, compilation_config: CompilationConfig) -> None:
def __init__(
self, compilation_config: CompilationConfig, *, use_aot_compile: bool
) -> None:
self.cache: dict[tuple[Range, int, str], Any] = dict()
self.is_cache_updated = False
self.compilation_config = compilation_config
self.compiler = make_compiler(compilation_config)
self.use_aot_compile = use_aot_compile
self.compiler = make_compiler(
compilation_config, use_aot_compile=use_aot_compile
)
self.loaded_artifacts: dict[str, Any] = {}
self.prefix: str = ""
@@ -185,7 +194,11 @@ class CompilerManager:
self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
self.prefix = prefix
if not disable_cache and os.path.exists(self.cache_file_path):
if (
self.use_aot_compile
and not disable_cache
and os.path.exists(self.cache_file_path)
):
# load the cache from the file
with open(self.cache_file_path) as f:
# we use ast.literal_eval to parse the data
@@ -216,7 +229,7 @@ class CompilerManager:
)
def save_to_file(self) -> None:
if self.disable_cache or not self.is_cache_updated:
if not self.use_aot_compile or self.disable_cache or not self.is_cache_updated:
return
printer = pprint.PrettyPrinter(indent=4)
data = printer.pformat(self.cache)
@@ -230,6 +243,8 @@ class CompilerManager:
graph_index: int,
compile_range: Range,
) -> Callable[..., Any] | None:
if not self.use_aot_compile:
return None
if (compile_range, graph_index, self.compiler.name) not in self.cache:
return None
@@ -370,7 +385,11 @@ class CompilerManager:
assert compiled_graph is not None, "Failed to compile the graph"
# store the artifact in the cache
if is_compile_cache_enabled(additional_inductor_config) and handle is not None:
if (
self.use_aot_compile
and is_compile_cache_enabled(additional_inductor_config)
and handle is not None
):
self.cache[(compile_range, graph_index, self.compiler.name)] = {
"graph_handle": handle,
"cache_key": cache_key,
@@ -839,6 +858,7 @@ class VllmBackend:
vllm_config: VllmConfig,
prefix: str = "",
is_encoder: bool = False,
use_aot_compile: bool = True,
) -> None:
# if the model is initialized with a non-empty prefix,
# then usually it's enough to use that prefix,
@@ -859,9 +879,10 @@ class VllmBackend:
self.vllm_config = vllm_config
self.compilation_config = vllm_config.compilation_config
self.use_mega_artifact = use_aot_compile and envs.VLLM_USE_MEGA_AOT_ARTIFACT
self.compiler_manager: CompilerManager = CompilerManager(
self.compilation_config
self.compilation_config, use_aot_compile=use_aot_compile
)
# Deepcopy the inductor config to detach the post-grad custom pass
@@ -892,7 +913,7 @@ class VllmBackend:
returns_tuple
"""
if not envs.VLLM_USE_MEGA_AOT_ARTIFACT:
if not self.use_mega_artifact:
return None, None, None
from .caching import StandaloneCompiledArtifacts
@@ -1193,7 +1214,7 @@ class VllmBackend:
# keep a split_gm copy from BEFORE the interpreter replaces
# submodules with PiecewiseBackend -- used for serialization
original_split_gm = None
if envs.VLLM_USE_MEGA_AOT_ARTIFACT:
if self.use_mega_artifact:
original_split_gm = deepcopy(self.split_gm)
from torch._dynamo.utils import lazy_format_graph_code
@@ -1285,9 +1306,7 @@ class VllmBackend:
logger.debug_once("Computation graph saved to %s", graph_path)
self._called = True
graph_to_serialize = (
original_split_gm if envs.VLLM_USE_MEGA_AOT_ARTIFACT else self.graph
)
graph_to_serialize = original_split_gm if self.use_mega_artifact else self.graph
execution_code, submod_names, consts = generate_execution_code(self.split_gm)
# Use getattr to get correct callables: __dict__ has PiecewiseBackend