[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)

Signed-off-by: fangyao fan <2281693949@qq.com>
Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taimys
2026-10-13 19:17:29 +00:00
committed by GitHub
co-authored by Taimys Harry Mellor
parent 3f550129e8
commit c068db0e1d
9 changed files with 387 additions and 54 deletions
+12 -3
View File
@@ -110,6 +110,7 @@ def support_torch_compile(
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
use_aot_compile: bool = True,
) -> Callable[[type[_T]], type[_T]]: ...
@@ -124,6 +125,7 @@ def support_torch_compile(
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
use_aot_compile: bool = True,
) -> Callable[[type[_T]], type[_T]] | type[_T]:
"""A decorator to add support for compiling the forward method of a class.
@@ -175,6 +177,10 @@ def support_torch_compile(
the lifetime of the model, otherwise, it cannot be captured as a single
computation graph.
`use_aot_compile=False` disables model AOT in DYNAMO_TRACE_ONCE and
VLLM_COMPILE modes. In VLLM_COMPILE mode, it also skips mega and direct
compiled artifacts while retaining the underlying PyTorch graph caches.
`enable_if` is a function that takes a `VllmConfig` object as input and
returns a boolean value indicating whether to compile the model or not.
This is useful if you want to compile the model only when certain
@@ -243,6 +249,7 @@ def support_torch_compile(
mark_unbacked_dims,
enable_if,
is_encoder,
use_aot_compile,
)
if cls is not None:
@@ -347,6 +354,7 @@ def _support_torch_compile(
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
use_aot_compile: bool = True,
) -> type[_T]:
"""Internal implementation of support_torch_compile decorator."""
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
@@ -396,6 +404,7 @@ def _support_torch_compile(
self.vllm_config = vllm_config
self.compilation_config = self.vllm_config.compilation_config
self._allow_aot_compile = use_aot_compile
enable_compile = enable_if is None or enable_if(vllm_config)
# for CompilationMode.STOCK_TORCH_COMPILE , the upper level model runner
# will handle the compilation, so we don't need to do anything here.
@@ -544,7 +553,7 @@ def _support_torch_compile(
ds_type = self.compilation_config.dynamic_shapes_config.type
cache_dir = None
aot_compilation_path = None
if envs.VLLM_USE_AOT_COMPILE:
if self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE:
"""
When using torch.compile in AOT mode, we store the cache artifacts
under VLLM_CACHE_ROOT/torch_compile_cache/torch_aot_compile/{hash}
@@ -597,7 +606,7 @@ def _support_torch_compile(
if self.compiled:
assert (
not envs.VLLM_USE_AOT_COMPILE
not (self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE)
or self.vllm_config.compilation_config.backend == "eager"
)
return TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs) # type: ignore[arg-type]
@@ -680,7 +689,7 @@ def _support_torch_compile(
torch.fx.experimental._config.patch(**fx_config_patches),
torch._inductor.config.patch(**inductor_config_patches),
):
use_aot_compile = envs.VLLM_USE_AOT_COMPILE
use_aot_compile = self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE
if self.vllm_config.compilation_config.backend == "eager":
logger.warning("Detected eager backend, disabling AOT compile.")
use_aot_compile = False