[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)
Signed-off-by: fangyao fan <2281693949@qq.com> Signed-off-by: Taimys <ffy2281693949@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Taimys <ffy2281693949@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Taimys
Harry Mellor
parent
3f550129e8
commit
c068db0e1d
@@ -110,6 +110,7 @@ def support_torch_compile(
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
use_aot_compile: bool = True,
|
||||
) -> Callable[[type[_T]], type[_T]]: ...
|
||||
|
||||
|
||||
@@ -124,6 +125,7 @@ def support_torch_compile(
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
use_aot_compile: bool = True,
|
||||
) -> Callable[[type[_T]], type[_T]] | type[_T]:
|
||||
"""A decorator to add support for compiling the forward method of a class.
|
||||
|
||||
@@ -175,6 +177,10 @@ def support_torch_compile(
|
||||
the lifetime of the model, otherwise, it cannot be captured as a single
|
||||
computation graph.
|
||||
|
||||
`use_aot_compile=False` disables model AOT in DYNAMO_TRACE_ONCE and
|
||||
VLLM_COMPILE modes. In VLLM_COMPILE mode, it also skips mega and direct
|
||||
compiled artifacts while retaining the underlying PyTorch graph caches.
|
||||
|
||||
`enable_if` is a function that takes a `VllmConfig` object as input and
|
||||
returns a boolean value indicating whether to compile the model or not.
|
||||
This is useful if you want to compile the model only when certain
|
||||
@@ -243,6 +249,7 @@ def support_torch_compile(
|
||||
mark_unbacked_dims,
|
||||
enable_if,
|
||||
is_encoder,
|
||||
use_aot_compile,
|
||||
)
|
||||
|
||||
if cls is not None:
|
||||
@@ -347,6 +354,7 @@ def _support_torch_compile(
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
use_aot_compile: bool = True,
|
||||
) -> type[_T]:
|
||||
"""Internal implementation of support_torch_compile decorator."""
|
||||
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
|
||||
@@ -396,6 +404,7 @@ def _support_torch_compile(
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.compilation_config = self.vllm_config.compilation_config
|
||||
self._allow_aot_compile = use_aot_compile
|
||||
enable_compile = enable_if is None or enable_if(vllm_config)
|
||||
# for CompilationMode.STOCK_TORCH_COMPILE , the upper level model runner
|
||||
# will handle the compilation, so we don't need to do anything here.
|
||||
@@ -544,7 +553,7 @@ def _support_torch_compile(
|
||||
ds_type = self.compilation_config.dynamic_shapes_config.type
|
||||
cache_dir = None
|
||||
aot_compilation_path = None
|
||||
if envs.VLLM_USE_AOT_COMPILE:
|
||||
if self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE:
|
||||
"""
|
||||
When using torch.compile in AOT mode, we store the cache artifacts
|
||||
under VLLM_CACHE_ROOT/torch_compile_cache/torch_aot_compile/{hash}
|
||||
@@ -597,7 +606,7 @@ def _support_torch_compile(
|
||||
|
||||
if self.compiled:
|
||||
assert (
|
||||
not envs.VLLM_USE_AOT_COMPILE
|
||||
not (self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE)
|
||||
or self.vllm_config.compilation_config.backend == "eager"
|
||||
)
|
||||
return TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs) # type: ignore[arg-type]
|
||||
@@ -680,7 +689,7 @@ def _support_torch_compile(
|
||||
torch.fx.experimental._config.patch(**fx_config_patches),
|
||||
torch._inductor.config.patch(**inductor_config_patches),
|
||||
):
|
||||
use_aot_compile = envs.VLLM_USE_AOT_COMPILE
|
||||
use_aot_compile = self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE
|
||||
if self.vllm_config.compilation_config.backend == "eager":
|
||||
logger.warning("Detected eager backend, disabling AOT compile.")
|
||||
use_aot_compile = False
|
||||
|
||||
Reference in New Issue
Block a user