[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)
Signed-off-by: fangyao fan <2281693949@qq.com> Signed-off-by: Taimys <ffy2281693949@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Taimys <ffy2281693949@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Taimys
Harry Mellor
parent
3f550129e8
commit
c068db0e1d
@@ -85,6 +85,8 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
return ctx.result
|
||||
return callable_fn(*args, **kwargs)
|
||||
|
||||
_allow_aot_compile: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
compile_prefix: str = "",
|
||||
@@ -104,7 +106,10 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
raise RuntimeError("Compilation mode cannot be NO_COMPILATION")
|
||||
|
||||
backend = vllm_config.compilation_config.init_backend(
|
||||
vllm_config, prefix=compile_prefix, is_encoder=is_encoder
|
||||
vllm_config,
|
||||
prefix=compile_prefix,
|
||||
is_encoder=is_encoder,
|
||||
use_aot_compile=self._allow_aot_compile,
|
||||
)
|
||||
options = {}
|
||||
|
||||
@@ -151,7 +156,7 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
_apply_constrain_to_fx_strides_patch()
|
||||
|
||||
aot_context = nullcontext()
|
||||
if envs.VLLM_USE_AOT_COMPILE:
|
||||
if self._allow_aot_compile and envs.VLLM_USE_AOT_COMPILE:
|
||||
if hasattr(torch._dynamo.config, "enable_aot_compile"):
|
||||
aot_context = torch._dynamo.config.patch(enable_aot_compile=True)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user