[Bugfix][Compile] Replace Transformers AOT caching with graph caching (#58918)
Signed-off-by: fangyao fan <2281693949@qq.com> Signed-off-by: Taimys <ffy2281693949@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Taimys <ffy2281693949@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Taimys
Harry Mellor
parent
3f550129e8
commit
c068db0e1d
@@ -1085,6 +1085,7 @@ class CompilationConfig:
|
||||
vllm_config: "VllmConfig",
|
||||
prefix: str = "",
|
||||
is_encoder: bool = False,
|
||||
use_aot_compile: bool = True,
|
||||
) -> str | Callable:
|
||||
"""Initialize the backend for the compilation config from a vllm config.
|
||||
|
||||
@@ -1093,6 +1094,7 @@ class CompilationConfig:
|
||||
prefix: Cache directory prefix for this compiled module.
|
||||
is_encoder: Whether this module is used in an encoder (as
|
||||
opposed to a text backbone).
|
||||
use_aot_compile: Whether the module permits AOT and direct artifacts.
|
||||
|
||||
Returns:
|
||||
The backend for the compilation config.
|
||||
@@ -1124,7 +1126,12 @@ class CompilationConfig:
|
||||
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
|
||||
return VllmBackend(vllm_config, prefix=prefix, is_encoder=is_encoder)
|
||||
return VllmBackend(
|
||||
vllm_config,
|
||||
prefix=prefix,
|
||||
is_encoder=is_encoder,
|
||||
use_aot_compile=use_aot_compile,
|
||||
)
|
||||
|
||||
def post_init_cudagraph_sizes(self) -> None:
|
||||
"""To complete the initialization after cudagraph related
|
||||
|
||||
Reference in New Issue
Block a user