AOT Compilation for torch.compile (Bundled) (#24274)
Signed-off-by: zhxchen17 <zhxchen17@fb.com>
This commit is contained in:
@@ -10,6 +10,7 @@ from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import CompilationLevel, CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -44,6 +45,19 @@ class TorchCompileWrapperWithCustomDispatcher:
|
||||
options = (
|
||||
get_current_vllm_config().compilation_config.inductor_compile_config
|
||||
)
|
||||
if envs.VLLM_USE_AOT_COMPILE:
|
||||
options = options or {}
|
||||
# This effectively drop all the guards.
|
||||
# We need this because bytecode hook is not used any more to
|
||||
# drop guards in the AOT compile mode.
|
||||
options["guard_filter_fn"] = lambda guards: [False for _ in guards]
|
||||
if hasattr(torch._dynamo.config, "enable_aot_compile"):
|
||||
torch._dynamo.config.enable_aot_compile = True
|
||||
else:
|
||||
msg = "torch._dynamo.config.enable_aot_compile is not "
|
||||
msg += "available. AOT compile is disabled and please "
|
||||
msg += "upgrade PyTorch version to use AOT compile."
|
||||
logger.warning(msg)
|
||||
|
||||
compiled_callable = torch.compile(
|
||||
self.forward, fullgraph=True, backend=backend, options=options
|
||||
@@ -61,6 +75,15 @@ class TorchCompileWrapperWithCustomDispatcher:
|
||||
compilation_level >= CompilationLevel.DYNAMO_ONCE
|
||||
)
|
||||
|
||||
def aot_compile(self, *args, **kwargs):
|
||||
if not hasattr(self.compiled_callable, "aot_compile"):
|
||||
raise RuntimeError(
|
||||
"aot_compile is not supported by the current configuration. "
|
||||
+ "Please make sure torch.compile is enabled with the latest "
|
||||
+ f"version of PyTorch (current using torch: {torch.__version__})"
|
||||
)
|
||||
return self.compiled_callable.aot_compile((args, kwargs))
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Implement the dispatch logic here, beyond the torch.compile level.
|
||||
NOTE: this function can have additional arguments beyond the forward
|
||||
|
||||
Reference in New Issue
Block a user