AOT Compilation for torch.compile (Bundled) (#24274)

Signed-off-by: zhxchen17 <zhxchen17@fb.com>
This commit is contained in:
Zhengxu Chen
2025-10-10 19:02:11 -04:00
committed by GitHub
parent e317414ce1
commit eef921f45e
9 changed files with 484 additions and 41 deletions
+23
View File
@@ -10,6 +10,7 @@ from typing import Callable, Optional
import torch
import vllm.envs as envs
from vllm.config import CompilationLevel, CUDAGraphMode, get_current_vllm_config
from vllm.logger import init_logger
@@ -44,6 +45,19 @@ class TorchCompileWrapperWithCustomDispatcher:
options = (
get_current_vllm_config().compilation_config.inductor_compile_config
)
if envs.VLLM_USE_AOT_COMPILE:
options = options or {}
# This effectively drop all the guards.
# We need this because bytecode hook is not used any more to
# drop guards in the AOT compile mode.
options["guard_filter_fn"] = lambda guards: [False for _ in guards]
if hasattr(torch._dynamo.config, "enable_aot_compile"):
torch._dynamo.config.enable_aot_compile = True
else:
msg = "torch._dynamo.config.enable_aot_compile is not "
msg += "available. AOT compile is disabled and please "
msg += "upgrade PyTorch version to use AOT compile."
logger.warning(msg)
compiled_callable = torch.compile(
self.forward, fullgraph=True, backend=backend, options=options
@@ -61,6 +75,15 @@ class TorchCompileWrapperWithCustomDispatcher:
compilation_level >= CompilationLevel.DYNAMO_ONCE
)
def aot_compile(self, *args, **kwargs):
if not hasattr(self.compiled_callable, "aot_compile"):
raise RuntimeError(
"aot_compile is not supported by the current configuration. "
+ "Please make sure torch.compile is enabled with the latest "
+ f"version of PyTorch (current using torch: {torch.__version__})"
)
return self.compiled_callable.aot_compile((args, kwargs))
def __call__(self, *args, **kwargs):
"""Implement the dispatch logic here, beyond the torch.compile level.
NOTE: this function can have additional arguments beyond the forward