AOT Compilation for torch.compile (Bundled) (#24274)

Signed-off-by: zhxchen17 <zhxchen17@fb.com>
This commit is contained in:
Zhengxu Chen
2025-10-10 19:02:11 -04:00
committed by GitHub
parent e317414ce1
commit eef921f45e
9 changed files with 484 additions and 41 deletions
+105 -8
View File
@@ -2,7 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import hashlib
import inspect
import os
import sys
from typing import Callable, Optional, TypeVar, Union, overload
from unittest.mock import patch
@@ -11,9 +14,10 @@ import torch.nn as nn
from packaging import version
from torch._dynamo.symbolic_convert import InliningInstructionTranslator
import vllm.envs as envs
from vllm.compilation.counter import compilation_counter
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
from vllm.config import CompilationLevel, VllmConfig
from vllm.config import CompilationLevel, VllmConfig, set_current_vllm_config
from vllm.logger import init_logger
from vllm.sequence import IntermediateTensors
from vllm.utils import resolve_obj_by_qualname, supports_dynamo
@@ -176,6 +180,33 @@ def support_torch_compile(
return cls_decorator_helper
def _model_hash_key(fn) -> str:
import vllm
sha256_hash = hashlib.sha256()
sha256_hash.update(vllm.__version__.encode())
sha256_hash.update(fn.__qualname__.encode())
sha256_hash.update(str(fn.__code__.co_firstlineno).encode())
return sha256_hash.hexdigest()
def _verify_source_unchanged(source_info, vllm_config) -> None:
from .caching import _compute_code_hash, _compute_code_hash_with_content
file_contents = {}
for source in source_info.inlined_sources:
module = sys.modules[source.module]
file = inspect.getfile(module)
vllm_config.compilation_config.traced_files.add(file)
file_contents[file] = source.content
expected_checksum = _compute_code_hash_with_content(file_contents)
actual_checksum = _compute_code_hash(set(file_contents.keys()))
if expected_checksum != actual_checksum:
raise RuntimeError(
"Source code has changed since the last compilation. Recompiling the model."
)
def _support_torch_compile(
cls: _T,
dynamic_arg_dims: dict[str, Union[int, list[int]]],
@@ -227,6 +258,64 @@ def _support_torch_compile(
if self.do_not_compile or torch.compiler.is_compiling():
return self.forward(*args, **kwargs)
if getattr(self, "aot_compiled_fn", None) is not None:
return self.aot_compiled_fn(self, *args, **kwargs)
cache_dir = None
aot_compilation_path = None
if envs.VLLM_USE_AOT_COMPILE:
"""
When using torch.compile in AOT mode, we store the cache artifacts
under VLLM_CACHE_ROOT/torch_aot_compile/{hash}/rank_i_j. The {hash}
contains all of the factors except for the source files being
traced through, because we don't actually know which source files
to check at this point (before dynamo runs).
On loading we will actually look at the source files being traced
through. If any source file have changed (compared with the
serialized backend artifacts), then we need to generate a new AOT
compile artifact from scratch.
"""
from .caching import compilation_config_hash_factors
factors: list[str] = compilation_config_hash_factors(self.vllm_config)
factors.append(_model_hash_key(self.forward))
hash_key = hashlib.sha256(str(factors).encode()).hexdigest()
cache_dir = os.path.join(
envs.VLLM_CACHE_ROOT,
"torch_aot_compile",
hash_key,
)
rank = self.vllm_config.parallel_config.rank
dp_rank = self.vllm_config.parallel_config.data_parallel_rank
cache_dir = os.path.join(cache_dir, f"rank_{rank}_{dp_rank}")
aot_compilation_path = os.path.join(cache_dir, "model")
try:
with (
set_current_vllm_config(self.vllm_config),
open(aot_compilation_path, "rb") as f,
):
start_monitoring_torch_compile(self.vllm_config)
loaded_fn = torch.compiler.load_compiled_function(f)
_verify_source_unchanged(loaded_fn.source_info(), self.vllm_config)
self.aot_compiled_fn = loaded_fn
except Exception as e:
if os.path.exists(aot_compilation_path):
logger.warning(
"Cannot load aot compilation from path %s, error: %s",
aot_compilation_path,
str(e),
)
if envs.VLLM_FORCE_AOT_LOAD:
raise e
if getattr(self, "aot_compiled_fn", None) is not None:
logger.info(
"Directly load AOT compilation from path %s", aot_compilation_path
)
return self.aot_compiled_fn(self, *args, **kwargs)
# the first compilation needs to have dynamic shapes marked
if len(self.compiled_codes) < 1:
sig = inspect.signature(self.__class__.forward)
@@ -275,15 +364,15 @@ def _support_torch_compile(
)
# 2. every time Dynamo sees a function call, it will inline
# the function by calling InliningInstructionTranslator.inline_call
# the function by calling InliningInstructionTranslator.inline_call_
# we hijack this function to know all the functions called
# during Dynamo tracing, and their corresponding files
inline_call = InliningInstructionTranslator.inline_call
inline_call = InliningInstructionTranslator.inline_call_
def patched_inline_call(parent, func, args, kwargs):
code = func.get_code()
def patched_inline_call(self_):
code = self_.f_code
self.vllm_config.compilation_config.traced_files.add(code.co_filename)
return inline_call(parent, func, args, kwargs)
return inline_call(self_)
# Disable the C++ compilation of symbolic shape guards. C++-fication
# of symbolic shape guards can improve guard overhead. But, since
@@ -300,13 +389,21 @@ def _support_torch_compile(
with (
patch.object(
InliningInstructionTranslator, "inline_call", patched_inline_call
InliningInstructionTranslator, "inline_call_", patched_inline_call
),
torch._dynamo.config.patch(**dynamo_config_patches),
maybe_use_cudagraph_partition_wrapper(self.vllm_config),
_torch27_patch_tensor_subclasses(),
):
output = self.compiled_callable(*args, **kwargs)
if envs.VLLM_USE_AOT_COMPILE:
self.aot_compiled_fn = self.aot_compile(*args, **kwargs)
output = self.aot_compiled_fn(self, *args, **kwargs)
assert aot_compilation_path is not None
assert cache_dir is not None
os.makedirs(cache_dir, exist_ok=True)
self.aot_compiled_fn.save_compiled_function(aot_compilation_path)
else:
output = self.compiled_callable(*args, **kwargs)
return output
# usually, capturing the model once is enough, and then we can