AOT Compilation for torch.compile (Bundled) (#24274)
Signed-off-by: zhxchen17 <zhxchen17@fb.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
import ast
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import os
|
||||
import pprint
|
||||
import time
|
||||
@@ -25,6 +26,7 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import is_torch_equal_or_newer, resolve_obj_by_qualname
|
||||
|
||||
from .caching import VllmSerializableFunction
|
||||
from .compiler_interface import (
|
||||
CompilerInterface,
|
||||
EagerAdaptor,
|
||||
@@ -195,6 +197,7 @@ class CompilerManager:
|
||||
# there can be multiple graphs due to piecewise compilation.
|
||||
now = time.time()
|
||||
elapsed = now - compilation_start_time
|
||||
compilation_config.compilation_time += elapsed
|
||||
if runtime_shape is None:
|
||||
logger.info(
|
||||
"Directly load the compiled graph(s) for dynamic shape "
|
||||
@@ -549,7 +552,11 @@ class VllmBackend:
|
||||
self.post_grad_pass_manager.add(inductor_config[PASS_KEY])
|
||||
inductor_config[PASS_KEY] = self.post_grad_pass_manager
|
||||
|
||||
def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
|
||||
def __call__(
|
||||
self, graph: fx.GraphModule, example_inputs
|
||||
) -> VllmSerializableFunction:
|
||||
from .caching import _compute_code_hash, compilation_config_hash_factors
|
||||
|
||||
vllm_config = self.vllm_config
|
||||
if not self.compilation_config.cache_dir:
|
||||
# no provided cache dir, generate one based on the known factors
|
||||
@@ -557,39 +564,11 @@ class VllmBackend:
|
||||
# the cache dir will be the same so that we can reuse the compiled
|
||||
# graph.
|
||||
|
||||
factors = []
|
||||
# 0. factors come from the env, for example, The values of
|
||||
# VLLM_PP_LAYER_PARTITION will affect the computation graph.
|
||||
env_hash = envs.compute_hash()
|
||||
factors.append(env_hash)
|
||||
|
||||
# 1. factors come from the vllm_config (it mainly summarizes how the
|
||||
# model is created)
|
||||
config_hash = vllm_config.compute_hash()
|
||||
factors.append(config_hash)
|
||||
|
||||
factors = compilation_config_hash_factors(vllm_config)
|
||||
# 2. factors come from the code files that are traced by Dynamo (
|
||||
# it mainly summarizes how the model is used in forward pass)
|
||||
forward_code_files = list(sorted(self.compilation_config.traced_files))
|
||||
code_hash = _compute_code_hash(self.compilation_config.traced_files)
|
||||
self.compilation_config.traced_files.clear()
|
||||
logger.debug(
|
||||
"Traced files (to be considered for compilation cache):\n%s",
|
||||
"\n".join(forward_code_files),
|
||||
)
|
||||
hash_content = []
|
||||
for filepath in forward_code_files:
|
||||
hash_content.append(filepath)
|
||||
if filepath == "<string>":
|
||||
# This means the function was dynamically generated, with
|
||||
# e.g. exec(). We can't actually check these.
|
||||
continue
|
||||
with open(filepath) as f:
|
||||
hash_content.append(f.read())
|
||||
import hashlib
|
||||
|
||||
code_hash = hashlib.md5(
|
||||
"\n".join(hash_content).encode(), usedforsecurity=False
|
||||
).hexdigest()
|
||||
factors.append(code_hash)
|
||||
|
||||
# 3. compiler hash
|
||||
@@ -695,7 +674,9 @@ class VllmBackend:
|
||||
self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
|
||||
or not self.compilation_config.cudagraph_copy_inputs
|
||||
):
|
||||
return self.split_gm
|
||||
return VllmSerializableFunction(
|
||||
graph, example_inputs, self.prefix, self.split_gm
|
||||
)
|
||||
|
||||
# if we need to copy input buffers for cudagraph
|
||||
from torch._guards import detect_fake_mode
|
||||
@@ -740,4 +721,6 @@ class VllmBackend:
|
||||
list_args[index] = static_tensor
|
||||
return self.split_gm(*list_args)
|
||||
|
||||
return copy_and_call
|
||||
return VllmSerializableFunction(
|
||||
graph, example_inputs, self.prefix, copy_and_call
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user