[torch.compile] integration with compilation control (#9058)
This commit is contained in:
@@ -1,8 +1,17 @@
|
||||
import copy
|
||||
import operator
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.fx as fx
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .compile_context import get_compile_context
|
||||
from .levels import CompilationLevel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def fix_functionalization(graph: fx.Graph):
|
||||
"""
|
||||
@@ -148,9 +157,113 @@ def fix_functionalization(graph: fx.Graph):
|
||||
# print(graph.python_code(root_module="self", verbose=True).src, file=f)
|
||||
|
||||
|
||||
def vllm_backend(graph, example_inputs):
|
||||
def wrap_inductor(graph, example_inputs, additional_inductor_config):
|
||||
from torch._inductor import config
|
||||
current_config = config.shallow_copy_dict()
|
||||
from torch._inductor.compile_fx import compile_fx
|
||||
|
||||
if additional_inductor_config is not None:
|
||||
current_config.update(additional_inductor_config)
|
||||
if current_config['post_grad_custom_post_pass'] is not None:
|
||||
logger.warning(
|
||||
"post_grad_custom_post_pass is already set in the config. "
|
||||
"Overwriting it with the fix_functionalization")
|
||||
current_config['post_grad_custom_post_pass'] = fix_functionalization
|
||||
return compile_fx(graph, example_inputs, config_patches=current_config)
|
||||
|
||||
|
||||
def vllm_backend(
|
||||
graph,
|
||||
example_inputs,
|
||||
additional_inductor_config: Optional[Dict] = None) -> Callable:
|
||||
|
||||
context = get_compile_context()
|
||||
context = copy.deepcopy(context) if context is not None else []
|
||||
sizes_to_specialize: List[int] = context
|
||||
|
||||
# flags for all the seen shapes, whether we need to specialize
|
||||
runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {}
|
||||
|
||||
# if we need to specialize, the compiled graph for that shape
|
||||
runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {}
|
||||
|
||||
# this is the first compilation, we will compile a graph with
|
||||
# dynamic shape, as the caller will mark first dimension as dynamic
|
||||
logger.info("Compiling a graph for general shapes")
|
||||
graph_for_symbolic_shape = wrap_inductor(graph, example_inputs,
|
||||
additional_inductor_config)
|
||||
|
||||
# TODO: Dynamo does not pass all dynamic shapes.
|
||||
# Need to investigate why. It works now because all the dynamic
|
||||
# shapes have the same value, and either of them can be used.
|
||||
sym_shape_indices = [
|
||||
i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt)
|
||||
]
|
||||
|
||||
first_run = True
|
||||
|
||||
# this is the function we return to Dynamo to run finally
|
||||
def compiled_graph_wrapper(*args):
|
||||
|
||||
runtime_shapes: Tuple[int,
|
||||
...] = tuple(args[i] for i in sym_shape_indices)
|
||||
|
||||
nonlocal first_run
|
||||
nonlocal runtime_shapes_to_compile_flags
|
||||
nonlocal runtime_shapes_to_compiled_graph
|
||||
|
||||
if first_run:
|
||||
# the first compilation is for profiling, we directly run it
|
||||
first_run = False
|
||||
return graph_for_symbolic_shape(*args)
|
||||
|
||||
if runtime_shapes not in runtime_shapes_to_compile_flags:
|
||||
# we haven't seen this shape before
|
||||
# query if we need to specialize for this shape
|
||||
# we only specialize for the first dimension.
|
||||
# TODO: investigate if any model needs to specialize
|
||||
# beyond the first dimension
|
||||
runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[
|
||||
0] in sizes_to_specialize
|
||||
|
||||
if not runtime_shapes_to_compile_flags[runtime_shapes]:
|
||||
# we don't need to specialize for this shape
|
||||
return graph_for_symbolic_shape(*args)
|
||||
|
||||
if runtime_shapes not in runtime_shapes_to_compiled_graph:
|
||||
# we need to specialize for this shape, and we haven't compiled
|
||||
# compile the graph for this shape
|
||||
logger.info("Compiling a graph for shapes %s", runtime_shapes)
|
||||
runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor(
|
||||
graph, args, additional_inductor_config)
|
||||
|
||||
return runtime_shapes_to_compiled_graph[runtime_shapes](*args)
|
||||
|
||||
return compiled_graph_wrapper
|
||||
|
||||
|
||||
def select_default_backend(level: int) -> Union[str, Callable]:
|
||||
if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
|
||||
backend = "eager"
|
||||
return backend
|
||||
assert level in [
|
||||
CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
|
||||
], f"Invalid level {level}"
|
||||
|
||||
from vllm.compilation.backends import vllm_backend
|
||||
from vllm.plugins import get_inductor_additional_configs
|
||||
additional_configs = get_inductor_additional_configs()
|
||||
|
||||
if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE:
|
||||
if "max_autotune" in additional_configs and not additional_configs[
|
||||
"max_autotune"]:
|
||||
logger.warning(
|
||||
"max_autotune is disabled, but is overridden by level %s",
|
||||
CompilationLevel.INDUCTOR_MAX_AUTOTUNE)
|
||||
additional_configs['max_autotune'] = True
|
||||
|
||||
from functools import partial
|
||||
backend = partial(vllm_backend,
|
||||
additional_inductor_config=additional_configs)
|
||||
|
||||
return backend
|
||||
|
||||
Reference in New Issue
Block a user