[torch.compile] integration with compilation control (#9058)
This commit is contained in:
@@ -1,8 +1,17 @@
|
||||
import copy
|
||||
import operator
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.fx as fx
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .compile_context import get_compile_context
|
||||
from .levels import CompilationLevel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def fix_functionalization(graph: fx.Graph):
|
||||
"""
|
||||
@@ -148,9 +157,113 @@ def fix_functionalization(graph: fx.Graph):
|
||||
# print(graph.python_code(root_module="self", verbose=True).src, file=f)
|
||||
|
||||
|
||||
def vllm_backend(graph, example_inputs):
|
||||
def wrap_inductor(graph, example_inputs, additional_inductor_config):
|
||||
from torch._inductor import config
|
||||
current_config = config.shallow_copy_dict()
|
||||
from torch._inductor.compile_fx import compile_fx
|
||||
|
||||
if additional_inductor_config is not None:
|
||||
current_config.update(additional_inductor_config)
|
||||
if current_config['post_grad_custom_post_pass'] is not None:
|
||||
logger.warning(
|
||||
"post_grad_custom_post_pass is already set in the config. "
|
||||
"Overwriting it with the fix_functionalization")
|
||||
current_config['post_grad_custom_post_pass'] = fix_functionalization
|
||||
return compile_fx(graph, example_inputs, config_patches=current_config)
|
||||
|
||||
|
||||
def vllm_backend(
|
||||
graph,
|
||||
example_inputs,
|
||||
additional_inductor_config: Optional[Dict] = None) -> Callable:
|
||||
|
||||
context = get_compile_context()
|
||||
context = copy.deepcopy(context) if context is not None else []
|
||||
sizes_to_specialize: List[int] = context
|
||||
|
||||
# flags for all the seen shapes, whether we need to specialize
|
||||
runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {}
|
||||
|
||||
# if we need to specialize, the compiled graph for that shape
|
||||
runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {}
|
||||
|
||||
# this is the first compilation, we will compile a graph with
|
||||
# dynamic shape, as the caller will mark first dimension as dynamic
|
||||
logger.info("Compiling a graph for general shapes")
|
||||
graph_for_symbolic_shape = wrap_inductor(graph, example_inputs,
|
||||
additional_inductor_config)
|
||||
|
||||
# TODO: Dynamo does not pass all dynamic shapes.
|
||||
# Need to investigate why. It works now because all the dynamic
|
||||
# shapes have the same value, and either of them can be used.
|
||||
sym_shape_indices = [
|
||||
i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt)
|
||||
]
|
||||
|
||||
first_run = True
|
||||
|
||||
# this is the function we return to Dynamo to run finally
|
||||
def compiled_graph_wrapper(*args):
|
||||
|
||||
runtime_shapes: Tuple[int,
|
||||
...] = tuple(args[i] for i in sym_shape_indices)
|
||||
|
||||
nonlocal first_run
|
||||
nonlocal runtime_shapes_to_compile_flags
|
||||
nonlocal runtime_shapes_to_compiled_graph
|
||||
|
||||
if first_run:
|
||||
# the first compilation is for profiling, we directly run it
|
||||
first_run = False
|
||||
return graph_for_symbolic_shape(*args)
|
||||
|
||||
if runtime_shapes not in runtime_shapes_to_compile_flags:
|
||||
# we haven't seen this shape before
|
||||
# query if we need to specialize for this shape
|
||||
# we only specialize for the first dimension.
|
||||
# TODO: investigate if any model needs to specialize
|
||||
# beyond the first dimension
|
||||
runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[
|
||||
0] in sizes_to_specialize
|
||||
|
||||
if not runtime_shapes_to_compile_flags[runtime_shapes]:
|
||||
# we don't need to specialize for this shape
|
||||
return graph_for_symbolic_shape(*args)
|
||||
|
||||
if runtime_shapes not in runtime_shapes_to_compiled_graph:
|
||||
# we need to specialize for this shape, and we haven't compiled
|
||||
# compile the graph for this shape
|
||||
logger.info("Compiling a graph for shapes %s", runtime_shapes)
|
||||
runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor(
|
||||
graph, args, additional_inductor_config)
|
||||
|
||||
return runtime_shapes_to_compiled_graph[runtime_shapes](*args)
|
||||
|
||||
return compiled_graph_wrapper
|
||||
|
||||
|
||||
def select_default_backend(level: int) -> Union[str, Callable]:
|
||||
if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
|
||||
backend = "eager"
|
||||
return backend
|
||||
assert level in [
|
||||
CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
|
||||
], f"Invalid level {level}"
|
||||
|
||||
from vllm.compilation.backends import vllm_backend
|
||||
from vllm.plugins import get_inductor_additional_configs
|
||||
additional_configs = get_inductor_additional_configs()
|
||||
|
||||
if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE:
|
||||
if "max_autotune" in additional_configs and not additional_configs[
|
||||
"max_autotune"]:
|
||||
logger.warning(
|
||||
"max_autotune is disabled, but is overridden by level %s",
|
||||
CompilationLevel.INDUCTOR_MAX_AUTOTUNE)
|
||||
additional_configs['max_autotune'] = True
|
||||
|
||||
from functools import partial
|
||||
backend = partial(vllm_backend,
|
||||
additional_inductor_config=additional_configs)
|
||||
|
||||
return backend
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
_compile_context: Any = None
|
||||
|
||||
|
||||
def get_compile_context() -> Any:
|
||||
"""Get the current compile context."""
|
||||
return _compile_context
|
||||
|
||||
|
||||
@contextmanager
|
||||
def set_compile_context(context: Any):
|
||||
"""A context manager that stores the current compile context,
|
||||
usually it is a list of sizes to specialize.
|
||||
"""
|
||||
global _compile_context
|
||||
prev_context = _compile_context
|
||||
_compile_context = context
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_compile_context = prev_context
|
||||
@@ -0,0 +1,85 @@
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.attention import AttentionMetadata
|
||||
from vllm.compilation.levels import CompilationLevel
|
||||
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils import supports_dynamo
|
||||
|
||||
|
||||
def support_compile_llama_style(cls: type):
|
||||
"""
|
||||
A decorator to add support for compiling the forward method of a class.
|
||||
If a module's **forward signature** is compatible with llama, this
|
||||
decorator can be used to enable the compilation of the forward method.
|
||||
"""
|
||||
|
||||
# for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
|
||||
# will handle the compilation, so we don't need to do anything here.
|
||||
if envs.VLLM_TORCH_COMPILE_LEVEL in [
|
||||
CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
|
||||
] or not supports_dynamo():
|
||||
return cls
|
||||
|
||||
# take care of method resolution order
|
||||
# make sure super().__init__ is called on the base class
|
||||
# other than TorchCompileWrapperWithCustomDispatcher
|
||||
cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
|
||||
|
||||
old_init = cls.__init__
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
old_init(self, *args, **kwargs)
|
||||
TorchCompileWrapperWithCustomDispatcher.__init__(self)
|
||||
|
||||
cls.__init__ = __init__
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
input_ids: Optional[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
kv_caches: List[torch.Tensor],
|
||||
attn_metadata: AttentionMetadata,
|
||||
intermediate_tensors: Optional[IntermediateTensors],
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
# torch.compiler.is_compiling() means we are inside the compilation
|
||||
# e.g. TPU has the compilation logic in model runner, so we don't
|
||||
# need to compile the model inside.
|
||||
if torch.compiler.is_compiling():
|
||||
return self.forward(input_ids, positions, kv_caches, attn_metadata,
|
||||
intermediate_tensors, inputs_embeds)
|
||||
|
||||
# the first compilation needs to have dynamic shapes marked
|
||||
if len(self.compiled_codes) < 1:
|
||||
if input_ids is not None:
|
||||
torch._dynamo.mark_dynamic(input_ids, 0)
|
||||
torch._dynamo.mark_dynamic(positions, 0)
|
||||
if inputs_embeds is not None:
|
||||
torch._dynamo.mark_dynamic(inputs_embeds, 0)
|
||||
if intermediate_tensors is not None:
|
||||
for tensors in intermediate_tensors.tensors.values():
|
||||
torch._dynamo.mark_dynamic(tensors, 0)
|
||||
|
||||
# if we don't use custom dispatcher, we can directly call the
|
||||
# compiled function and let torch.compile handle the dispatching,
|
||||
# with the overhead of guard evaluation and recompilation.
|
||||
if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
|
||||
return self.compiled_callable(input_ids, positions, kv_caches,
|
||||
attn_metadata, intermediate_tensors,
|
||||
inputs_embeds)
|
||||
|
||||
# usually, capturing the model once is enough, and then we can
|
||||
# dispatch to the compiled code directly, without going through
|
||||
# the Dynamo guard mechanism.
|
||||
with self.dispatch_to_code(0):
|
||||
model_output = self.forward(input_ids, positions, kv_caches,
|
||||
attn_metadata, intermediate_tensors,
|
||||
inputs_embeds)
|
||||
return model_output
|
||||
|
||||
cls.__call__ = __call__
|
||||
return cls
|
||||
@@ -0,0 +1,9 @@
|
||||
# constants for the levels of the compilation process
|
||||
|
||||
|
||||
class CompilationLevel:
|
||||
NO_COMPILATION = 0
|
||||
DYNAMO_AS_IS = 1
|
||||
DYNAMO_ONCE = 2
|
||||
INDUCTOR = 3
|
||||
INDUCTOR_MAX_AUTOTUNE = 4
|
||||
@@ -3,12 +3,14 @@ import sys
|
||||
from abc import abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from types import CodeType
|
||||
from typing import Callable, List
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
from .levels import CompilationLevel
|
||||
|
||||
|
||||
class TorchCompileWrapperWithCustomDispatcher:
|
||||
"""
|
||||
@@ -23,7 +25,26 @@ class TorchCompileWrapperWithCustomDispatcher:
|
||||
`torch.compile` over the forward method.
|
||||
"""
|
||||
|
||||
def __init__(self, compiled_callable: Callable):
|
||||
def __init__(self, compiled_callable: Optional[Callable] = None):
|
||||
|
||||
if compiled_callable is None:
|
||||
# default compilation settings
|
||||
# compiling the forward method
|
||||
|
||||
# choose the compile backend
|
||||
|
||||
# if the user has set the backend, use it
|
||||
from vllm.plugins import get_torch_compile_backend
|
||||
backend = get_torch_compile_backend()
|
||||
if backend is None:
|
||||
from vllm.compilation.backends import select_default_backend
|
||||
backend = select_default_backend(envs.VLLM_TORCH_COMPILE_LEVEL)
|
||||
|
||||
compiled_callable = torch.compile(
|
||||
self.forward,
|
||||
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
|
||||
backend=backend)
|
||||
|
||||
self.compiled_callable = compiled_callable
|
||||
self.original_code_object = self.__class__.forward.__code__
|
||||
self.compiled_codes: List[CodeType] = []
|
||||
@@ -33,7 +54,7 @@ class TorchCompileWrapperWithCustomDispatcher:
|
||||
# subclasses can use this to switch between the custom dispatcher
|
||||
# and the default Dynamo guard mechanism.
|
||||
self.use_custom_dispatcher: bool = \
|
||||
envs.VLLM_DYNAMO_USE_CUSTOM_DISPATCHER
|
||||
envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.DYNAMO_ONCE
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Implement the dispatch logic here, beyond the torch.compile level.
|
||||
|
||||
Reference in New Issue
Block a user