[torch.compile] rework compile control with piecewise cudagraph (#9715)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
+296
-80
@@ -1,13 +1,16 @@
|
||||
import copy
|
||||
import dataclasses
|
||||
import operator
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.fx as fx
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import weak_ref_tensors
|
||||
|
||||
from .compile_context import get_compile_context
|
||||
from .config import CompilationConfig
|
||||
from .counter import compilation_counter
|
||||
from .levels import CompilationLevel
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -157,113 +160,326 @@ def fix_functionalization(graph: fx.Graph):
|
||||
# print(graph.python_code(root_module="self", verbose=True).src, file=f)
|
||||
|
||||
|
||||
def wrap_inductor(graph, example_inputs, additional_inductor_config):
|
||||
def wrap_inductor(graph,
|
||||
example_inputs,
|
||||
additional_inductor_config,
|
||||
do_logging=False,
|
||||
runtime_shape: Optional[int] = None,
|
||||
use_inductor: bool = True):
|
||||
if not use_inductor:
|
||||
return graph
|
||||
|
||||
compilation_counter.num_inductor_compilations += 1
|
||||
|
||||
if do_logging:
|
||||
if runtime_shape is None:
|
||||
logger.info("Compiling a graph for general shape")
|
||||
else:
|
||||
logger.info("Compiling a graph for shape %s", runtime_shape)
|
||||
|
||||
from torch._inductor import config
|
||||
current_config = config.shallow_copy_dict()
|
||||
from torch._inductor.compile_fx import compile_fx
|
||||
|
||||
if additional_inductor_config is not None:
|
||||
current_config.update(additional_inductor_config)
|
||||
if current_config['post_grad_custom_post_pass'] is not None:
|
||||
logger.warning(
|
||||
"post_grad_custom_post_pass is already set in the config. "
|
||||
"Overwriting it with the fix_functionalization")
|
||||
current_config['post_grad_custom_post_pass'] = fix_functionalization
|
||||
|
||||
# inductor can inplace modify the graph, so we need to copy it
|
||||
# see https://github.com/pytorch/pytorch/issues/138980
|
||||
graph = copy.deepcopy(graph)
|
||||
return compile_fx(graph, example_inputs, config_patches=current_config)
|
||||
|
||||
|
||||
def vllm_backend(
|
||||
@dataclasses.dataclass
|
||||
class SplitItem:
|
||||
submod_name: str
|
||||
is_splitting_graph: bool
|
||||
graph: fx.GraphModule
|
||||
|
||||
|
||||
def split_graph(graph: fx.GraphModule,
|
||||
ops: List[str]) -> Tuple[fx.GraphModule, List[SplitItem]]:
|
||||
# split graph by ops
|
||||
subgraph_id = 0
|
||||
node_to_subgraph_id = {}
|
||||
split_op_graphs = []
|
||||
for node in graph.graph.nodes:
|
||||
if node.op in ("output", "placeholder"):
|
||||
continue
|
||||
if node.op == 'call_function' and str(node.target) in ops:
|
||||
subgraph_id += 1
|
||||
node_to_subgraph_id[node] = subgraph_id
|
||||
split_op_graphs.append(subgraph_id)
|
||||
subgraph_id += 1
|
||||
else:
|
||||
node_to_subgraph_id[node] = subgraph_id
|
||||
|
||||
# `keep_original_order` is important!
|
||||
# otherwise pytorch might reorder the nodes and
|
||||
# the semantics of the graph will change when we
|
||||
# have mutations in the graph
|
||||
split_gm = torch.fx.passes.split_module.split_module(
|
||||
graph,
|
||||
example_inputs,
|
||||
additional_inductor_config: Optional[Dict] = None) -> Callable:
|
||||
None,
|
||||
lambda node: node_to_subgraph_id[node],
|
||||
keep_original_order=True)
|
||||
|
||||
context = get_compile_context()
|
||||
context = copy.deepcopy(context) if context is not None else []
|
||||
sizes_to_specialize: List[int] = context
|
||||
outputs = []
|
||||
|
||||
# flags for all the seen shapes, whether we need to specialize
|
||||
runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {}
|
||||
# sort the names to make sure the order is deterministic
|
||||
names = [name for (name, module) in split_gm.named_modules()]
|
||||
names.sort()
|
||||
|
||||
# if we need to specialize, the compiled graph for that shape
|
||||
runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {}
|
||||
for name in names:
|
||||
if "." in name or name == "":
|
||||
# recursive child module or the root module
|
||||
continue
|
||||
|
||||
# this is the first compilation, we will compile a graph with
|
||||
# dynamic shape, as the caller will mark first dimension as dynamic
|
||||
logger.info("Compiling a graph for general shapes")
|
||||
graph_for_symbolic_shape = wrap_inductor(graph, example_inputs,
|
||||
additional_inductor_config)
|
||||
module = getattr(split_gm, name)
|
||||
|
||||
# TODO: Dynamo does not pass all dynamic shapes.
|
||||
# Need to investigate why. It works now because all the dynamic
|
||||
# shapes have the same value, and either of them can be used.
|
||||
sym_shape_indices = [
|
||||
i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt)
|
||||
]
|
||||
graph_id = int(name.replace("submod_", ""))
|
||||
outputs.append(SplitItem(name, graph_id in split_op_graphs, module))
|
||||
|
||||
first_run = True
|
||||
return split_gm, outputs
|
||||
|
||||
# this is the function we return to Dynamo to run finally
|
||||
def compiled_graph_wrapper(*args):
|
||||
|
||||
runtime_shapes: Tuple[int,
|
||||
...] = tuple(args[i] for i in sym_shape_indices)
|
||||
class VllmBackend:
|
||||
"""The compilation backend for `torch.compile` with VLLM.
|
||||
It is used for compilation level of `CompilationLevel.PIECEWISE`,
|
||||
where we customize the compilation.
|
||||
|
||||
nonlocal first_run
|
||||
nonlocal runtime_shapes_to_compile_flags
|
||||
nonlocal runtime_shapes_to_compiled_graph
|
||||
The major work of this backend is to split the graph into
|
||||
piecewise graphs, and pass them to the piecewise backend.
|
||||
"""
|
||||
|
||||
if first_run:
|
||||
# the first compilation is for profiling, we directly run it
|
||||
first_run = False
|
||||
return graph_for_symbolic_shape(*args)
|
||||
compilation_configs: CompilationConfig
|
||||
graph_pool: Any
|
||||
_called: bool = False
|
||||
# the graph we compiled
|
||||
graph: fx.GraphModule
|
||||
# the stiching graph module for all the piecewise graphs
|
||||
split_gm: fx.GraphModule
|
||||
piecewise_graphs: List[SplitItem]
|
||||
returned_callable: Callable
|
||||
|
||||
if runtime_shapes not in runtime_shapes_to_compile_flags:
|
||||
# we haven't seen this shape before
|
||||
# query if we need to specialize for this shape
|
||||
# we only specialize for the first dimension.
|
||||
# TODO: investigate if any model needs to specialize
|
||||
# beyond the first dimension
|
||||
runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[
|
||||
0] in sizes_to_specialize
|
||||
def __init__(self, ):
|
||||
# every instance of VllmBackend has its own graph pool
|
||||
self.graph_pool = torch.cuda.graph_pool_handle()
|
||||
|
||||
if not runtime_shapes_to_compile_flags[runtime_shapes]:
|
||||
# we don't need to specialize for this shape
|
||||
return graph_for_symbolic_shape(*args)
|
||||
# `torch.compile` is JIT compiled, so we don't need to
|
||||
# do anything here
|
||||
|
||||
if runtime_shapes not in runtime_shapes_to_compiled_graph:
|
||||
# we need to specialize for this shape, and we haven't compiled
|
||||
# compile the graph for this shape
|
||||
logger.info("Compiling a graph for shapes %s", runtime_shapes)
|
||||
runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor(
|
||||
graph, args, additional_inductor_config)
|
||||
def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
|
||||
|
||||
return runtime_shapes_to_compiled_graph[runtime_shapes](*args)
|
||||
compilation_counter.num_graphs_seen += 1
|
||||
|
||||
return compiled_graph_wrapper
|
||||
# we control the compilation process, each instance can only be
|
||||
# called once
|
||||
assert not self._called, "VllmBackend can only be called once"
|
||||
|
||||
self.graph = graph
|
||||
# config is read now, because only here can
|
||||
# we get the sizes to capture for cudagraph
|
||||
# from compilation context
|
||||
self.compilation_configs = CompilationConfig.select_and_init_config()
|
||||
|
||||
self.split_gm, self.piecewise_graphs = split_graph(
|
||||
graph, self.compilation_configs.non_cudagraph_ops)
|
||||
|
||||
returned_callable: Callable # type: ignore
|
||||
|
||||
if len(self.piecewise_graphs) == 0:
|
||||
compilation_counter.num_piecewise_graphs_seen += 1
|
||||
compilation_counter.num_piecewise_capturable_graphs_seen += 1
|
||||
returned_callable = PiecewiseBackend(graph,
|
||||
self.compilation_configs,
|
||||
self.graph_pool,
|
||||
is_first_graph=True)
|
||||
else:
|
||||
from torch._dynamo.utils import lazy_format_graph_code
|
||||
logger.debug(
|
||||
"%s", lazy_format_graph_code("stiching module", self.split_gm))
|
||||
|
||||
is_first_graph = True
|
||||
|
||||
for item in self.piecewise_graphs:
|
||||
compilation_counter.num_piecewise_graphs_seen += 1
|
||||
compilation_counter.num_piecewise_capturable_graphs_seen += not item.is_splitting_graph # noqa
|
||||
if not item.is_splitting_graph:
|
||||
# cannot setattr to a module, so we need to set
|
||||
# the attribute in the __dict__
|
||||
self.split_gm.__dict__[
|
||||
item.submod_name] = PiecewiseBackend(
|
||||
item.graph, self.compilation_configs,
|
||||
self.graph_pool, is_first_graph)
|
||||
is_first_graph = False
|
||||
returned_callable = self.split_gm
|
||||
|
||||
self.returned_callable = returned_callable
|
||||
# trigger the first compilation
|
||||
# code borrowed from https://github.com/pytorch/pytorch/blob/4e3e08b71171fa34172b2362ff668553fac75f27/torch/_dynamo/backends/distributed.py#L206 # noqa
|
||||
# to turn the inputs into fake tensors
|
||||
import torch._guards
|
||||
from torch._guards import detect_fake_mode
|
||||
fake_mode = detect_fake_mode(example_inputs)
|
||||
fake_args = []
|
||||
for arg in example_inputs:
|
||||
if isinstance(arg, torch.Tensor) and not isinstance(
|
||||
arg, torch._subclasses.FakeTensor):
|
||||
fake_args.append(
|
||||
torch._dynamo.utils.to_fake_tensor(arg, fake_mode))
|
||||
else:
|
||||
fake_args.append(arg)
|
||||
self.returned_callable(*fake_args)
|
||||
|
||||
self._called = True
|
||||
|
||||
return self.returned_callable
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ConcreteSizeEntry:
|
||||
runtime_shape: int
|
||||
need_to_compile: bool # the size is in compile_sizes
|
||||
use_cudagraph: bool # the size is in capture_sizes
|
||||
|
||||
compiled: bool = False
|
||||
runnable: Callable = None # type: ignore
|
||||
num_finished_warmup: int = 0
|
||||
cudagraph: Optional[torch.cuda.CUDAGraph] = None
|
||||
output: Optional[Any] = None
|
||||
|
||||
|
||||
class PiecewiseBackend:
|
||||
|
||||
def __init__(self,
|
||||
graph: fx.GraphModule,
|
||||
compilation_configs: CompilationConfig,
|
||||
graph_pool: Any,
|
||||
is_first_graph: bool = False):
|
||||
"""
|
||||
The backend for piecewise compilation.
|
||||
It mainly handles the compilation and cudagraph capturing.
|
||||
|
||||
We will compile `self.graph` once for the general shape,
|
||||
and then compile for different shapes specified in
|
||||
`compilation_configs.compile_sizes`.
|
||||
|
||||
Independently, we will capture cudagraph for different shapes.
|
||||
|
||||
If a shape needs both compilation and cudagraph, we will
|
||||
compile it first, and then capture cudagraph.
|
||||
"""
|
||||
self.graph = graph
|
||||
self.compilation_configs = compilation_configs
|
||||
self.graph_pool = graph_pool
|
||||
self.is_first_graph = is_first_graph
|
||||
|
||||
self.compile_sizes: Set[int] = set(
|
||||
self.compilation_configs.compile_sizes)
|
||||
self.capture_sizes: Set[int] = set(
|
||||
self.compilation_configs.capture_sizes
|
||||
) if self.compilation_configs.use_cudagraph else set()
|
||||
|
||||
self.compile_finished = False
|
||||
self.first_run_finished = False
|
||||
|
||||
self.compiled_graph_for_general_shape: Callable = None # type: ignore
|
||||
|
||||
self.sym_shape_indices: List[int] = []
|
||||
|
||||
# the entries for different shapes that we need to either
|
||||
# compile or capture cudagraph
|
||||
self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
|
||||
for shape in self.compile_sizes.union(self.capture_sizes):
|
||||
self.concrete_size_entries[shape] = ConcreteSizeEntry(
|
||||
runtime_shape=shape,
|
||||
need_to_compile=shape in self.compile_sizes,
|
||||
use_cudagraph=shape in self.capture_sizes,
|
||||
)
|
||||
|
||||
def __call__(self, *args) -> Any:
|
||||
|
||||
if not self.compile_finished:
|
||||
self.compile_finished = True
|
||||
|
||||
# this is the first compilation, we will compile a graph with
|
||||
# dynamic shape, as the caller will mark first dimension as dynamic
|
||||
|
||||
self.sym_shape_indices = [
|
||||
i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
|
||||
]
|
||||
|
||||
self.compiled_graph_for_general_shape = wrap_inductor(
|
||||
self.graph,
|
||||
args,
|
||||
self.compilation_configs.inductor_compile_config,
|
||||
runtime_shape=None,
|
||||
do_logging=self.is_first_graph,
|
||||
use_inductor=self.compilation_configs.use_inductor)
|
||||
|
||||
return self.graph(*args)
|
||||
|
||||
if not self.first_run_finished:
|
||||
self.first_run_finished = True
|
||||
return self.compiled_graph_for_general_shape(*args)
|
||||
|
||||
runtime_shape = args[self.sym_shape_indices[0]]
|
||||
if runtime_shape not in self.concrete_size_entries:
|
||||
# we don't need to do anything for this shape
|
||||
return self.compiled_graph_for_general_shape(*args)
|
||||
|
||||
entry = self.concrete_size_entries[runtime_shape]
|
||||
|
||||
if entry.runnable is None:
|
||||
entry.runnable = self.compiled_graph_for_general_shape
|
||||
|
||||
if entry.need_to_compile and not entry.compiled:
|
||||
entry.compiled = True
|
||||
# args are real arguments
|
||||
entry.runnable = wrap_inductor(
|
||||
self.graph,
|
||||
args,
|
||||
self.compilation_configs.inductor_compile_config,
|
||||
runtime_shape=runtime_shape,
|
||||
do_logging=self.is_first_graph,
|
||||
use_inductor=self.compilation_configs.use_inductor)
|
||||
|
||||
if not entry.use_cudagraph:
|
||||
return entry.runnable(*args)
|
||||
|
||||
if entry.cudagraph is None:
|
||||
if entry.num_finished_warmup < self.compilation_configs.cudagraph_num_of_warmups: # noqa
|
||||
entry.num_finished_warmup += 1
|
||||
if self.is_first_graph:
|
||||
logger.debug(
|
||||
"Warming up %s/%s for shape %s",
|
||||
entry.num_finished_warmup,
|
||||
self.compilation_configs.cudagraph_num_of_warmups,
|
||||
runtime_shape)
|
||||
return entry.runnable(*args)
|
||||
|
||||
if self.is_first_graph:
|
||||
logger.info("Capturing a cudagraph for shape %s",
|
||||
runtime_shape)
|
||||
|
||||
cudagraph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(cudagraph, pool=self.graph_pool):
|
||||
entry.output = weak_ref_tensors(entry.runnable(*args))
|
||||
|
||||
compilation_counter.num_cudagraph_caputured += 1
|
||||
|
||||
entry.cudagraph = cudagraph
|
||||
return entry.output
|
||||
|
||||
entry.cudagraph.replay()
|
||||
return entry.output
|
||||
|
||||
|
||||
def select_default_backend(level: int) -> Union[str, Callable]:
|
||||
if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
|
||||
backend_str = "eager"
|
||||
return backend_str
|
||||
assert level in [
|
||||
CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
|
||||
], f"Invalid level {level}"
|
||||
assert level == CompilationLevel.PIECEWISE
|
||||
|
||||
from vllm.compilation.backends import vllm_backend
|
||||
from vllm.plugins import get_inductor_additional_configs
|
||||
additional_configs = get_inductor_additional_configs()
|
||||
|
||||
if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE:
|
||||
if "max_autotune" in additional_configs and not additional_configs[
|
||||
"max_autotune"]:
|
||||
logger.warning(
|
||||
"max_autotune is disabled, but is overridden by level %s",
|
||||
CompilationLevel.INDUCTOR_MAX_AUTOTUNE)
|
||||
additional_configs['max_autotune'] = True
|
||||
|
||||
from functools import partial
|
||||
backend = partial(vllm_backend,
|
||||
additional_inductor_config=additional_configs)
|
||||
|
||||
return backend
|
||||
return VllmBackend()
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import copy
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, PrivateAttr
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .compile_context import get_compile_context
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CompilationConfig(BaseModel):
|
||||
"""
|
||||
Configuration for compilation.
|
||||
It has two parts:
|
||||
- CudaGraph capture:
|
||||
- use_cudagraph: whether to use cudagraph inside compilation.
|
||||
- False: cudagraph inside compilation is not used.
|
||||
- True: cudagraph inside compilation is used. It requires
|
||||
that all input buffers have fixed addresses.
|
||||
Note that this is orthogonal to the cudagraph capture out
|
||||
side of compilation.
|
||||
TODO: move outside cudagraph logic into compilation.
|
||||
torch.compile will handle cudagraph capture logic in the future.
|
||||
- cudagraph_capture_sizes: sizes to capture cudagraph.
|
||||
- None: capture sizes are inferred from compilation context.
|
||||
- List[int]: capture sizes are specified.
|
||||
- cudagraph_num_of_warmups: number of warmup runs for cudagraph.
|
||||
It means the first several runs will be treated as warmup runs.
|
||||
Only after that, the execution will be recorded, and the recorded
|
||||
cudagraph will be used for subsequent runs.
|
||||
- Inductor compilation:
|
||||
- use_inductor: whether to use inductor compilation.
|
||||
- False: inductor compilation is not used. graph runs in eager.
|
||||
- True: inductor compilation is used. one graph for symbolic shape
|
||||
is compiled. In addition, compile for different sizes specified
|
||||
in inductor_compile_sizes, using configurations
|
||||
in inductor_compile_config.
|
||||
- inductor_compile_sizes: sizes to compile for inductor.
|
||||
- inductor_specialize_for_cudagraph_no_more_than: an optional integer
|
||||
to specialize inductor for cudagraph sizes no more than the
|
||||
specified size. It is useful when we want to specialize inductor
|
||||
with a subset of cudagraph sizes.
|
||||
- inductor_compile_config: additional configurations for inductor.
|
||||
- None: use default configurations.
|
||||
- inductor_passes: additional passes for inductor. It is a dictionary
|
||||
from pass name to pass function qualified name. We use function
|
||||
name because the config uses json format. If we pass the config
|
||||
from Python, functions can also be passed directly via Python object
|
||||
constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
|
||||
|
||||
Why we have different sizes for cudagraph and inductor:
|
||||
- cudagraph: a cudagraph captured for a specific size can only be used
|
||||
for the same size. We need to capture all the sizes we want to use.
|
||||
- inductor: a graph compiled by inductor for a general shape can be used
|
||||
for different sizes. Inductor can also compile for specific sizes,
|
||||
where it can have more information to optimize the graph with fully
|
||||
static shapes. However, we find the general shape compilation is
|
||||
sufficient for most cases. It might be beneficial to compile for
|
||||
certain small batchsizes, where inductor is good at optimizing.
|
||||
"""
|
||||
use_inductor: bool = True
|
||||
inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
|
||||
inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
|
||||
inductor_compile_config: Dict = Field(default_factory=dict)
|
||||
inductor_passes: Dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
use_cudagraph: bool = False
|
||||
non_cudagraph_ops: List[str] = Field(default_factory=list)
|
||||
cudagraph_num_of_warmups: int = 0
|
||||
cudagraph_capture_sizes: Optional[List[int]] = None
|
||||
|
||||
# not configurable, computed after init
|
||||
compile_sizes: List[int] = PrivateAttr
|
||||
capture_sizes: List[int] = PrivateAttr
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
for k, v in self.inductor_passes.items():
|
||||
if not isinstance(v, str):
|
||||
assert callable(v), (
|
||||
f"pass {k} should be a function or a qualified name")
|
||||
self.inductor_passes[k] = v
|
||||
continue
|
||||
|
||||
# resolve function from qualified name
|
||||
names = v.split(".")
|
||||
module = ".".join(names[:-1])
|
||||
func_name = names[-1]
|
||||
func = __import__(module).__dict__[func_name]
|
||||
self.inductor_compile_config[k] = func
|
||||
|
||||
from vllm.compilation.backends import fix_functionalization
|
||||
from vllm.utils import combine_fx_passes
|
||||
if "post_grad_custom_post_pass" in self.inductor_compile_config:
|
||||
self.inductor_compile_config[
|
||||
"post_grad_custom_post_pass"] = combine_fx_passes(
|
||||
fix_functionalization,
|
||||
self.inductor_compile_config["post_grad_custom_post_pass"],
|
||||
)
|
||||
else:
|
||||
self.inductor_compile_config[
|
||||
"post_grad_custom_post_pass"] = fix_functionalization
|
||||
|
||||
def init_during_runtime(self):
|
||||
"""To complete the initialization of config,
|
||||
we need to know the compile context, which is only available
|
||||
during the first run of the model.
|
||||
"""
|
||||
context = get_compile_context()
|
||||
context = copy.deepcopy(context) if context is not None else []
|
||||
sizes_to_specialize: List[int] = context
|
||||
if self.cudagraph_capture_sizes is None:
|
||||
self.capture_sizes = sizes_to_specialize
|
||||
else:
|
||||
self.capture_sizes = self.cudagraph_capture_sizes
|
||||
logger.info(("cudagraph sizes specified by model runner"
|
||||
" %s is overridden by config %s"),
|
||||
sizes_to_specialize, self.cudagraph_capture_sizes)
|
||||
if self.inductor_specialize_for_cudagraph_no_more_than is not None:
|
||||
assert self.inductor_compile_sizes is None, (
|
||||
"inductor_compile_sizes should be None when "
|
||||
"inductor_specialize_for_cudagraph_no_more_than is not None")
|
||||
self.compile_sizes = [
|
||||
x for x in self.capture_sizes
|
||||
if x <= self.inductor_specialize_for_cudagraph_no_more_than
|
||||
]
|
||||
else:
|
||||
assert self.inductor_compile_sizes is not None, (
|
||||
"inductor_compile_sizes should not be None when "
|
||||
"inductor_specialize_for_cudagraph_no_more_than is None")
|
||||
self.compile_sizes = self.inductor_compile_sizes
|
||||
|
||||
@staticmethod
|
||||
def select_and_init_config() -> "CompilationConfig":
|
||||
"""The order of selecting config is:
|
||||
1. Use the config specified in environment variable.
|
||||
2. Use the config specified in plugins.
|
||||
3. Use the default config.
|
||||
"""
|
||||
config_path = envs.VLLM_TORCH_COMPILE_CONFIG
|
||||
if config_path is not None:
|
||||
with open(config_path) as json_file:
|
||||
config = CompilationConfig.model_validate_json(
|
||||
json_file.read())
|
||||
else:
|
||||
from vllm.plugins import get_compilation_config
|
||||
predefined_config = get_compilation_config()
|
||||
config = predefined_config if predefined_config is not None else (
|
||||
CompilationConfig())
|
||||
|
||||
config.init_during_runtime()
|
||||
return config
|
||||
@@ -0,0 +1,30 @@
|
||||
import copy
|
||||
import dataclasses
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class CompilationCounter:
|
||||
num_graphs_seen: int = 0
|
||||
# including the splitting ops
|
||||
num_piecewise_graphs_seen: int = 0
|
||||
# not including the splitting ops
|
||||
num_piecewise_capturable_graphs_seen: int = 0
|
||||
num_inductor_compilations: int = 0
|
||||
num_cudagraph_caputured: int = 0
|
||||
|
||||
def clone(self) -> "CompilationCounter":
|
||||
return copy.deepcopy(self)
|
||||
|
||||
@contextmanager
|
||||
def expect(self, **kwargs):
|
||||
old = self.clone()
|
||||
yield
|
||||
for k, v in kwargs.items():
|
||||
assert getattr(self, k) - getattr(old, k) == v, (
|
||||
f"{k} not as expected, before it is {getattr(old, k)}"
|
||||
f", after it is {getattr(self, k)}, "
|
||||
f"expected diff is {v}")
|
||||
|
||||
|
||||
compilation_counter = CompilationCounter()
|
||||
@@ -121,7 +121,10 @@ def _support_torch_compile(cls: type,
|
||||
# take care of method resolution order
|
||||
# make sure super().__init__ is called on the base class
|
||||
# other than TorchCompileWrapperWithCustomDispatcher
|
||||
cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
|
||||
if TorchCompileWrapperWithCustomDispatcher not in cls.__bases__:
|
||||
# support decorating multiple times
|
||||
cls.__bases__ = cls.__bases__ + (
|
||||
TorchCompileWrapperWithCustomDispatcher, )
|
||||
|
||||
old_init = cls.__init__ # type: ignore
|
||||
|
||||
@@ -160,6 +163,11 @@ def _support_torch_compile(cls: type,
|
||||
# compiled function and let torch.compile handle the dispatching,
|
||||
# with the overhead of guard evaluation and recompilation.
|
||||
if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
|
||||
# it seems Dynamo reuse the compilation across instances,
|
||||
# while we need to make sure the compiled code is not reused.
|
||||
# we need to control all the compilation of the model.
|
||||
torch._dynamo.eval_frame.remove_from_cache(
|
||||
self.original_code_object)
|
||||
return self.compiled_callable(*args, **kwargs)
|
||||
|
||||
# usually, capturing the model once is enough, and then we can
|
||||
|
||||
@@ -5,5 +5,4 @@ class CompilationLevel:
|
||||
NO_COMPILATION = 0
|
||||
DYNAMO_AS_IS = 1
|
||||
DYNAMO_ONCE = 2
|
||||
INDUCTOR = 3
|
||||
INDUCTOR_MAX_AUTOTUNE = 4
|
||||
PIECEWISE = 3
|
||||
|
||||
Reference in New Issue
Block a user