Add evaluate_guards option to DynamicShapesConfig (#27432)
Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.compilation.partition_rules import (
|
||||
should_split,
|
||||
)
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig
|
||||
from vllm.config.compilation import DynamicShapesType
|
||||
from vllm.config.utils import Range, hash_factors
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logging_utils import lazy
|
||||
@@ -722,6 +723,29 @@ class VllmBackend:
|
||||
self.split_gm, submod_names_to_compile, self.vllm_config, self
|
||||
).run(*fake_args)
|
||||
|
||||
from torch._guards import detect_fake_mode
|
||||
|
||||
fake_mode = detect_fake_mode()
|
||||
|
||||
if (
|
||||
self.compilation_config.dynamic_shapes_config.evaluate_guards
|
||||
and self.compilation_config.dynamic_shapes_config.type
|
||||
== DynamicShapesType.BACKED
|
||||
):
|
||||
from torch.utils._sympy.value_ranges import ValueRanges
|
||||
|
||||
# Drop counter-0/1 specializations guards; for backed dynamic shapes,
|
||||
# torch.compile will specialize for 0/1 inputs or otherwise guards that
|
||||
# shape is >= 2. This is because it's really hard not to hit a check
|
||||
# against 0/1. When we evaluate shape guards, we exclude checking those
|
||||
# guards (We would fail always otherwise).
|
||||
|
||||
# We avoid that by updating the ranges of backed sizes when the min is
|
||||
# 2 for any, we assume it's 0.
|
||||
for s, r in fake_mode.shape_env.var_to_range.items():
|
||||
if r.lower == 2:
|
||||
fake_mode.shape_env.var_to_range[s] = ValueRanges(0, r.upper)
|
||||
|
||||
graph_path = os.path.join(local_cache_dir, "computation_graph.py")
|
||||
if not os.path.exists(graph_path):
|
||||
# code adapted from
|
||||
@@ -749,8 +773,6 @@ class VllmBackend:
|
||||
graph, example_inputs, self.prefix, self.split_gm
|
||||
)
|
||||
|
||||
# if we need to copy input buffers for cudagraph
|
||||
#
|
||||
# index of tensors that have symbolic shapes (batch size)
|
||||
# for weights and static buffers, they will have concrete shapes.
|
||||
# symbolic shape only happens for input tensors.
|
||||
|
||||
Reference in New Issue
Block a user