Replace shape_invariants with simpler apprach in dynamic_arg_dims utilizing shape_id property. (#36194)
Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
@@ -55,12 +55,10 @@ def test_dynamic_shapes_compilation(
|
||||
evaluate_guards,
|
||||
):
|
||||
"""Test that all dynamic shapes types compile successfully"""
|
||||
if use_bytecode_hook and shapes_type == DynamicShapesType.UNBACKED:
|
||||
pytest.skip("UNBACKED dynamic shapes require VLLM_USE_BYTECODE_HOOK=0")
|
||||
|
||||
if evaluate_guards and shapes_type == DynamicShapesType.UNBACKED:
|
||||
pytest.skip("unbacked dynamic shapes do not add guards")
|
||||
|
||||
# TODO is this still a requirement?
|
||||
if evaluate_guards and use_aot_compile:
|
||||
pytest.skip("evaluate_guards requires use_aot_compile=0")
|
||||
|
||||
|
||||
@@ -32,6 +32,9 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
from .monitor import monitor_profiling_run, monitor_torch_compile
|
||||
|
||||
# shape_id parameter was added to mark_unbacked in PyTorch 2.11.0
|
||||
_SUPPORTS_SHAPE_ID = is_torch_equal_or_newer("2.11.0")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Only added on nightly/2.10 so wrap
|
||||
try:
|
||||
@@ -89,7 +92,7 @@ def support_torch_compile(
|
||||
@overload
|
||||
def support_torch_compile(
|
||||
*,
|
||||
dynamic_arg_dims: dict[str, int | list[int]] | None,
|
||||
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None,
|
||||
) -> Callable[[type[_T]], type[_T]]: ...
|
||||
|
||||
|
||||
@@ -103,7 +106,7 @@ def support_torch_compile(
|
||||
@overload
|
||||
def support_torch_compile(
|
||||
*,
|
||||
dynamic_arg_dims: dict[str, int | list[int]] | None,
|
||||
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None,
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None,
|
||||
) -> Callable[[type[_T]], type[_T]]: ...
|
||||
|
||||
@@ -115,11 +118,10 @@ def support_torch_compile(cls: type[_T]) -> type[_T]: ...
|
||||
def support_torch_compile(
|
||||
cls: type[_T] | None = None,
|
||||
*,
|
||||
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
|
||||
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None,
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
shape_invariants: Callable[..., None] = lambda *args, **kwargs: None,
|
||||
) -> Callable[[type[_T]], type[_T]] | type[_T]:
|
||||
"""
|
||||
A decorator to add support for compiling the forward method of a class.
|
||||
@@ -141,8 +143,12 @@ def support_torch_compile(
|
||||
```
|
||||
|
||||
`dynamic_arg_dims` is a dictionary that maps argument names to the dynamic
|
||||
dimensions of the argument. The dynamic dimensions can be either a single
|
||||
integer or a list of integers.
|
||||
dimensions of the argument. The value can be:
|
||||
- int: a single dimension index (e.g., 0)
|
||||
- list[int]: multiple dimension indices (e.g., [0, 1])
|
||||
- dict[int, str]: dimension to shape_id mapping for shape relations
|
||||
(e.g., {0: "b"}). Dimensions with the same shape_id share the same
|
||||
unbacked symbol.
|
||||
|
||||
if `dynamic_arg_dims` is `None`, it is inferred from the type annotation
|
||||
of the `forward` method, based on the following default rules:
|
||||
@@ -189,7 +195,7 @@ def support_torch_compile(
|
||||
torch._check(input_ids.size()[0] == inputs_embeds.size()[0])
|
||||
This enforces constraints on the symbolic shapes without hardcoding
|
||||
specific values. It is needed for some models to avoid data dependent
|
||||
errors.
|
||||
errors and maximize perf when unbacked shapes are used.
|
||||
"""
|
||||
|
||||
def cls_decorator_helper(cls: type[_T]) -> type[_T]:
|
||||
@@ -229,13 +235,13 @@ def support_torch_compile(
|
||||
raise ValueError(
|
||||
f"Argument {k} not found in the forward method of {cls}"
|
||||
)
|
||||
|
||||
return _support_torch_compile(
|
||||
cls,
|
||||
inferred_dynamic_arg_dims,
|
||||
mark_unbacked_dims,
|
||||
enable_if,
|
||||
is_encoder,
|
||||
shape_invariants,
|
||||
)
|
||||
|
||||
if cls is not None:
|
||||
@@ -324,15 +330,13 @@ def _try_load_aot_compiled_fn(
|
||||
|
||||
def _support_torch_compile(
|
||||
cls: type[_T],
|
||||
dynamic_arg_dims: dict[str, int | list[int]],
|
||||
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]],
|
||||
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
|
||||
enable_if: Callable[[VllmConfig], bool] | None = None,
|
||||
is_encoder: bool = False,
|
||||
shape_invariants: Callable[..., None] = lambda *args, **kwargs: None,
|
||||
) -> type[_T]:
|
||||
"""
|
||||
A decorator to add support for compiling the forward method of a class.
|
||||
"""
|
||||
"""Internal implementation of support_torch_compile decorator."""
|
||||
|
||||
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
|
||||
# support decorating multiple times
|
||||
return cls
|
||||
@@ -392,7 +396,8 @@ def _support_torch_compile(
|
||||
if self.do_not_compile:
|
||||
return
|
||||
|
||||
self._check_shape_invariants = shape_invariants
|
||||
self._dynamic_arg_dims = dynamic_arg_dims
|
||||
|
||||
self.was_aot_compile_fn_loaded_from_disk = False
|
||||
compilation_counter.num_models_seen += 1
|
||||
self.compiled = False
|
||||
@@ -409,48 +414,83 @@ def _support_torch_compile(
|
||||
def _mark_dynamic_inputs(
|
||||
mod: type[_T], ds_type: DynamicShapesType, *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
def mark_dynamic(arg: torch.Tensor, dims: list[int]) -> None:
|
||||
def mark_dynamic(
|
||||
arg: torch.Tensor, dim_shape_pairs: list[tuple[int, str | None]]
|
||||
) -> None:
|
||||
if ds_type == DynamicShapesType.UNBACKED:
|
||||
if is_torch_equal_or_newer("2.10.0"):
|
||||
for dim in dims:
|
||||
torch._dynamo.decorators.mark_unbacked(
|
||||
arg, dim, hint_override=arg.size()[dim]
|
||||
)
|
||||
for dim, shape_id in dim_shape_pairs:
|
||||
if shape_id is not None:
|
||||
if not _SUPPORTS_SHAPE_ID:
|
||||
raise RuntimeError(
|
||||
f"shape_id='{shape_id}' requires PyTorch >= 2.11.0"
|
||||
)
|
||||
torch._dynamo.decorators.mark_unbacked(
|
||||
arg,
|
||||
dim,
|
||||
hint_override=arg.size()[dim],
|
||||
shape_id=shape_id,
|
||||
)
|
||||
else:
|
||||
torch._dynamo.decorators.mark_unbacked(
|
||||
arg,
|
||||
dim,
|
||||
hint_override=arg.size()[dim],
|
||||
)
|
||||
else:
|
||||
# For older versions, we can't use hint_override or shape_id
|
||||
dims = [dim for dim, _ in dim_shape_pairs]
|
||||
torch._dynamo.decorators.mark_unbacked(arg, dims)
|
||||
else:
|
||||
dims = [dim for dim, _ in dim_shape_pairs]
|
||||
torch._dynamo.mark_dynamic(arg, dims)
|
||||
|
||||
sig = inspect.signature(mod.__class__.forward) # type: ignore[attr-defined]
|
||||
bound_args = sig.bind(mod, *args, **kwargs)
|
||||
bound_args.apply_defaults()
|
||||
for k, dims in dynamic_arg_dims.items():
|
||||
|
||||
# Normalize dynamic_arg_dims to dict[str, dict[int, str | None]]
|
||||
normalized_dims: dict[str, dict[int, str | None]] = {}
|
||||
for k, v in dynamic_arg_dims.items():
|
||||
if isinstance(v, dict):
|
||||
normalized_dims[k] = {dim: shape_id for dim, shape_id in v.items()}
|
||||
elif isinstance(v, int):
|
||||
normalized_dims[k] = {v: None}
|
||||
else:
|
||||
normalized_dims[k] = {d: None for d in v}
|
||||
|
||||
for k, dim_to_shape_id in normalized_dims.items():
|
||||
arg = bound_args.arguments.get(k)
|
||||
|
||||
if arg is not None:
|
||||
dims = [dims] if isinstance(dims, int) else dims
|
||||
dims = list(dim_to_shape_id.keys())
|
||||
|
||||
if isinstance(arg, torch.Tensor):
|
||||
# In case dims is specified with negative indexing
|
||||
dims = [arg.ndim + dim if dim < 0 else dim for dim in dims]
|
||||
mark_dynamic(arg, dims)
|
||||
dim_shape_pairs = [
|
||||
(arg.ndim + d if d < 0 else d, dim_to_shape_id.get(d))
|
||||
for d in dims
|
||||
]
|
||||
mark_dynamic(arg, dim_shape_pairs)
|
||||
elif isinstance(arg, IntermediateTensors):
|
||||
for tensor in arg.tensors.values():
|
||||
# In case dims is specified with negative indexing
|
||||
dims = [tensor.ndim + dim if dim < 0 else dim for dim in dims]
|
||||
mark_dynamic(tensor, dims)
|
||||
dim_shape_pairs = [
|
||||
(tensor.ndim + d if d < 0 else d, dim_to_shape_id.get(d))
|
||||
for d in dims
|
||||
]
|
||||
mark_dynamic(tensor, dim_shape_pairs)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Unsupported dynamic dimensions"
|
||||
f" {dims} for argument {k} with type {type(arg)}."
|
||||
f"Unsupported dynamic dimensions {dims} "
|
||||
f"for argument {k} with type {type(arg)}."
|
||||
)
|
||||
|
||||
if mark_unbacked_dims:
|
||||
for k, dims in mark_unbacked_dims.items():
|
||||
for k, dims_val in mark_unbacked_dims.items():
|
||||
arg = bound_args.arguments.get(k)
|
||||
if arg is not None:
|
||||
dims = [dims] if isinstance(dims, int) else dims
|
||||
dims = [dims_val] if isinstance(dims_val, int) else list(dims_val)
|
||||
if isinstance(arg, torch.Tensor):
|
||||
# In case dims is specified with negative indexing
|
||||
dims = [arg.ndim + dim if dim < 0 else dim for dim in dims]
|
||||
dims = [arg.ndim + d if d < 0 else d for d in dims]
|
||||
if is_torch_equal_or_newer("2.10.0"):
|
||||
for dim in dims:
|
||||
torch._dynamo.decorators.mark_unbacked(
|
||||
|
||||
@@ -53,12 +53,6 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
since we drop all guards.
|
||||
"""
|
||||
|
||||
def check_invariants_and_forward(self, *args: Any, **kwargs: Any) -> Any:
|
||||
assert hasattr(self, "_check_shape_invariants")
|
||||
self._check_shape_invariants(*args, **kwargs)
|
||||
|
||||
return self.forward(*args, **kwargs)
|
||||
|
||||
def _call_with_optional_nvtx_range(
|
||||
self, callable_fn: Callable[P, R], *args: P.args, **kwargs: P.kwargs
|
||||
) -> Any:
|
||||
@@ -115,6 +109,9 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
"compilation_config.dynamic_shapes_config.evaluate_guards "
|
||||
"requires VLLM_USE_BYTECODE_HOOK=0. "
|
||||
)
|
||||
assert ds_type != DynamicShapesType.UNBACKED, (
|
||||
"UNBACKED dynamic shapes do not add guards"
|
||||
)
|
||||
|
||||
options["guard_filter_fn"] = lambda x: [
|
||||
entry.guard_type == "SHAPE_ENV" for entry in x
|
||||
@@ -130,19 +127,6 @@ class TorchCompileWithNoGuardsWrapper:
|
||||
compiled_ptr: Any = self.forward
|
||||
# Validate that unbacked dynamic shapes require VLLM_USE_BYTECODE_HOOK=False
|
||||
|
||||
if ds_type == DynamicShapesType.UNBACKED:
|
||||
# reason is that bytecode does torch._dynamo.eval_frame.
|
||||
# remove_from_cache(self.original_code_object()) to force a new
|
||||
# re-compilation. And if we use
|
||||
# compiled_ptr = self.check_invariants_and_forward
|
||||
# it will reset all entries.
|
||||
assert not envs.VLLM_USE_BYTECODE_HOOK, (
|
||||
"UNBACKED dynamic shapes requires VLLM_USE_BYTECODE_HOOK=0. "
|
||||
)
|
||||
assert not self.evaluate_guards, "UNBACKED dynamic shapes do not add guards"
|
||||
|
||||
compiled_ptr = self.check_invariants_and_forward
|
||||
|
||||
# Apply the constrain_to_fx_strides patch before first compilation.
|
||||
# This covers STOCK_TORCH_COMPILE and DYNAMO_ONCE paths. The VLLM
|
||||
# compile paths call this from their own compile() methods too.
|
||||
|
||||
@@ -337,20 +337,15 @@ class LlamaDecoderLayer(nn.Module):
|
||||
return vllm_config.quant_config
|
||||
|
||||
|
||||
def llama_model_invariants(
|
||||
input_ids, positions, intermediate_tensors=None, inputs_embeds=None
|
||||
):
|
||||
"""Shape invariants for Llama model compilation, those are translated to
|
||||
runtime assertions for unbacked dynamic shapes and are compiled away for
|
||||
backed"""
|
||||
if input_ids is not None:
|
||||
torch._check(positions.size()[0] == input_ids.size()[0])
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
# TODO[#32068]: Investigate recompilation
|
||||
# mark_unbacked_dims={"input_ids": 0},
|
||||
shape_invariants=llama_model_invariants
|
||||
dynamic_arg_dims={
|
||||
"input_ids": {0: "b"},
|
||||
"positions": {0: "b"},
|
||||
"intermediate_tensors": {0: "b"},
|
||||
"inputs_embeds": {0: "b"},
|
||||
},
|
||||
)
|
||||
class LlamaModel(nn.Module, EagleModelMixin):
|
||||
def __init__(
|
||||
|
||||
@@ -312,48 +312,15 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def qwen_2_model_invariants(
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
):
|
||||
"""Shape invariants for Qwen2Model Model, those are translated to
|
||||
runtime assertions for unbacked dynamic shapes and are compiled away for
|
||||
backed"""
|
||||
# All these should be equal.
|
||||
# input_ids.size()[0]
|
||||
# positions.size()[-1]
|
||||
# intermediate_tensors["hidden_states"].size()[0]
|
||||
# inputs_embeds.size()[0]
|
||||
torch._check(input_ids.size()[0] == positions.size()[-1])
|
||||
if intermediate_tensors is not None:
|
||||
torch._check(
|
||||
input_ids.size()[0] == intermediate_tensors["hidden_states"].size()[0]
|
||||
)
|
||||
|
||||
if inputs_embeds is not None:
|
||||
torch._check(input_ids.size()[0] == inputs_embeds.size()[0])
|
||||
|
||||
# Hidden dimensions should match (hidden_size)
|
||||
# intermediate_tensors["hidden_states"].size()[1]
|
||||
# inputs_embeds.size()[1]
|
||||
if inputs_embeds is not None and intermediate_tensors is not None:
|
||||
torch._check(
|
||||
inputs_embeds.size()[1] == intermediate_tensors["hidden_states"].size()[1]
|
||||
)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"input_ids": {0: "b"},
|
||||
# positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
|
||||
# otherwise (seq_len, ).
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
},
|
||||
shape_invariants=qwen_2_model_invariants,
|
||||
"positions": {-1: "b"},
|
||||
"intermediate_tensors": {0: "b"},
|
||||
"inputs_embeds": {0: "b"},
|
||||
}
|
||||
)
|
||||
class Qwen2Model(nn.Module, EagleModelMixin):
|
||||
def __init__(
|
||||
|
||||
Reference in New Issue
Block a user