From 6f20f81cbf1d12dc9d499d25ea0a64ef4c816c00 Mon Sep 17 00:00:00 2001 From: Laith Sakka Date: Wed, 29 Apr 2026 11:32:15 -0700 Subject: [PATCH] Replace shape_invariants with simpler apprach in dynamic_arg_dims utilizing shape_id property. (#36194) Signed-off-by: Laith Sakka --- .../test_dynamic_shapes_compilation.py | 4 +- vllm/compilation/decorators.py | 106 ++++++++++++------ vllm/compilation/wrapper.py | 22 +--- vllm/model_executor/models/llama.py | 17 +-- vllm/model_executor/models/qwen2.py | 43 +------ 5 files changed, 88 insertions(+), 104 deletions(-) diff --git a/tests/compile/test_dynamic_shapes_compilation.py b/tests/compile/test_dynamic_shapes_compilation.py index 373d856d1c..e45e5cf425 100644 --- a/tests/compile/test_dynamic_shapes_compilation.py +++ b/tests/compile/test_dynamic_shapes_compilation.py @@ -55,12 +55,10 @@ def test_dynamic_shapes_compilation( evaluate_guards, ): """Test that all dynamic shapes types compile successfully""" - if use_bytecode_hook and shapes_type == DynamicShapesType.UNBACKED: - pytest.skip("UNBACKED dynamic shapes require VLLM_USE_BYTECODE_HOOK=0") - if evaluate_guards and shapes_type == DynamicShapesType.UNBACKED: pytest.skip("unbacked dynamic shapes do not add guards") + # TODO is this still a requirement? if evaluate_guards and use_aot_compile: pytest.skip("evaluate_guards requires use_aot_compile=0") diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 90b5c0c44e..ad710a6fe3 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -32,6 +32,9 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer from .monitor import monitor_profiling_run, monitor_torch_compile +# shape_id parameter was added to mark_unbacked in PyTorch 2.11.0 +_SUPPORTS_SHAPE_ID = is_torch_equal_or_newer("2.11.0") + if TYPE_CHECKING: # Only added on nightly/2.10 so wrap try: @@ -89,7 +92,7 @@ def support_torch_compile( @overload def support_torch_compile( *, - dynamic_arg_dims: dict[str, int | list[int]] | None, + dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None, ) -> Callable[[type[_T]], type[_T]]: ... @@ -103,7 +106,7 @@ def support_torch_compile( @overload def support_torch_compile( *, - dynamic_arg_dims: dict[str, int | list[int]] | None, + dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None, mark_unbacked_dims: dict[str, int | list[int]] | None, ) -> Callable[[type[_T]], type[_T]]: ... @@ -115,11 +118,10 @@ def support_torch_compile(cls: type[_T]) -> type[_T]: ... def support_torch_compile( cls: type[_T] | None = None, *, - dynamic_arg_dims: dict[str, int | list[int]] | None = None, + dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, - shape_invariants: Callable[..., None] = lambda *args, **kwargs: None, ) -> Callable[[type[_T]], type[_T]] | type[_T]: """ A decorator to add support for compiling the forward method of a class. @@ -141,8 +143,12 @@ def support_torch_compile( ``` `dynamic_arg_dims` is a dictionary that maps argument names to the dynamic - dimensions of the argument. The dynamic dimensions can be either a single - integer or a list of integers. + dimensions of the argument. The value can be: + - int: a single dimension index (e.g., 0) + - list[int]: multiple dimension indices (e.g., [0, 1]) + - dict[int, str]: dimension to shape_id mapping for shape relations + (e.g., {0: "b"}). Dimensions with the same shape_id share the same + unbacked symbol. if `dynamic_arg_dims` is `None`, it is inferred from the type annotation of the `forward` method, based on the following default rules: @@ -189,7 +195,7 @@ def support_torch_compile( torch._check(input_ids.size()[0] == inputs_embeds.size()[0]) This enforces constraints on the symbolic shapes without hardcoding specific values. It is needed for some models to avoid data dependent - errors. + errors and maximize perf when unbacked shapes are used. """ def cls_decorator_helper(cls: type[_T]) -> type[_T]: @@ -229,13 +235,13 @@ def support_torch_compile( raise ValueError( f"Argument {k} not found in the forward method of {cls}" ) + return _support_torch_compile( cls, inferred_dynamic_arg_dims, mark_unbacked_dims, enable_if, is_encoder, - shape_invariants, ) if cls is not None: @@ -324,15 +330,13 @@ def _try_load_aot_compiled_fn( def _support_torch_compile( cls: type[_T], - dynamic_arg_dims: dict[str, int | list[int]], + dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]], mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, - shape_invariants: Callable[..., None] = lambda *args, **kwargs: None, ) -> type[_T]: - """ - A decorator to add support for compiling the forward method of a class. - """ + """Internal implementation of support_torch_compile decorator.""" + if TorchCompileWithNoGuardsWrapper in cls.__bases__: # support decorating multiple times return cls @@ -392,7 +396,8 @@ def _support_torch_compile( if self.do_not_compile: return - self._check_shape_invariants = shape_invariants + self._dynamic_arg_dims = dynamic_arg_dims + self.was_aot_compile_fn_loaded_from_disk = False compilation_counter.num_models_seen += 1 self.compiled = False @@ -409,48 +414,83 @@ def _support_torch_compile( def _mark_dynamic_inputs( mod: type[_T], ds_type: DynamicShapesType, *args: Any, **kwargs: Any ) -> None: - def mark_dynamic(arg: torch.Tensor, dims: list[int]) -> None: + def mark_dynamic( + arg: torch.Tensor, dim_shape_pairs: list[tuple[int, str | None]] + ) -> None: if ds_type == DynamicShapesType.UNBACKED: if is_torch_equal_or_newer("2.10.0"): - for dim in dims: - torch._dynamo.decorators.mark_unbacked( - arg, dim, hint_override=arg.size()[dim] - ) + for dim, shape_id in dim_shape_pairs: + if shape_id is not None: + if not _SUPPORTS_SHAPE_ID: + raise RuntimeError( + f"shape_id='{shape_id}' requires PyTorch >= 2.11.0" + ) + torch._dynamo.decorators.mark_unbacked( + arg, + dim, + hint_override=arg.size()[dim], + shape_id=shape_id, + ) + else: + torch._dynamo.decorators.mark_unbacked( + arg, + dim, + hint_override=arg.size()[dim], + ) else: + # For older versions, we can't use hint_override or shape_id + dims = [dim for dim, _ in dim_shape_pairs] torch._dynamo.decorators.mark_unbacked(arg, dims) else: + dims = [dim for dim, _ in dim_shape_pairs] torch._dynamo.mark_dynamic(arg, dims) sig = inspect.signature(mod.__class__.forward) # type: ignore[attr-defined] bound_args = sig.bind(mod, *args, **kwargs) bound_args.apply_defaults() - for k, dims in dynamic_arg_dims.items(): + + # Normalize dynamic_arg_dims to dict[str, dict[int, str | None]] + normalized_dims: dict[str, dict[int, str | None]] = {} + for k, v in dynamic_arg_dims.items(): + if isinstance(v, dict): + normalized_dims[k] = {dim: shape_id for dim, shape_id in v.items()} + elif isinstance(v, int): + normalized_dims[k] = {v: None} + else: + normalized_dims[k] = {d: None for d in v} + + for k, dim_to_shape_id in normalized_dims.items(): arg = bound_args.arguments.get(k) if arg is not None: - dims = [dims] if isinstance(dims, int) else dims + dims = list(dim_to_shape_id.keys()) + if isinstance(arg, torch.Tensor): - # In case dims is specified with negative indexing - dims = [arg.ndim + dim if dim < 0 else dim for dim in dims] - mark_dynamic(arg, dims) + dim_shape_pairs = [ + (arg.ndim + d if d < 0 else d, dim_to_shape_id.get(d)) + for d in dims + ] + mark_dynamic(arg, dim_shape_pairs) elif isinstance(arg, IntermediateTensors): for tensor in arg.tensors.values(): - # In case dims is specified with negative indexing - dims = [tensor.ndim + dim if dim < 0 else dim for dim in dims] - mark_dynamic(tensor, dims) + dim_shape_pairs = [ + (tensor.ndim + d if d < 0 else d, dim_to_shape_id.get(d)) + for d in dims + ] + mark_dynamic(tensor, dim_shape_pairs) else: raise ValueError( - "Unsupported dynamic dimensions" - f" {dims} for argument {k} with type {type(arg)}." + f"Unsupported dynamic dimensions {dims} " + f"for argument {k} with type {type(arg)}." ) + if mark_unbacked_dims: - for k, dims in mark_unbacked_dims.items(): + for k, dims_val in mark_unbacked_dims.items(): arg = bound_args.arguments.get(k) if arg is not None: - dims = [dims] if isinstance(dims, int) else dims + dims = [dims_val] if isinstance(dims_val, int) else list(dims_val) if isinstance(arg, torch.Tensor): - # In case dims is specified with negative indexing - dims = [arg.ndim + dim if dim < 0 else dim for dim in dims] + dims = [arg.ndim + d if d < 0 else d for d in dims] if is_torch_equal_or_newer("2.10.0"): for dim in dims: torch._dynamo.decorators.mark_unbacked( diff --git a/vllm/compilation/wrapper.py b/vllm/compilation/wrapper.py index dc48528e9f..5635fe03ae 100644 --- a/vllm/compilation/wrapper.py +++ b/vllm/compilation/wrapper.py @@ -53,12 +53,6 @@ class TorchCompileWithNoGuardsWrapper: since we drop all guards. """ - def check_invariants_and_forward(self, *args: Any, **kwargs: Any) -> Any: - assert hasattr(self, "_check_shape_invariants") - self._check_shape_invariants(*args, **kwargs) - - return self.forward(*args, **kwargs) - def _call_with_optional_nvtx_range( self, callable_fn: Callable[P, R], *args: P.args, **kwargs: P.kwargs ) -> Any: @@ -115,6 +109,9 @@ class TorchCompileWithNoGuardsWrapper: "compilation_config.dynamic_shapes_config.evaluate_guards " "requires VLLM_USE_BYTECODE_HOOK=0. " ) + assert ds_type != DynamicShapesType.UNBACKED, ( + "UNBACKED dynamic shapes do not add guards" + ) options["guard_filter_fn"] = lambda x: [ entry.guard_type == "SHAPE_ENV" for entry in x @@ -130,19 +127,6 @@ class TorchCompileWithNoGuardsWrapper: compiled_ptr: Any = self.forward # Validate that unbacked dynamic shapes require VLLM_USE_BYTECODE_HOOK=False - if ds_type == DynamicShapesType.UNBACKED: - # reason is that bytecode does torch._dynamo.eval_frame. - # remove_from_cache(self.original_code_object()) to force a new - # re-compilation. And if we use - # compiled_ptr = self.check_invariants_and_forward - # it will reset all entries. - assert not envs.VLLM_USE_BYTECODE_HOOK, ( - "UNBACKED dynamic shapes requires VLLM_USE_BYTECODE_HOOK=0. " - ) - assert not self.evaluate_guards, "UNBACKED dynamic shapes do not add guards" - - compiled_ptr = self.check_invariants_and_forward - # Apply the constrain_to_fx_strides patch before first compilation. # This covers STOCK_TORCH_COMPILE and DYNAMO_ONCE paths. The VLLM # compile paths call this from their own compile() methods too. diff --git a/vllm/model_executor/models/llama.py b/vllm/model_executor/models/llama.py index 2ecced3df8..3c797d05e9 100644 --- a/vllm/model_executor/models/llama.py +++ b/vllm/model_executor/models/llama.py @@ -337,20 +337,15 @@ class LlamaDecoderLayer(nn.Module): return vllm_config.quant_config -def llama_model_invariants( - input_ids, positions, intermediate_tensors=None, inputs_embeds=None -): - """Shape invariants for Llama model compilation, those are translated to - runtime assertions for unbacked dynamic shapes and are compiled away for - backed""" - if input_ids is not None: - torch._check(positions.size()[0] == input_ids.size()[0]) - - @support_torch_compile( # TODO[#32068]: Investigate recompilation # mark_unbacked_dims={"input_ids": 0}, - shape_invariants=llama_model_invariants + dynamic_arg_dims={ + "input_ids": {0: "b"}, + "positions": {0: "b"}, + "intermediate_tensors": {0: "b"}, + "inputs_embeds": {0: "b"}, + }, ) class LlamaModel(nn.Module, EagleModelMixin): def __init__( diff --git a/vllm/model_executor/models/qwen2.py b/vllm/model_executor/models/qwen2.py index 27aa6175b9..b83fedc70d 100644 --- a/vllm/model_executor/models/qwen2.py +++ b/vllm/model_executor/models/qwen2.py @@ -312,48 +312,15 @@ class Qwen2DecoderLayer(nn.Module): return hidden_states, residual -def qwen_2_model_invariants( - input_ids: torch.Tensor, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, -): - """Shape invariants for Qwen2Model Model, those are translated to - runtime assertions for unbacked dynamic shapes and are compiled away for - backed""" - # All these should be equal. - # input_ids.size()[0] - # positions.size()[-1] - # intermediate_tensors["hidden_states"].size()[0] - # inputs_embeds.size()[0] - torch._check(input_ids.size()[0] == positions.size()[-1]) - if intermediate_tensors is not None: - torch._check( - input_ids.size()[0] == intermediate_tensors["hidden_states"].size()[0] - ) - - if inputs_embeds is not None: - torch._check(input_ids.size()[0] == inputs_embeds.size()[0]) - - # Hidden dimensions should match (hidden_size) - # intermediate_tensors["hidden_states"].size()[1] - # inputs_embeds.size()[1] - if inputs_embeds is not None and intermediate_tensors is not None: - torch._check( - inputs_embeds.size()[1] == intermediate_tensors["hidden_states"].size()[1] - ) - - @support_torch_compile( dynamic_arg_dims={ - "input_ids": 0, + "input_ids": {0: "b"}, # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl, # otherwise (seq_len, ). - "positions": -1, - "intermediate_tensors": 0, - "inputs_embeds": 0, - }, - shape_invariants=qwen_2_model_invariants, + "positions": {-1: "b"}, + "intermediate_tensors": {0: "b"}, + "inputs_embeds": {0: "b"}, + } ) class Qwen2Model(nn.Module, EagleModelMixin): def __init__(