Replace shape_invariants with simpler apprach in dynamic_arg_dims utilizing shape_id property. (#36194)

Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
Laith Sakka
2026-04-29 18:32:15 +00:00
committed by GitHub
parent d1a75e303d
commit 6f20f81cbf
5 changed files with 88 additions and 104 deletions
@@ -55,12 +55,10 @@ def test_dynamic_shapes_compilation(
evaluate_guards,
):
"""Test that all dynamic shapes types compile successfully"""
if use_bytecode_hook and shapes_type == DynamicShapesType.UNBACKED:
pytest.skip("UNBACKED dynamic shapes require VLLM_USE_BYTECODE_HOOK=0")
if evaluate_guards and shapes_type == DynamicShapesType.UNBACKED:
pytest.skip("unbacked dynamic shapes do not add guards")
# TODO is this still a requirement?
if evaluate_guards and use_aot_compile:
pytest.skip("evaluate_guards requires use_aot_compile=0")
+73 -33
View File
@@ -32,6 +32,9 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer
from .monitor import monitor_profiling_run, monitor_torch_compile
# shape_id parameter was added to mark_unbacked in PyTorch 2.11.0
_SUPPORTS_SHAPE_ID = is_torch_equal_or_newer("2.11.0")
if TYPE_CHECKING:
# Only added on nightly/2.10 so wrap
try:
@@ -89,7 +92,7 @@ def support_torch_compile(
@overload
def support_torch_compile(
*,
dynamic_arg_dims: dict[str, int | list[int]] | None,
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None,
) -> Callable[[type[_T]], type[_T]]: ...
@@ -103,7 +106,7 @@ def support_torch_compile(
@overload
def support_torch_compile(
*,
dynamic_arg_dims: dict[str, int | list[int]] | None,
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None,
mark_unbacked_dims: dict[str, int | list[int]] | None,
) -> Callable[[type[_T]], type[_T]]: ...
@@ -115,11 +118,10 @@ def support_torch_compile(cls: type[_T]) -> type[_T]: ...
def support_torch_compile(
cls: type[_T] | None = None,
*,
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None,
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
shape_invariants: Callable[..., None] = lambda *args, **kwargs: None,
) -> Callable[[type[_T]], type[_T]] | type[_T]:
"""
A decorator to add support for compiling the forward method of a class.
@@ -141,8 +143,12 @@ def support_torch_compile(
```
`dynamic_arg_dims` is a dictionary that maps argument names to the dynamic
dimensions of the argument. The dynamic dimensions can be either a single
integer or a list of integers.
dimensions of the argument. The value can be:
- int: a single dimension index (e.g., 0)
- list[int]: multiple dimension indices (e.g., [0, 1])
- dict[int, str]: dimension to shape_id mapping for shape relations
(e.g., {0: "b"}). Dimensions with the same shape_id share the same
unbacked symbol.
if `dynamic_arg_dims` is `None`, it is inferred from the type annotation
of the `forward` method, based on the following default rules:
@@ -189,7 +195,7 @@ def support_torch_compile(
torch._check(input_ids.size()[0] == inputs_embeds.size()[0])
This enforces constraints on the symbolic shapes without hardcoding
specific values. It is needed for some models to avoid data dependent
errors.
errors and maximize perf when unbacked shapes are used.
"""
def cls_decorator_helper(cls: type[_T]) -> type[_T]:
@@ -229,13 +235,13 @@ def support_torch_compile(
raise ValueError(
f"Argument {k} not found in the forward method of {cls}"
)
return _support_torch_compile(
cls,
inferred_dynamic_arg_dims,
mark_unbacked_dims,
enable_if,
is_encoder,
shape_invariants,
)
if cls is not None:
@@ -324,15 +330,13 @@ def _try_load_aot_compiled_fn(
def _support_torch_compile(
cls: type[_T],
dynamic_arg_dims: dict[str, int | list[int]],
dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]],
mark_unbacked_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
is_encoder: bool = False,
shape_invariants: Callable[..., None] = lambda *args, **kwargs: None,
) -> type[_T]:
"""
A decorator to add support for compiling the forward method of a class.
"""
"""Internal implementation of support_torch_compile decorator."""
if TorchCompileWithNoGuardsWrapper in cls.__bases__:
# support decorating multiple times
return cls
@@ -392,7 +396,8 @@ def _support_torch_compile(
if self.do_not_compile:
return
self._check_shape_invariants = shape_invariants
self._dynamic_arg_dims = dynamic_arg_dims
self.was_aot_compile_fn_loaded_from_disk = False
compilation_counter.num_models_seen += 1
self.compiled = False
@@ -409,48 +414,83 @@ def _support_torch_compile(
def _mark_dynamic_inputs(
mod: type[_T], ds_type: DynamicShapesType, *args: Any, **kwargs: Any
) -> None:
def mark_dynamic(arg: torch.Tensor, dims: list[int]) -> None:
def mark_dynamic(
arg: torch.Tensor, dim_shape_pairs: list[tuple[int, str | None]]
) -> None:
if ds_type == DynamicShapesType.UNBACKED:
if is_torch_equal_or_newer("2.10.0"):
for dim in dims:
torch._dynamo.decorators.mark_unbacked(
arg, dim, hint_override=arg.size()[dim]
)
for dim, shape_id in dim_shape_pairs:
if shape_id is not None:
if not _SUPPORTS_SHAPE_ID:
raise RuntimeError(
f"shape_id='{shape_id}' requires PyTorch >= 2.11.0"
)
torch._dynamo.decorators.mark_unbacked(
arg,
dim,
hint_override=arg.size()[dim],
shape_id=shape_id,
)
else:
torch._dynamo.decorators.mark_unbacked(
arg,
dim,
hint_override=arg.size()[dim],
)
else:
# For older versions, we can't use hint_override or shape_id
dims = [dim for dim, _ in dim_shape_pairs]
torch._dynamo.decorators.mark_unbacked(arg, dims)
else:
dims = [dim for dim, _ in dim_shape_pairs]
torch._dynamo.mark_dynamic(arg, dims)
sig = inspect.signature(mod.__class__.forward) # type: ignore[attr-defined]
bound_args = sig.bind(mod, *args, **kwargs)
bound_args.apply_defaults()
for k, dims in dynamic_arg_dims.items():
# Normalize dynamic_arg_dims to dict[str, dict[int, str | None]]
normalized_dims: dict[str, dict[int, str | None]] = {}
for k, v in dynamic_arg_dims.items():
if isinstance(v, dict):
normalized_dims[k] = {dim: shape_id for dim, shape_id in v.items()}
elif isinstance(v, int):
normalized_dims[k] = {v: None}
else:
normalized_dims[k] = {d: None for d in v}
for k, dim_to_shape_id in normalized_dims.items():
arg = bound_args.arguments.get(k)
if arg is not None:
dims = [dims] if isinstance(dims, int) else dims
dims = list(dim_to_shape_id.keys())
if isinstance(arg, torch.Tensor):
# In case dims is specified with negative indexing
dims = [arg.ndim + dim if dim < 0 else dim for dim in dims]
mark_dynamic(arg, dims)
dim_shape_pairs = [
(arg.ndim + d if d < 0 else d, dim_to_shape_id.get(d))
for d in dims
]
mark_dynamic(arg, dim_shape_pairs)
elif isinstance(arg, IntermediateTensors):
for tensor in arg.tensors.values():
# In case dims is specified with negative indexing
dims = [tensor.ndim + dim if dim < 0 else dim for dim in dims]
mark_dynamic(tensor, dims)
dim_shape_pairs = [
(tensor.ndim + d if d < 0 else d, dim_to_shape_id.get(d))
for d in dims
]
mark_dynamic(tensor, dim_shape_pairs)
else:
raise ValueError(
"Unsupported dynamic dimensions"
f" {dims} for argument {k} with type {type(arg)}."
f"Unsupported dynamic dimensions {dims} "
f"for argument {k} with type {type(arg)}."
)
if mark_unbacked_dims:
for k, dims in mark_unbacked_dims.items():
for k, dims_val in mark_unbacked_dims.items():
arg = bound_args.arguments.get(k)
if arg is not None:
dims = [dims] if isinstance(dims, int) else dims
dims = [dims_val] if isinstance(dims_val, int) else list(dims_val)
if isinstance(arg, torch.Tensor):
# In case dims is specified with negative indexing
dims = [arg.ndim + dim if dim < 0 else dim for dim in dims]
dims = [arg.ndim + d if d < 0 else d for d in dims]
if is_torch_equal_or_newer("2.10.0"):
for dim in dims:
torch._dynamo.decorators.mark_unbacked(
+3 -19
View File
@@ -53,12 +53,6 @@ class TorchCompileWithNoGuardsWrapper:
since we drop all guards.
"""
def check_invariants_and_forward(self, *args: Any, **kwargs: Any) -> Any:
assert hasattr(self, "_check_shape_invariants")
self._check_shape_invariants(*args, **kwargs)
return self.forward(*args, **kwargs)
def _call_with_optional_nvtx_range(
self, callable_fn: Callable[P, R], *args: P.args, **kwargs: P.kwargs
) -> Any:
@@ -115,6 +109,9 @@ class TorchCompileWithNoGuardsWrapper:
"compilation_config.dynamic_shapes_config.evaluate_guards "
"requires VLLM_USE_BYTECODE_HOOK=0. "
)
assert ds_type != DynamicShapesType.UNBACKED, (
"UNBACKED dynamic shapes do not add guards"
)
options["guard_filter_fn"] = lambda x: [
entry.guard_type == "SHAPE_ENV" for entry in x
@@ -130,19 +127,6 @@ class TorchCompileWithNoGuardsWrapper:
compiled_ptr: Any = self.forward
# Validate that unbacked dynamic shapes require VLLM_USE_BYTECODE_HOOK=False
if ds_type == DynamicShapesType.UNBACKED:
# reason is that bytecode does torch._dynamo.eval_frame.
# remove_from_cache(self.original_code_object()) to force a new
# re-compilation. And if we use
# compiled_ptr = self.check_invariants_and_forward
# it will reset all entries.
assert not envs.VLLM_USE_BYTECODE_HOOK, (
"UNBACKED dynamic shapes requires VLLM_USE_BYTECODE_HOOK=0. "
)
assert not self.evaluate_guards, "UNBACKED dynamic shapes do not add guards"
compiled_ptr = self.check_invariants_and_forward
# Apply the constrain_to_fx_strides patch before first compilation.
# This covers STOCK_TORCH_COMPILE and DYNAMO_ONCE paths. The VLLM
# compile paths call this from their own compile() methods too.
+6 -11
View File
@@ -337,20 +337,15 @@ class LlamaDecoderLayer(nn.Module):
return vllm_config.quant_config
def llama_model_invariants(
input_ids, positions, intermediate_tensors=None, inputs_embeds=None
):
"""Shape invariants for Llama model compilation, those are translated to
runtime assertions for unbacked dynamic shapes and are compiled away for
backed"""
if input_ids is not None:
torch._check(positions.size()[0] == input_ids.size()[0])
@support_torch_compile(
# TODO[#32068]: Investigate recompilation
# mark_unbacked_dims={"input_ids": 0},
shape_invariants=llama_model_invariants
dynamic_arg_dims={
"input_ids": {0: "b"},
"positions": {0: "b"},
"intermediate_tensors": {0: "b"},
"inputs_embeds": {0: "b"},
},
)
class LlamaModel(nn.Module, EagleModelMixin):
def __init__(
+5 -38
View File
@@ -312,48 +312,15 @@ class Qwen2DecoderLayer(nn.Module):
return hidden_states, residual
def qwen_2_model_invariants(
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
):
"""Shape invariants for Qwen2Model Model, those are translated to
runtime assertions for unbacked dynamic shapes and are compiled away for
backed"""
# All these should be equal.
# input_ids.size()[0]
# positions.size()[-1]
# intermediate_tensors["hidden_states"].size()[0]
# inputs_embeds.size()[0]
torch._check(input_ids.size()[0] == positions.size()[-1])
if intermediate_tensors is not None:
torch._check(
input_ids.size()[0] == intermediate_tensors["hidden_states"].size()[0]
)
if inputs_embeds is not None:
torch._check(input_ids.size()[0] == inputs_embeds.size()[0])
# Hidden dimensions should match (hidden_size)
# intermediate_tensors["hidden_states"].size()[1]
# inputs_embeds.size()[1]
if inputs_embeds is not None and intermediate_tensors is not None:
torch._check(
inputs_embeds.size()[1] == intermediate_tensors["hidden_states"].size()[1]
)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"input_ids": {0: "b"},
# positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
# otherwise (seq_len, ).
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
},
shape_invariants=qwen_2_model_invariants,
"positions": {-1: "b"},
"intermediate_tensors": {0: "b"},
"inputs_embeds": {0: "b"},
}
)
class Qwen2Model(nn.Module, EagleModelMixin):
def __init__(