Replace shape_invariants with simpler apprach in dynamic_arg_dims utilizing shape_id property. (#36194)

Signed-off-by: Laith Sakka <lsakka@meta.com>
This commit is contained in:
Laith Sakka
2026-04-29 18:32:15 +00:00
committed by GitHub
parent d1a75e303d
commit 6f20f81cbf
5 changed files with 88 additions and 104 deletions
+6 -11
View File
@@ -337,20 +337,15 @@ class LlamaDecoderLayer(nn.Module):
return vllm_config.quant_config
def llama_model_invariants(
input_ids, positions, intermediate_tensors=None, inputs_embeds=None
):
"""Shape invariants for Llama model compilation, those are translated to
runtime assertions for unbacked dynamic shapes and are compiled away for
backed"""
if input_ids is not None:
torch._check(positions.size()[0] == input_ids.size()[0])
@support_torch_compile(
# TODO[#32068]: Investigate recompilation
# mark_unbacked_dims={"input_ids": 0},
shape_invariants=llama_model_invariants
dynamic_arg_dims={
"input_ids": {0: "b"},
"positions": {0: "b"},
"intermediate_tensors": {0: "b"},
"inputs_embeds": {0: "b"},
},
)
class LlamaModel(nn.Module, EagleModelMixin):
def __init__(
+5 -38
View File
@@ -312,48 +312,15 @@ class Qwen2DecoderLayer(nn.Module):
return hidden_states, residual
def qwen_2_model_invariants(
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
):
"""Shape invariants for Qwen2Model Model, those are translated to
runtime assertions for unbacked dynamic shapes and are compiled away for
backed"""
# All these should be equal.
# input_ids.size()[0]
# positions.size()[-1]
# intermediate_tensors["hidden_states"].size()[0]
# inputs_embeds.size()[0]
torch._check(input_ids.size()[0] == positions.size()[-1])
if intermediate_tensors is not None:
torch._check(
input_ids.size()[0] == intermediate_tensors["hidden_states"].size()[0]
)
if inputs_embeds is not None:
torch._check(input_ids.size()[0] == inputs_embeds.size()[0])
# Hidden dimensions should match (hidden_size)
# intermediate_tensors["hidden_states"].size()[1]
# inputs_embeds.size()[1]
if inputs_embeds is not None and intermediate_tensors is not None:
torch._check(
inputs_embeds.size()[1] == intermediate_tensors["hidden_states"].size()[1]
)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"input_ids": {0: "b"},
# positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
# otherwise (seq_len, ).
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
},
shape_invariants=qwen_2_model_invariants,
"positions": {-1: "b"},
"intermediate_tensors": {0: "b"},
"inputs_embeds": {0: "b"},
}
)
class Qwen2Model(nn.Module, EagleModelMixin):
def __init__(