[torch.compile][BE][Multimodal] Remove requirement to set_model_tag to avoid cache conflict (#37345)
Signed-off-by: Lucas Kabela <lucaskabela@meta.com>
This commit is contained in:
@@ -272,6 +272,7 @@ class Siglip2MLP(nn.Module):
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={"hidden_states": [0, 1], "cu_seqlens": 0},
|
||||
enable_if=should_torch_compile_mm_encoder,
|
||||
is_encoder=True,
|
||||
)
|
||||
class Siglip2EncoderLayer(nn.Module):
|
||||
def __init__(
|
||||
@@ -395,16 +396,12 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
embed_dim = config.hidden_size
|
||||
self.config = config
|
||||
self.embeddings = Siglip2VisionEmbeddings(config)
|
||||
# Keep the import local to avoid circular dependencies during model init.
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
with set_model_tag("Siglip2Encoder", is_encoder=True):
|
||||
self.encoder = Siglip2Encoder(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers_override,
|
||||
prefix=f"{prefix}.encoder",
|
||||
)
|
||||
self.encoder = Siglip2Encoder(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers_override,
|
||||
prefix=f"{prefix}.encoder",
|
||||
)
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
if len(self.encoder.layers) > config.num_hidden_layers:
|
||||
raise ValueError(
|
||||
|
||||
@@ -453,7 +453,9 @@ class Llama4UnfoldConvolution(nn.Module):
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={"images_flattened": 0}, enable_if=should_torch_compile_mm_encoder
|
||||
dynamic_arg_dims={"images_flattened": 0},
|
||||
enable_if=should_torch_compile_mm_encoder,
|
||||
is_encoder=True,
|
||||
)
|
||||
class Llama4VisionModel(nn.Module):
|
||||
def __init__(
|
||||
@@ -754,12 +756,7 @@ class Llama4ForConditionalGeneration(
|
||||
self.multimodal_config = multimodal_config
|
||||
|
||||
with self._mark_tower_model(vllm_config, "image"):
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_model_tag("Llama4VisionModel", is_encoder=True),
|
||||
):
|
||||
with set_current_vllm_config(vllm_config):
|
||||
self.vision_model = Llama4VisionModel(
|
||||
config=config.vision_config,
|
||||
quant_config=None,
|
||||
|
||||
@@ -427,6 +427,7 @@ class Qwen2_5_VisionAttention(nn.Module):
|
||||
"rotary_pos_emb_sin": 0,
|
||||
},
|
||||
enable_if=should_torch_compile_mm_encoder,
|
||||
is_encoder=True,
|
||||
)
|
||||
class Qwen2_5_VisionBlock(nn.Module):
|
||||
def __init__(
|
||||
@@ -486,6 +487,7 @@ class Qwen2_5_VisionBlock(nn.Module):
|
||||
"x": 0,
|
||||
},
|
||||
enable_if=should_torch_compile_mm_encoder,
|
||||
is_encoder=True,
|
||||
)
|
||||
class Qwen2_5_VisionPatchEmbed(nn.Module):
|
||||
def __init__(
|
||||
@@ -521,6 +523,7 @@ class Qwen2_5_VisionPatchEmbed(nn.Module):
|
||||
"x": 0,
|
||||
},
|
||||
enable_if=should_torch_compile_mm_encoder,
|
||||
is_encoder=True,
|
||||
)
|
||||
class Qwen2_5_VisionPatchMerger(nn.Module):
|
||||
def __init__(
|
||||
@@ -592,18 +595,12 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
self.spatial_merge_size = vision_config.spatial_merge_size
|
||||
self.fullatt_block_indexes = vision_config.fullatt_block_indexes
|
||||
self.spatial_merge_unit = self.spatial_merge_size**2
|
||||
# TODO[@lucaskabela]: Investigate fixing this usage
|
||||
# see https://github.com/vllm-project/vllm/issues/27044
|
||||
# DO NOT MOVE THIS IMPORT
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
with set_model_tag("Qwen2_5_VisionPatchEmbed", is_encoder=True):
|
||||
self.patch_embed = Qwen2_5_VisionPatchEmbed(
|
||||
patch_size=patch_size,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
in_channels=in_channels,
|
||||
hidden_size=self.hidden_size,
|
||||
)
|
||||
self.patch_embed = Qwen2_5_VisionPatchEmbed(
|
||||
patch_size=patch_size,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
in_channels=in_channels,
|
||||
hidden_size=self.hidden_size,
|
||||
)
|
||||
|
||||
norm_layer = partial(RMSNorm, eps=norm_eps)
|
||||
head_dim = self.hidden_size // self.num_heads
|
||||
@@ -619,31 +616,29 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
dtype=torch.get_default_dtype(),
|
||||
)
|
||||
|
||||
with set_model_tag("Qwen2_5_VisionBlock", is_encoder=True):
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
Qwen2_5_VisionBlock(
|
||||
dim=self.hidden_size,
|
||||
num_heads=self.num_heads,
|
||||
mlp_hidden_dim=vision_config.intermediate_size,
|
||||
act_fn=get_act_and_mul_fn(vision_config.hidden_act),
|
||||
norm_layer=norm_layer,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.blocks.{layer_idx}",
|
||||
)
|
||||
for layer_idx in range(depth)
|
||||
]
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
Qwen2_5_VisionBlock(
|
||||
dim=self.hidden_size,
|
||||
num_heads=self.num_heads,
|
||||
mlp_hidden_dim=vision_config.intermediate_size,
|
||||
act_fn=get_act_and_mul_fn(vision_config.hidden_act),
|
||||
norm_layer=norm_layer,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.blocks.{layer_idx}",
|
||||
)
|
||||
for layer_idx in range(depth)
|
||||
]
|
||||
)
|
||||
|
||||
with set_model_tag("Qwen2_5_VisionPatchMerger", is_encoder=True):
|
||||
self.merger = Qwen2_5_VisionPatchMerger(
|
||||
d_model=vision_config.out_hidden_size,
|
||||
context_dim=self.hidden_size,
|
||||
norm_layer=norm_layer,
|
||||
spatial_merge_size=self.spatial_merge_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.merger",
|
||||
)
|
||||
self.merger = Qwen2_5_VisionPatchMerger(
|
||||
d_model=vision_config.out_hidden_size,
|
||||
context_dim=self.hidden_size,
|
||||
norm_layer=norm_layer,
|
||||
spatial_merge_size=self.spatial_merge_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.merger",
|
||||
)
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
|
||||
Reference in New Issue
Block a user