[Model] Refactor BLIP/BLIP-2 to support composite model loading (#8407)
This commit is contained in:
@@ -391,6 +391,7 @@ class CLIPVisionModel(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
num_hidden_layers_override: Optional[int] = None):
|
||||
super().__init__()
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
num_heads = config.num_attention_heads
|
||||
self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
|
||||
@@ -400,10 +401,6 @@ class CLIPVisionModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers_override)
|
||||
|
||||
@property
|
||||
def _require_post_layernorm(self) -> bool:
|
||||
return self.vision_model.post_layernorm is not None
|
||||
|
||||
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
|
||||
return self.vision_model(pixel_values)
|
||||
|
||||
@@ -425,12 +422,12 @@ class CLIPVisionModel(nn.Module):
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# post_layernorm is not needed in CLIPVisionModel
|
||||
if ("vision_model.post_layernorm" in name
|
||||
and not self._require_post_layernorm):
|
||||
if (name.startswith("vision_model.post_layernorm")
|
||||
and self.vision_model.post_layernorm is None):
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if "vision_model.encoder.layers." in name:
|
||||
if name.startswith("vision_model.encoder.layers"):
|
||||
layer_idx = int(name.split(".")[3])
|
||||
if layer_idx >= layer_count:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user