[VLM][Model] TP support for ViTs (#7186)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
Roger Wang
Roger Wang
parent
afd39a4511
commit
f97be32d1d
@@ -71,6 +71,23 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
|
||||
projection_dim=768)
|
||||
|
||||
|
||||
def _init_img_processor(hf_config: PretrainedConfig):
|
||||
clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
|
||||
layer_idx = hf_config.img_processor.get('layer_idx', -2)
|
||||
|
||||
# Initialize the CLIP only up to the required feature layer
|
||||
if layer_idx < 0:
|
||||
num_hidden_layers = clip_config.num_hidden_layers + \
|
||||
layer_idx + 1
|
||||
else:
|
||||
num_hidden_layers = layer_idx + 1
|
||||
|
||||
img_processor = CLIPVisionModel(
|
||||
clip_config, num_hidden_layers_override=num_hidden_layers)
|
||||
|
||||
return img_processor
|
||||
|
||||
|
||||
class Phi3VImagePixelInputs(TypedDict):
|
||||
type: Literal["pixel_values"]
|
||||
data: Union[torch.Tensor, List[torch.Tensor]]
|
||||
@@ -139,18 +156,8 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
|
||||
hidden_size = config.n_embd if hasattr(
|
||||
config, 'n_embd') else config.hidden_size
|
||||
|
||||
clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
|
||||
self.layer_idx = config.img_processor.get('layer_idx', -2)
|
||||
self.img_processor = _init_img_processor(config)
|
||||
|
||||
# Initialize the CLIP only up to the required feature layer
|
||||
if self.layer_idx < 0:
|
||||
num_hidden_layers = clip_config.num_hidden_layers + \
|
||||
self.layer_idx + 1
|
||||
else:
|
||||
num_hidden_layers = self.layer_idx + 1
|
||||
|
||||
self.img_processor = CLIPVisionModel(
|
||||
clip_config, num_hidden_layers_override=num_hidden_layers)
|
||||
image_dim_out = config.img_processor['image_dim_out']
|
||||
self.num_img_tokens = config.img_processor['num_img_tokens']
|
||||
|
||||
@@ -656,23 +663,27 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
]
|
||||
|
||||
# TODO(ChristopherCho): This is a temporary fix to load
|
||||
# the vision weights with CLIPVisionModel.load_weights()
|
||||
vision_weights = []
|
||||
params_dict = dict(self.named_parameters())
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
# post_layernorm is not needed in CLIPVisionModel
|
||||
if "vision_model.post_layernorm" in name:
|
||||
# Skip loading the img_processor weights since they are
|
||||
# loaded separately.
|
||||
if "vision_embed_tokens.img_processor" in name:
|
||||
vision_weights.append((name, loaded_weight))
|
||||
continue
|
||||
|
||||
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
|
||||
if key_to_modify in name:
|
||||
name = name.replace(key_to_modify, new_key)
|
||||
for (param_name, weight_name, shard_id) in stacked_params_mapping:
|
||||
# We only do sharding for language model
|
||||
# and not vision model for now.
|
||||
if "vision_embed_tokens" in name and self.vision_embed_tokens:
|
||||
continue
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
param = params_dict[name.replace(weight_name, param_name)]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
@@ -686,3 +697,11 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
|
||||
weight_loader = getattr(param, "weight_loader",
|
||||
default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
# We use regex to extract the sub-module name
|
||||
# from "model.vision_embed_tokens.img_processor.*"
|
||||
vision_weights = [
|
||||
(re.search(r"vision_embed_tokens\.img_processor\.(.*)",
|
||||
n).group(1), w) for n, w in vision_weights
|
||||
]
|
||||
self.vision_embed_tokens.img_processor.load_weights(vision_weights)
|
||||
|
||||
Reference in New Issue
Block a user