[Bugfix]Disable the post_norm layer of the vision encoder for LLaVA models (#9653)

This commit is contained in:
litianjian
2024-10-24 07:52:07 -07:00
committed by GitHub
parent b979143d5b
commit f58454968f
4 changed files with 8 additions and 4 deletions
+2 -1
View File
@@ -277,7 +277,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
self.multimodal_config = multimodal_config
# TODO: Optionally initializes this for supporting embeddings.
self.vision_tower = init_vision_tower_for_llava(config, quant_config)
self.vision_tower = init_vision_tower_for_llava(
config, quant_config, require_post_norm=False)
self.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size))
self.multi_modal_projector = LlavaMultiModalProjector(