[Bugfix] Fix OpenPangu multimodal embedding merge (#55941)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -1039,17 +1039,20 @@ class OpenPanguVLForConditionalGeneration(
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings=None,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is not None:
|
||||
inputs_embeds = self.embed_input_ids(
|
||||
input_ids,
|
||||
inputs_embeds,
|
||||
multimodal_embeddings,
|
||||
[self.config.image_token_id, self.config.video_token_id],
|
||||
)
|
||||
return inputs_embeds
|
||||
if multimodal_embeddings is None:
|
||||
return self.language_model.embed_input_ids(input_ids)
|
||||
|
||||
mm_token_ids = input_ids.new_tensor(
|
||||
[self.config.image_token_id, self.config.video_token_id]
|
||||
)
|
||||
is_multimodal = torch.isin(input_ids, mm_token_ids)
|
||||
return self.embed_input_ids(
|
||||
input_ids,
|
||||
multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def _process_image_input(self, image_input) -> tuple[torch.Tensor, ...]:
|
||||
grid_thw = image_input["image_grid_thw"]
|
||||
|
||||
Reference in New Issue
Block a user