[Bugfix] Fixes Phi3v & Ultravox Multimodal EmbeddingInputs (#8979)

This commit is contained in:
hhzhang16
2024-10-04 22:05:37 -07:00
committed by GitHub
parent dac914b0d6
commit 53b3a33027
2 changed files with 41 additions and 23 deletions
+14 -6
View File
@@ -467,9 +467,10 @@ def input_processor_for_phi3v(ctx: InputContext,
input_height=h,
num_crops=num_crops))
elif isinstance(image_data, torch.Tensor):
num_images, image_feature_size, hidden_size = image_data.shape
image_feature_size = [image_data.shape[0]]
image_data = [image_data]
elif is_list_of(image_data, torch.Tensor):
image_feature_size = [item.shape[1] for item in image_data]
image_feature_size = [item.shape[0] for item in image_data]
else:
raise TypeError(f"Invalid image type: {type(image_data)}")
@@ -611,9 +612,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
image_sizes = kwargs.pop("image_sizes", None)
image_embeds = kwargs.pop("image_embeds", None)
if pixel_values is None:
return None
if pixel_values is None and image_embeds is None:
return None
@@ -650,7 +648,17 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
) -> torch.Tensor:
if image_input["type"] == "image_embeds":
return image_input["data"]
image_data = image_input["data"]
if is_list_of(image_data, torch.Tensor):
# it's already a list of tensors
return image_data
if len(image_data.shape) == 3:
# 3D tensor
return list(torch.unbind(image_data, dim=0))
raise ValueError(
"We expect batched 2D tensors;"
"this can be either a list of 2D tensors or a single 3D tensor."
)
assert self.vision_embed_tokens is not None
image_embeds = self.vision_embed_tokens(image_input["data"],