[Core][VLM] Support image embeddings as input (#6613)

This commit is contained in:
Roger Wang
2024-08-12 16:16:06 +08:00
committed by GitHub
parent ec2affa8ae
commit e6e42e4b17
13 changed files with 517 additions and 138 deletions
+4 -1
View File
@@ -115,6 +115,7 @@ class ImagePlugin(MultiModalPlugin):
data: object) -> MultiModalInputs:
model_config = ctx.model_config
# PIL image
if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
image_processor = self._get_hf_image_processor(model_config)
if image_processor is None:
@@ -129,8 +130,10 @@ class ImagePlugin(MultiModalPlugin):
raise
return MultiModalInputs(batch_data)
# Image embedding
elif isinstance(data, torch.Tensor) or is_list_of(data, torch.Tensor):
raise NotImplementedError("Embeddings input is not supported yet")
return MultiModalInputs({"image_embeds": data})
raise TypeError(f"Invalid image type: {type(data)}")