[VLM] Update compatibility with transformers 4.49
This commit is contained in:
@@ -293,16 +293,29 @@ class PixtralHFMultiModalProcessor(
|
||||
|
||||
pixel_values = processed_outputs.get("pixel_values")
|
||||
if pixel_values is not None:
|
||||
images = mm_data["images"]
|
||||
assert isinstance(images, list)
|
||||
# Before/after https://github.com/huggingface/transformers/pull/35122
|
||||
if Version(TRANSFORMERS_VERSION) <= Version("4.48.2"):
|
||||
images = mm_data["images"]
|
||||
assert isinstance(images, list)
|
||||
|
||||
# Original output: (1, num_images, C, H, W)
|
||||
# New output: (num_images, C, H, W)
|
||||
assert (isinstance(pixel_values, list) and len(pixel_values) == 1)
|
||||
assert (isinstance(pixel_values[0], list)
|
||||
and len(pixel_values[0]) == len(images))
|
||||
# Original output: (1, num_images, C, H, W)
|
||||
# New output: (num_images, C, H, W)
|
||||
assert (isinstance(pixel_values, list)
|
||||
and len(pixel_values) == 1)
|
||||
assert (isinstance(pixel_values[0], list)
|
||||
and len(pixel_values[0]) == len(images))
|
||||
|
||||
processed_outputs["pixel_values"] = pixel_values[0]
|
||||
processed_outputs["pixel_values"] = pixel_values[0]
|
||||
else:
|
||||
# Avoid padding since we need the output for each image to be
|
||||
# independent of other images for the cache to work correctly
|
||||
image_sizes = processed_outputs["image_sizes"]
|
||||
assert len(pixel_values) == len(image_sizes)
|
||||
|
||||
processed_outputs["pixel_values"] = [
|
||||
p[:, :h, :w]
|
||||
for p, (h, w) in zip(pixel_values, image_sizes)
|
||||
]
|
||||
|
||||
return processed_outputs
|
||||
|
||||
|
||||
@@ -73,7 +73,15 @@ class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
|
||||
return self.ctx.get_hf_config(LlavaNextConfig)
|
||||
|
||||
def get_hf_processor(self):
|
||||
return self.ctx.get_hf_processor(LlavaNextProcessor)
|
||||
hf_processor = self.ctx.get_hf_processor(LlavaNextProcessor)
|
||||
|
||||
# In case patch_size is omitted from `processor_config.json`
|
||||
# e.g. for E5-V: https://huggingface.co/royokong/e5-v
|
||||
if hf_processor.patch_size is None:
|
||||
patch_size = self.get_vision_encoder_info().get_patch_size()
|
||||
hf_processor.patch_size = patch_size
|
||||
|
||||
return hf_processor
|
||||
|
||||
# Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
|
||||
def get_num_image_tokens(
|
||||
|
||||
@@ -342,6 +342,15 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
**kwargs: object,
|
||||
):
|
||||
hf_processor = self.ctx.get_hf_processor()
|
||||
|
||||
# NumPy arrays are considered as Iterable but not Sequence in
|
||||
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
|
||||
image_processor = hf_processor.image_processor # type: ignore
|
||||
for attr in ("mean", "std"):
|
||||
val = getattr(image_processor, attr)
|
||||
if isinstance(val, np.ndarray):
|
||||
setattr(image_processor, attr, val.tolist())
|
||||
|
||||
return hf_processor
|
||||
|
||||
def get_image_processor(self):
|
||||
|
||||
Reference in New Issue
Block a user