[VLM] Update compatibility with transformers 4.49

This commit is contained in:
Cyrus Leung
2025-02-05 19:09:45 -08:00
committed by GitHub
parent bf3b79efb8
commit 75404d041b
9 changed files with 48 additions and 57 deletions
+21 -8
View File
@@ -293,16 +293,29 @@ class PixtralHFMultiModalProcessor(
pixel_values = processed_outputs.get("pixel_values")
if pixel_values is not None:
images = mm_data["images"]
assert isinstance(images, list)
# Before/after https://github.com/huggingface/transformers/pull/35122
if Version(TRANSFORMERS_VERSION) <= Version("4.48.2"):
images = mm_data["images"]
assert isinstance(images, list)
# Original output: (1, num_images, C, H, W)
# New output: (num_images, C, H, W)
assert (isinstance(pixel_values, list) and len(pixel_values) == 1)
assert (isinstance(pixel_values[0], list)
and len(pixel_values[0]) == len(images))
# Original output: (1, num_images, C, H, W)
# New output: (num_images, C, H, W)
assert (isinstance(pixel_values, list)
and len(pixel_values) == 1)
assert (isinstance(pixel_values[0], list)
and len(pixel_values[0]) == len(images))
processed_outputs["pixel_values"] = pixel_values[0]
processed_outputs["pixel_values"] = pixel_values[0]
else:
# Avoid padding since we need the output for each image to be
# independent of other images for the cache to work correctly
image_sizes = processed_outputs["image_sizes"]
assert len(pixel_values) == len(image_sizes)
processed_outputs["pixel_values"] = [
p[:, :h, :w]
for p, (h, w) in zip(pixel_values, image_sizes)
]
return processed_outputs
+9 -1
View File
@@ -73,7 +73,15 @@ class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
return self.ctx.get_hf_config(LlavaNextConfig)
def get_hf_processor(self):
return self.ctx.get_hf_processor(LlavaNextProcessor)
hf_processor = self.ctx.get_hf_processor(LlavaNextProcessor)
# In case patch_size is omitted from `processor_config.json`
# e.g. for E5-V: https://huggingface.co/royokong/e5-v
if hf_processor.patch_size is None:
patch_size = self.get_vision_encoder_info().get_patch_size()
hf_processor.patch_size = patch_size
return hf_processor
# Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
def get_num_image_tokens(
+9
View File
@@ -342,6 +342,15 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
**kwargs: object,
):
hf_processor = self.ctx.get_hf_processor()
# NumPy arrays are considered as Iterable but not Sequence in
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
image_processor = hf_processor.image_processor # type: ignore
for attr in ("mean", "std"):
val = getattr(image_processor, attr)
if isinstance(val, np.ndarray):
setattr(image_processor, attr, val.tolist())
return hf_processor
def get_image_processor(self):