Update to Transformers v4.56.2 (#24638)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Harry Mellor
2025-09-30 22:07:07 -07:00
committed by GitHub
co-authored by Cyrus Leung Cyrus Leung Isotr0py
parent 8d7da92fd7
commit 2a69ab4899
5 changed files with 20 additions and 34 deletions
+11 -27
View File
@@ -51,7 +51,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
BaseProcessingInfo)
from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors
from vllm.utils import is_list_of
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
SupportsMultiModal, SupportsPP, SupportsQuant)
@@ -217,9 +216,6 @@ def init_on_device_without_buffers(device: torch.device):
class MultiModalProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.model_config.hf_config
def get_supported_mm_limits(self):
return {"image": None}
@@ -784,6 +780,7 @@ def flatten_and_concat(x: list[torch.Tensor]) -> torch.Tensor:
},
enable_if=can_enable_torch_compile)
class TransformersForMultimodalLM(TransformersForCausalLM, SupportsMultiModal):
merge_by_field_config = True
# Backwards compatibility for prev released models. State dicts back then
# had different formats and cannot be loaded with `AutoModel` mapping as is
hf_to_vllm_mapper = WeightsMapper(
@@ -828,40 +825,27 @@ class TransformersForMultimodalLM(TransformersForCausalLM, SupportsMultiModal):
return self.model
def get_multimodal_embeddings(self, **kwargs):
pixel_values = kwargs.pop("pixel_values", None)
pixel_values = pixel_values if pixel_values is not None else kwargs.pop(
"image_patches", None)
image_embeds = kwargs.pop("image_embeds", None)
pixel_values: Optional[torch.Tensor] = kwargs.pop("pixel_values", None)
image_embeds: Optional[torch.Tensor] = kwargs.pop("image_embeds", None)
# Model might use `image_patches` instead of `pixel_values`
if pixel_values is None:
pixel_values = kwargs.pop("image_patches", None)
if image_embeds is not None:
return image_embeds
if pixel_values is None and image_embeds is None:
if pixel_values is None:
return None
num_image_patches = kwargs.pop("num_image_patches")
if pixel_values is not None:
if isinstance(pixel_values, torch.Tensor):
pixel_values = flatten_bn(pixel_values).to(self.dtype)
elif is_list_of(pixel_values, torch.Tensor):
pixel_values = flatten_and_concat(pixel_values).to(self.dtype)
else:
raise ValueError(
f"Unsupported pixel_values type {type(pixel_values)}. "
"Expected `torch.Tensor` or list of `torch.Tensor`.")
if isinstance(num_image_patches, list):
num_image_patches = torch.cat(num_image_patches)
vision_embeddings = self.model.get_image_features(
pixel_values,
**{
k: v.flatten(0, 1)
for k, v in kwargs.items()
},
)
pixel_values, **kwargs)
if isinstance(vision_embeddings, torch.Tensor):
if isinstance(num_image_patches, list):
num_image_patches = torch.cat(num_image_patches)
if vision_embeddings.ndim == 2:
vision_embeddings = vision_embeddings.unsqueeze(0)