Bump Transformers version to 5.15.0 (#51668)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-11 09:00:51 -07:00
committed by GitHub
parent 457a5f34db
commit c65aa2ee6c
11 changed files with 23 additions and 17 deletions
@@ -390,6 +390,14 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
split_sizes = mm_tokens_per_modality["num_image_tokens"]
if split_sizes:
image_token_ids = getattr(hf_processor, "image_token_ids", None)
if image_token_ids is None:
# Transformers <5.10.0
image_token_ids = [hf_processor.image_token_id]
image_token_ids = torch.tensor(
[i for i in image_token_ids if i is not None]
)
chunked_mm_positions = torch.split(mm_positions, split_sizes)
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
@@ -397,7 +405,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
PlaceholderRange(
offset=positions[0].item(),
length=positions.shape[0],
is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
is_embed=torch.isin(mm_tokens, image_token_ids),
)
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
]