Bump Transformers version to 5.10.4 (#41359)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-07 05:13:28 -07:00
committed by GitHub
parent 7ff656cc8b
commit dd94484577
51 changed files with 271 additions and 234 deletions
+4 -4
View File
@@ -20,7 +20,7 @@ from typing import Any
import numpy as np
import torch
import torch.nn as nn
from transformers import PreTrainedTokenizerFast, WhisperFeatureExtractor
from transformers import TokenizersBackend, WhisperFeatureExtractor
from transformers.activations import get_activation
from transformers.feature_extraction_utils import BatchFeature
from transformers.modeling_outputs import BaseModelOutput
@@ -556,15 +556,15 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
return WhisperFeatureExtractor.from_pretrained(self.model_id)
@cached_property
def speech_tokenizer(self) -> PreTrainedTokenizerFast:
return PreTrainedTokenizerFast.from_pretrained(
def speech_tokenizer(self) -> TokenizersBackend:
return TokenizersBackend.from_pretrained(
self.model_id, subfolder="speech_tokenizer"
)
def get_feature_extractor(self) -> WhisperFeatureExtractor:
return self.feature_extractor
def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
def get_speech_tokenizer(self) -> TokenizersBackend:
return self.speech_tokenizer
def get_data_parser(self):
+1
View File
@@ -552,6 +552,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
vendored_processor = MiniCPMVProcessor(
image_processor=hf_processor.image_processor,
tokenizer=hf_processor.tokenizer,
version=self.get_model_version(),
)
hf_processor = vendored_processor
+31
View File
@@ -35,6 +35,8 @@ from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessor
from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdateDetails,
ResolvedPromptUpdate,
_seq2text,
)
from vllm.sequence import IntermediateTensors
@@ -408,6 +410,35 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
for modality, pattern in placeholders
]
def _recompute_cached_prompt_update(
self, cached_update: ResolvedPromptUpdate, new_item_idx: int
) -> ResolvedPromptUpdate:
new_update = super()._recompute_cached_prompt_update(
cached_update, new_item_idx
)
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
# (the base class only rewrites the image modality).
if cached_update.modality == "video":
tokenizer = self.info.get_tokenizer()
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
text = _seq2text(tokenizer, cached_update.content.full)
prev_item_idx = cached_update.item_idx
new_update = new_update.with_content(
PromptUpdateDetails.select_text(
text.replace(
f"{id_start}{prev_item_idx}{id_end}",
f"{id_start}{new_item_idx}{id_end}",
1,
),
video_token,
)
)
return new_update
def _get_mm_fields_config(
self,
hf_inputs,
+38 -22
View File
@@ -33,7 +33,7 @@ import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BatchFeature
from transformers import BatchFeature, ProcessorMixin
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
smart_resize as image_smart_resize,
@@ -1191,34 +1191,31 @@ def _replace_video_token_placeholders(
target: list[int],
replacements: list[list[int]],
) -> list[int]:
"""Replace each 3-token video placeholder with its expanded sequence.
"""Replace each video placeholder with its expanded sequence.
Args:
prompt_ids: Token IDs of the original (unexpanded) prompt.
target: 3-element list ``[vision_start_id, video_pad_id,
vision_end_id]`` to search for.
target: Token-ID sequence to search for. New-style
(transformers>=5.10) processors expand only the bare
`[video_pad_id]`; older ones expand the full
`[vision_start_id, video_pad_id, vision_end_id]` triplet.
replacements: Per-video expanded token sequences, in prompt order.
Returns:
Token IDs with every placeholder triplet replaced.
Token IDs with every placeholder replaced.
"""
result: list[int] = []
repl_idx = 0
i = 0
n = len(prompt_ids)
t0, t1, t2 = target
k = len(target)
num_repl = len(replacements)
while i < n:
if (
i + 2 < n
and prompt_ids[i] == t0
and prompt_ids[i + 1] == t1
and prompt_ids[i + 2] == t2
):
if prompt_ids[i : i + k] == target:
result.extend(replacements[repl_idx])
repl_idx += 1
i += 3
i += k
else:
result.append(prompt_ids[i])
i += 1
@@ -1230,6 +1227,15 @@ def _replace_video_token_placeholders(
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
@staticmethod
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
"""Transformers>=5.10 processors override `replace_video_token`
to expand only the bare video token, keeping the prompt's outer
`<|vision_start|>`/`<|vision_end|>` markers."""
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
return proc_impl is not None and proc_impl is not mixin_impl
def _call_hf_processor(
self,
prompt: str,
@@ -1365,14 +1371,17 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
tok_kwargs=tok_kwargs,
)
# Replace each placeholder triplet with pre-computed video tokens.
# Replace each placeholder with pre-computed video tokens.
if video_input_ids_lst:
hf_config = self.info.get_hf_config()
video_target = [
hf_config.vision_start_token_id,
hf_config.video_token_id,
hf_config.vision_end_token_id,
]
if self._expands_only_video_token(self.info.get_hf_processor()):
video_target = [hf_config.video_token_id]
else:
video_target = [
hf_config.vision_start_token_id,
hf_config.video_token_id,
hf_config.vision_end_token_id,
]
input_ids = processed_outputs.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
@@ -1464,17 +1473,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
select_token_id=select_token_id,
)
if self._expands_only_video_token(hf_processor):
# transformers>=5.10 expands only the bare video_token
video_target = hf_processor.video_token
else:
# Old-style processors expand the full placeholder
# NOTE: We match string on purpose since searching sequence of
# token ids takes more time.
video_target = "<|vision_start|><|video_pad|><|vision_end|>"
return [
PromptReplacement(
modality="image",
target=hf_processor.image_token,
replacement=get_image_replacement_qwen3vl,
),
# NOTE: We match string on purpose since searching sequence of
# token ids takes more time.
PromptReplacement(
modality="video",
target="<|vision_start|><|video_pad|><|vision_end|>",
target=video_target,
replacement=get_video_replacement_qwen3vl,
),
]
@@ -276,13 +276,13 @@ class Base(
- Any quantization config specific mappings
"""
self.hf_to_vllm_mapper = WeightsMapper()
orig_to_new_renamings = self.hf_to_vllm_mapper.orig_to_new_renamings
orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming
orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex
for mapping in get_model_conversion_mapping(self.model):
# Handle weights which have been renamed in Transformers
if isinstance(mapping, WeightRenaming):
orig_to_new_renamings.append(mapping)
orig_to_new_renaming.append(mapping)
# TODO: Handle WeightConverter to enable layer merging
# Handle unexpected weights which should be ignored
+9 -7
View File
@@ -11,7 +11,6 @@ import regex as re
import torch
import torch.nn as nn
from torch.nn.modules.module import register_module_module_registration_hook
from transformers import PretrainedConfig
from vllm.config import VllmConfig
from vllm.distributed import (
@@ -33,6 +32,9 @@ from vllm.utils.torch_utils import (
)
if TYPE_CHECKING:
from transformers import PretrainedConfig
from transformers.conversion_mapping import WeightRenaming
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
@@ -46,7 +48,7 @@ class WeightsMapper:
If a key maps to a value of `None`, the corresponding weight is ignored."""
orig_to_new_renamings: list[Any] = field(default_factory=list)
orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)
orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)
orig_to_new_substr: Mapping[str, str | None] = field(default_factory=dict)
orig_to_new_stacked: Mapping[str, tuple[str, ShardId]] = field(default_factory=dict)
@@ -56,9 +58,9 @@ class WeightsMapper:
def __or__(self, other: "WeightsMapper") -> "WeightsMapper":
"""Combine two `WeightsMapper`s by merging their mappings."""
return WeightsMapper(
orig_to_new_renamings=[
*self.orig_to_new_renamings,
*other.orig_to_new_renamings,
orig_to_new_renaming=[
*self.orig_to_new_renaming,
*other.orig_to_new_renaming,
],
orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex},
orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr},
@@ -92,7 +94,7 @@ class WeightsMapper:
"k_scale to v_scale"
)
for renaming in self.orig_to_new_renamings:
for renaming in self.orig_to_new_renaming:
key, _ = renaming.rename_source_key(key)
for pattern, new_key in self.orig_to_new_regex.items():
@@ -426,7 +428,7 @@ def init_vllm_registered_model(
vllm_config: VllmConfig,
*,
prefix: str = "",
hf_config: PretrainedConfig | None = None,
hf_config: "PretrainedConfig | None" = None,
architectures: list[str] | None = None,
) -> nn.Module:
"""