Bump Transformers version to 5.10.4 (#41359)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -20,7 +20,7 @@ from typing import Any
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PreTrainedTokenizerFast, WhisperFeatureExtractor
|
||||
from transformers import TokenizersBackend, WhisperFeatureExtractor
|
||||
from transformers.activations import get_activation
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.modeling_outputs import BaseModelOutput
|
||||
@@ -556,15 +556,15 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
|
||||
return WhisperFeatureExtractor.from_pretrained(self.model_id)
|
||||
|
||||
@cached_property
|
||||
def speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
return PreTrainedTokenizerFast.from_pretrained(
|
||||
def speech_tokenizer(self) -> TokenizersBackend:
|
||||
return TokenizersBackend.from_pretrained(
|
||||
self.model_id, subfolder="speech_tokenizer"
|
||||
)
|
||||
|
||||
def get_feature_extractor(self) -> WhisperFeatureExtractor:
|
||||
return self.feature_extractor
|
||||
|
||||
def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
def get_speech_tokenizer(self) -> TokenizersBackend:
|
||||
return self.speech_tokenizer
|
||||
|
||||
def get_data_parser(self):
|
||||
|
||||
@@ -552,6 +552,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
vendored_processor = MiniCPMVProcessor(
|
||||
image_processor=hf_processor.image_processor,
|
||||
tokenizer=hf_processor.tokenizer,
|
||||
version=self.get_model_version(),
|
||||
)
|
||||
hf_processor = vendored_processor
|
||||
|
||||
|
||||
@@ -35,6 +35,8 @@ from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessor
|
||||
from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
ResolvedPromptUpdate,
|
||||
_seq2text,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -408,6 +410,35 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
for modality, pattern in placeholders
|
||||
]
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self, cached_update: ResolvedPromptUpdate, new_item_idx: int
|
||||
) -> ResolvedPromptUpdate:
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update, new_item_idx
|
||||
)
|
||||
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
|
||||
# (the base class only rewrites the image modality).
|
||||
if cached_update.modality == "video":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = _seq2text(tokenizer, cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_text(
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
),
|
||||
video_token,
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs,
|
||||
|
||||
@@ -33,7 +33,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import BatchFeature
|
||||
from transformers import BatchFeature, ProcessorMixin
|
||||
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
|
||||
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
|
||||
smart_resize as image_smart_resize,
|
||||
@@ -1191,34 +1191,31 @@ def _replace_video_token_placeholders(
|
||||
target: list[int],
|
||||
replacements: list[list[int]],
|
||||
) -> list[int]:
|
||||
"""Replace each 3-token video placeholder with its expanded sequence.
|
||||
"""Replace each video placeholder with its expanded sequence.
|
||||
|
||||
Args:
|
||||
prompt_ids: Token IDs of the original (unexpanded) prompt.
|
||||
target: 3-element list ``[vision_start_id, video_pad_id,
|
||||
vision_end_id]`` to search for.
|
||||
target: Token-ID sequence to search for. New-style
|
||||
(transformers>=5.10) processors expand only the bare
|
||||
`[video_pad_id]`; older ones expand the full
|
||||
`[vision_start_id, video_pad_id, vision_end_id]` triplet.
|
||||
replacements: Per-video expanded token sequences, in prompt order.
|
||||
|
||||
Returns:
|
||||
Token IDs with every placeholder triplet replaced.
|
||||
Token IDs with every placeholder replaced.
|
||||
"""
|
||||
result: list[int] = []
|
||||
repl_idx = 0
|
||||
i = 0
|
||||
n = len(prompt_ids)
|
||||
t0, t1, t2 = target
|
||||
k = len(target)
|
||||
num_repl = len(replacements)
|
||||
|
||||
while i < n:
|
||||
if (
|
||||
i + 2 < n
|
||||
and prompt_ids[i] == t0
|
||||
and prompt_ids[i + 1] == t1
|
||||
and prompt_ids[i + 2] == t2
|
||||
):
|
||||
if prompt_ids[i : i + k] == target:
|
||||
result.extend(replacements[repl_idx])
|
||||
repl_idx += 1
|
||||
i += 3
|
||||
i += k
|
||||
else:
|
||||
result.append(prompt_ids[i])
|
||||
i += 1
|
||||
@@ -1230,6 +1227,15 @@ def _replace_video_token_placeholders(
|
||||
|
||||
|
||||
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
|
||||
@staticmethod
|
||||
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
|
||||
"""Transformers>=5.10 processors override `replace_video_token`
|
||||
to expand only the bare video token, keeping the prompt's outer
|
||||
`<|vision_start|>`/`<|vision_end|>` markers."""
|
||||
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
|
||||
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
||||
return proc_impl is not None and proc_impl is not mixin_impl
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -1365,14 +1371,17 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# Replace each placeholder triplet with pre-computed video tokens.
|
||||
# Replace each placeholder with pre-computed video tokens.
|
||||
if video_input_ids_lst:
|
||||
hf_config = self.info.get_hf_config()
|
||||
video_target = [
|
||||
hf_config.vision_start_token_id,
|
||||
hf_config.video_token_id,
|
||||
hf_config.vision_end_token_id,
|
||||
]
|
||||
if self._expands_only_video_token(self.info.get_hf_processor()):
|
||||
video_target = [hf_config.video_token_id]
|
||||
else:
|
||||
video_target = [
|
||||
hf_config.vision_start_token_id,
|
||||
hf_config.video_token_id,
|
||||
hf_config.vision_end_token_id,
|
||||
]
|
||||
input_ids = processed_outputs.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
@@ -1464,17 +1473,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
select_token_id=select_token_id,
|
||||
)
|
||||
|
||||
if self._expands_only_video_token(hf_processor):
|
||||
# transformers>=5.10 expands only the bare video_token
|
||||
video_target = hf_processor.video_token
|
||||
else:
|
||||
# Old-style processors expand the full placeholder
|
||||
# NOTE: We match string on purpose since searching sequence of
|
||||
# token ids takes more time.
|
||||
video_target = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=hf_processor.image_token,
|
||||
replacement=get_image_replacement_qwen3vl,
|
||||
),
|
||||
# NOTE: We match string on purpose since searching sequence of
|
||||
# token ids takes more time.
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<|vision_start|><|video_pad|><|vision_end|>",
|
||||
target=video_target,
|
||||
replacement=get_video_replacement_qwen3vl,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -276,13 +276,13 @@ class Base(
|
||||
- Any quantization config specific mappings
|
||||
"""
|
||||
self.hf_to_vllm_mapper = WeightsMapper()
|
||||
orig_to_new_renamings = self.hf_to_vllm_mapper.orig_to_new_renamings
|
||||
orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming
|
||||
orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex
|
||||
|
||||
for mapping in get_model_conversion_mapping(self.model):
|
||||
# Handle weights which have been renamed in Transformers
|
||||
if isinstance(mapping, WeightRenaming):
|
||||
orig_to_new_renamings.append(mapping)
|
||||
orig_to_new_renaming.append(mapping)
|
||||
# TODO: Handle WeightConverter to enable layer merging
|
||||
|
||||
# Handle unexpected weights which should be ignored
|
||||
|
||||
@@ -11,7 +11,6 @@ import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn.modules.module import register_module_module_registration_hook
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -33,6 +32,9 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PretrainedConfig
|
||||
from transformers.conversion_mapping import WeightRenaming
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -46,7 +48,7 @@ class WeightsMapper:
|
||||
|
||||
If a key maps to a value of `None`, the corresponding weight is ignored."""
|
||||
|
||||
orig_to_new_renamings: list[Any] = field(default_factory=list)
|
||||
orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)
|
||||
orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)
|
||||
orig_to_new_substr: Mapping[str, str | None] = field(default_factory=dict)
|
||||
orig_to_new_stacked: Mapping[str, tuple[str, ShardId]] = field(default_factory=dict)
|
||||
@@ -56,9 +58,9 @@ class WeightsMapper:
|
||||
def __or__(self, other: "WeightsMapper") -> "WeightsMapper":
|
||||
"""Combine two `WeightsMapper`s by merging their mappings."""
|
||||
return WeightsMapper(
|
||||
orig_to_new_renamings=[
|
||||
*self.orig_to_new_renamings,
|
||||
*other.orig_to_new_renamings,
|
||||
orig_to_new_renaming=[
|
||||
*self.orig_to_new_renaming,
|
||||
*other.orig_to_new_renaming,
|
||||
],
|
||||
orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex},
|
||||
orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr},
|
||||
@@ -92,7 +94,7 @@ class WeightsMapper:
|
||||
"k_scale to v_scale"
|
||||
)
|
||||
|
||||
for renaming in self.orig_to_new_renamings:
|
||||
for renaming in self.orig_to_new_renaming:
|
||||
key, _ = renaming.rename_source_key(key)
|
||||
|
||||
for pattern, new_key in self.orig_to_new_regex.items():
|
||||
@@ -426,7 +428,7 @@ def init_vllm_registered_model(
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
hf_config: "PretrainedConfig | None" = None,
|
||||
architectures: list[str] | None = None,
|
||||
) -> nn.Module:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user