[Bugfix] Vendor MiniCPMV/MiniCPMO processors to unblock Transformers v5 (#44282)

Signed-off-by: guanwei-wu <b08901019@ntu.edu.tw>
Signed-off-by: wjinxu <1299461899@qq.com>
Co-authored-by: guanwei-wu <b08901019@ntu.edu.tw>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王金旭
2026-06-02 07:14:38 -07:00
committed by GitHub
co-authored by guanwei-wu Cursor
parent 0cbc48c4f9
commit 0bdfd5eb84
7 changed files with 965 additions and 16 deletions
+36 -1
View File
@@ -26,7 +26,7 @@
import os
from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias
from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias
import torch
from torch import nn
@@ -75,6 +75,9 @@ from .utils import AutoWeightsLoader, cast_overflow_tensors, maybe_prefix
CPU_DEVICE = torch.device("cpu")
if TYPE_CHECKING:
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
if os.getenv("USE_FLAGOS") == "1":
import flag_gems
@@ -270,6 +273,38 @@ class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
audio_pattern = "(<audio>./</audio>)"
def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
"""Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
Creates a vendored processor that reuses the HF image processor,
feature extractor, and tokenizer; applies the correct audio pooling
configuration; and converts numpy arrays in the image processor to
lists for serialization compatibility. The returned processor is
compatible with Transformers v5.
"""
import numpy as np
hf_processor = self.ctx.get_hf_processor(**kwargs)
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
# Create vendored processor with correct configuration
vendored_processor = MiniCPMOProcessor(
image_processor=hf_processor.image_processor,
feature_extractor=hf_processor.feature_extractor,
tokenizer=hf_processor.tokenizer,
pool_step=self.get_default_audio_pool_step(),
)
# Convert numpy arrays in image processor to lists for serialization
image_processor = vendored_processor.image_processor
for attr in ("mean", "std"):
val = getattr(image_processor, attr, None)
if val is not None and isinstance(val, np.ndarray):
setattr(image_processor, attr, val.tolist())
return vendored_processor
def get_data_parser(self):
return MiniCPMOMultiModalDataParser(
target_sr=self.get_default_audio_sampling_rate(),
+8
View File
@@ -545,6 +545,14 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object):
hf_processor = self.ctx.get_hf_processor(**kwargs)
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
vendored_processor = MiniCPMVProcessor(
image_processor=hf_processor.image_processor,
tokenizer=hf_processor.tokenizer,
)
hf_processor = vendored_processor
# NumPy arrays are considered as Iterable but not Sequence in
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
image_processor = hf_processor.image_processor # type: ignore