[Bugfix] Vendor MiniCPMV/MiniCPMO processors to unblock Transformers v5 (#44282)
Signed-off-by: guanwei-wu <b08901019@ntu.edu.tw> Signed-off-by: wjinxu <1299461899@qq.com> Co-authored-by: guanwei-wu <b08901019@ntu.edu.tw> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -26,7 +26,7 @@
|
||||
|
||||
import os
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -75,6 +75,9 @@ from .utils import AutoWeightsLoader, cast_overflow_tensors, maybe_prefix
|
||||
|
||||
CPU_DEVICE = torch.device("cpu")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
|
||||
|
||||
if os.getenv("USE_FLAGOS") == "1":
|
||||
import flag_gems
|
||||
|
||||
@@ -270,6 +273,38 @@ class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
|
||||
class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
|
||||
audio_pattern = "(<audio>./</audio>)"
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
|
||||
"""Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
|
||||
|
||||
Creates a vendored processor that reuses the HF image processor,
|
||||
feature extractor, and tokenizer; applies the correct audio pooling
|
||||
configuration; and converts numpy arrays in the image processor to
|
||||
lists for serialization compatibility. The returned processor is
|
||||
compatible with Transformers v5.
|
||||
"""
|
||||
import numpy as np
|
||||
|
||||
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
||||
|
||||
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
|
||||
|
||||
# Create vendored processor with correct configuration
|
||||
vendored_processor = MiniCPMOProcessor(
|
||||
image_processor=hf_processor.image_processor,
|
||||
feature_extractor=hf_processor.feature_extractor,
|
||||
tokenizer=hf_processor.tokenizer,
|
||||
pool_step=self.get_default_audio_pool_step(),
|
||||
)
|
||||
|
||||
# Convert numpy arrays in image processor to lists for serialization
|
||||
image_processor = vendored_processor.image_processor
|
||||
for attr in ("mean", "std"):
|
||||
val = getattr(image_processor, attr, None)
|
||||
if val is not None and isinstance(val, np.ndarray):
|
||||
setattr(image_processor, attr, val.tolist())
|
||||
|
||||
return vendored_processor
|
||||
|
||||
def get_data_parser(self):
|
||||
return MiniCPMOMultiModalDataParser(
|
||||
target_sr=self.get_default_audio_sampling_rate(),
|
||||
|
||||
@@ -545,6 +545,14 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
||||
|
||||
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
|
||||
|
||||
vendored_processor = MiniCPMVProcessor(
|
||||
image_processor=hf_processor.image_processor,
|
||||
tokenizer=hf_processor.tokenizer,
|
||||
)
|
||||
hf_processor = vendored_processor
|
||||
|
||||
# NumPy arrays are considered as Iterable but not Sequence in
|
||||
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
|
||||
image_processor = hf_processor.image_processor # type: ignore
|
||||
|
||||
Reference in New Issue
Block a user