fix(processor): route MiMo-V2-Omni media fetch through MediaConnector (#43117)
Signed-off-by: Ievgen Bondarenko <ibondarenko@student.sierracollege.edu> Signed-off-by: Ievgen (Jack) Bondarenko <ibondarenko@student.sierracollege.edu> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
Isotr0py
parent
0067311536
commit
54503ecec0
@@ -7,33 +7,21 @@ Ported from SGLang's MiMoV2OmniProcessor / MiMoVLProcessor implementations.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
import copy
|
|
||||||
import io
|
|
||||||
import logging
|
import logging
|
||||||
import math
|
import math
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from io import BytesIO
|
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import regex as re
|
import regex as re
|
||||||
import requests
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from transformers import BatchFeature, TensorType
|
from transformers import BatchFeature, TensorType
|
||||||
from transformers.processing_utils import ProcessorMixin
|
from transformers.processing_utils import ProcessorMixin
|
||||||
|
|
||||||
try:
|
|
||||||
from torchcodec.decoders import AudioDecoder
|
|
||||||
|
|
||||||
_HAS_TORCHCODEC = True
|
|
||||||
except ImportError:
|
|
||||||
AudioDecoder = None
|
|
||||||
_HAS_TORCHCODEC = False
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import torchaudio
|
import torchaudio
|
||||||
from torchaudio.transforms import MelSpectrogram as _MelSpectrogram
|
from torchaudio.transforms import MelSpectrogram as _MelSpectrogram
|
||||||
@@ -62,7 +50,7 @@ _mean_std_cache: dict[str, tuple[torch.Tensor, torch.Tensor]] = {}
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ImageInput:
|
class ImageInput:
|
||||||
# PIL.Image | str (path/url/base64) | bytes | torch.Tensor (C,H,W)
|
# PIL.Image | torch.Tensor (C,H,W)
|
||||||
image: Any
|
image: Any
|
||||||
max_pixels: int | None = None
|
max_pixels: int | None = None
|
||||||
min_pixels: int | None = None
|
min_pixels: int | None = None
|
||||||
@@ -87,7 +75,7 @@ class VideoInput:
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class AudioInput:
|
class AudioInput:
|
||||||
# str (path/url/base64) | bytes | tuple[waveform_1D, sr]
|
# tuple[waveform_1D, sr]
|
||||||
# | np.ndarray | torch.Tensor (T,n_vq)
|
# | np.ndarray | torch.Tensor (T,n_vq)
|
||||||
audio: Any
|
audio: Any
|
||||||
|
|
||||||
@@ -168,14 +156,6 @@ def _smart_resize(
|
|||||||
return int(h_bar), int(w_bar)
|
return int(h_bar), int(w_bar)
|
||||||
|
|
||||||
|
|
||||||
def _to_rgb(img: Image.Image) -> Image.Image:
|
|
||||||
if img.mode == "RGBA":
|
|
||||||
bg = Image.new("RGB", img.size, (255, 255, 255))
|
|
||||||
bg.paste(img, mask=img.split()[3])
|
|
||||||
return bg
|
|
||||||
return img.convert("RGB")
|
|
||||||
|
|
||||||
|
|
||||||
def _standardize(images: torch.Tensor) -> torch.Tensor:
|
def _standardize(images: torch.Tensor) -> torch.Tensor:
|
||||||
key = str(images.device)
|
key = str(images.device)
|
||||||
if key not in _mean_std_cache:
|
if key not in _mean_std_cache:
|
||||||
@@ -228,27 +208,6 @@ def _transform_single(
|
|||||||
return _standardize(out).squeeze(0), w_bar, h_bar
|
return _standardize(out).squeeze(0), w_bar, h_bar
|
||||||
|
|
||||||
|
|
||||||
def _fetch_image(src: Any) -> Image.Image:
|
|
||||||
if isinstance(src, Image.Image):
|
|
||||||
return _to_rgb(src)
|
|
||||||
if isinstance(src, bytes):
|
|
||||||
return _to_rgb(copy.deepcopy(Image.open(BytesIO(src))))
|
|
||||||
if isinstance(src, str):
|
|
||||||
if src.startswith(("http://", "https://")):
|
|
||||||
r = requests.get(src, timeout=30)
|
|
||||||
r.raise_for_status()
|
|
||||||
return _to_rgb(copy.deepcopy(Image.open(BytesIO(r.content))))
|
|
||||||
if src.startswith("file://"):
|
|
||||||
return _to_rgb(Image.open(src[7:]))
|
|
||||||
if src.startswith("data:image"):
|
|
||||||
import pybase64 as _b64
|
|
||||||
|
|
||||||
_, b64 = src.split("base64,", 1)
|
|
||||||
return _to_rgb(copy.deepcopy(Image.open(BytesIO(_b64.b64decode(b64)))))
|
|
||||||
return _to_rgb(Image.open(src))
|
|
||||||
raise ValueError(f"Unrecognized image source: {type(src)}")
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Core processor
|
# Core processor
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -259,6 +218,10 @@ class MiMoVLProcessor:
|
|||||||
|
|
||||||
Handles image/video/audio preprocessing and token sequence construction.
|
Handles image/video/audio preprocessing and token sequence construction.
|
||||||
Ported from SGLang's MiMoVLProcessor.
|
Ported from SGLang's MiMoVLProcessor.
|
||||||
|
|
||||||
|
Media strings (URL / path / data:) are resolved upstream in vLLM's media
|
||||||
|
pipeline; this processor accepts only already-decoded inputs (``PIL.Image``
|
||||||
|
for images, ``(waveform, sr)`` tuples for audio).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -451,33 +414,14 @@ class MiMoVLProcessor:
|
|||||||
return kw
|
return kw
|
||||||
|
|
||||||
def preprocess_audio(self, audio: Any) -> tuple[torch.Tensor, int]:
|
def preprocess_audio(self, audio: Any) -> tuple[torch.Tensor, int]:
|
||||||
"""Decode audio bytes/path/tuple → (mel_spec (T, n_mels), token_len)."""
|
"""Convert a pre-loaded ``(waveform, sr)`` tuple into (mel_spec, token_len)."""
|
||||||
if isinstance(audio, tuple):
|
if not isinstance(audio, tuple):
|
||||||
waveform, original_sr = audio
|
raise ValueError(
|
||||||
else:
|
f"Unsupported audio source type: {type(audio)}. Audio must be a "
|
||||||
if AudioDecoder is None:
|
"pre-decoded (waveform, sample_rate) tuple; URL/path/bytes "
|
||||||
raise RuntimeError(
|
"resolution is handled upstream in vLLM's media pipeline."
|
||||||
"torchcodec is required for audio. "
|
)
|
||||||
"Install with: pip install torchcodec"
|
waveform, original_sr = audio
|
||||||
)
|
|
||||||
if isinstance(audio, bytes):
|
|
||||||
file_obj: Any = io.BytesIO(audio)
|
|
||||||
elif isinstance(audio, str):
|
|
||||||
if audio.startswith("data:"):
|
|
||||||
import pybase64 as _b64
|
|
||||||
|
|
||||||
file_obj = io.BytesIO(_b64.b64decode(audio.split(",")[1]))
|
|
||||||
elif audio.startswith(("http://", "https://")):
|
|
||||||
r = requests.get(audio, timeout=30)
|
|
||||||
r.raise_for_status()
|
|
||||||
file_obj = io.BytesIO(r.content)
|
|
||||||
else:
|
|
||||||
file_obj = audio
|
|
||||||
else:
|
|
||||||
raise ValueError(f"Unsupported audio source type: {type(audio)}")
|
|
||||||
samples = AudioDecoder(file_obj).get_all_samples()
|
|
||||||
waveform = samples.data
|
|
||||||
original_sr = samples.sample_rate
|
|
||||||
|
|
||||||
if original_sr != self.audio_sampling_rate:
|
if original_sr != self.audio_sampling_rate:
|
||||||
if original_sr not in self._resamplers:
|
if original_sr not in self._resamplers:
|
||||||
@@ -504,8 +448,6 @@ class MiMoVLProcessor:
|
|||||||
def process_image(self, image: ImageInput) -> torch.Tensor:
|
def process_image(self, image: ImageInput) -> torch.Tensor:
|
||||||
kw = self._resolve_img_kw(image)
|
kw = self._resolve_img_kw(image)
|
||||||
src = image.image
|
src = image.image
|
||||||
if isinstance(src, (str, bytes)):
|
|
||||||
src = _fetch_image(src)
|
|
||||||
tensor, _, _ = _transform_single(
|
tensor, _, _ = _transform_single(
|
||||||
src,
|
src,
|
||||||
factor=self.patch_size * self.merge_size,
|
factor=self.patch_size * self.merge_size,
|
||||||
@@ -584,7 +526,7 @@ class MiMoVLProcessor:
|
|||||||
src = audio.audio
|
src = audio.audio
|
||||||
if isinstance(src, np.ndarray):
|
if isinstance(src, np.ndarray):
|
||||||
src = (torch.from_numpy(src).float(), self.audio_sampling_rate)
|
src = (torch.from_numpy(src).float(), self.audio_sampling_rate)
|
||||||
if isinstance(src, (str, bytes, tuple)):
|
if isinstance(src, tuple):
|
||||||
return self.preprocess_audio(src)
|
return self.preprocess_audio(src)
|
||||||
# Pre-tokenized tensor (T, n_vq)
|
# Pre-tokenized tensor (T, n_vq)
|
||||||
assert isinstance(src, torch.Tensor) and src.ndim == 2
|
assert isinstance(src, torch.Tensor) and src.ndim == 2
|
||||||
@@ -899,7 +841,7 @@ class MiMoOmniProcessor(ProcessorMixin):
|
|||||||
"""HuggingFace-compatible ProcessorMixin wrapper for MiMo-Omni.
|
"""HuggingFace-compatible ProcessorMixin wrapper for MiMo-Omni.
|
||||||
|
|
||||||
Accepts PIL images, pre-decoded video tuples (frames_TCHW, timestamps_T),
|
Accepts PIL images, pre-decoded video tuples (frames_TCHW, timestamps_T),
|
||||||
and audio (file path / bytes / (waveform, sr) tuple / numpy array).
|
and audio (file path / (waveform, sr) tuple / numpy array).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
attributes = ["tokenizer"]
|
attributes = ["tokenizer"]
|
||||||
@@ -1011,8 +953,12 @@ class MiMoOmniProcessor(ProcessorMixin):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_hf_config(cls, tokenizer: Any, hf_config: Any) -> "MiMoOmniProcessor":
|
def from_hf_config(
|
||||||
"""Convenience factory: instantiate directly from an HF model config object."""
|
cls,
|
||||||
|
tokenizer: Any,
|
||||||
|
hf_config: Any,
|
||||||
|
) -> "MiMoOmniProcessor":
|
||||||
|
"""Instantiate directly from an HF model config object."""
|
||||||
vc = hf_config.vision_config
|
vc = hf_config.vision_config
|
||||||
if isinstance(vc, dict):
|
if isinstance(vc, dict):
|
||||||
patch_size = vc.get("patch_size", 14)
|
patch_size = vc.get("patch_size", 14)
|
||||||
|
|||||||
Reference in New Issue
Block a user