[Misc][Minimax-M3]add default video_processor (#50092)

Signed-off-by: rongfu.leng <lenronfu@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
rongfu.leng
2026-07-29 01:24:51 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 32e657e689
commit df2735ea2e
2 changed files with 44 additions and 7 deletions
+7
View File
@@ -14,6 +14,7 @@ from transformers import AutoVideoProcessor
from transformers.video_utils import VideoMetadata from transformers.video_utils import VideoMetadata
from vllm.assets.base import get_vllm_public_assets from vllm.assets.base import get_vllm_public_assets
from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
from vllm.multimodal.video import ( from vllm.multimodal.video import (
PYNVVIDEOCODEC_DECODER_CACHE_SIZE, PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
PYNVVIDEOCODEC_VIDEO_BACKEND, PYNVVIDEOCODEC_VIDEO_BACKEND,
@@ -388,6 +389,12 @@ def test_cosmos3_edge_uses_qwen3_vl_video_backend():
{"fps": 2}, {"fps": 2},
id="qwen2_5_vl", id="qwen2_5_vl",
), ),
pytest.param(
"MiniMaxAI/MiniMax-M3",
MiniMaxM3VideoBackend,
None,
id="minimax_m3_vl",
),
], ],
) )
def test_video_processor_from_model_repo( def test_video_processor_from_model_repo(
+37 -7
View File
@@ -3,8 +3,9 @@
import math import math
from collections.abc import Mapping, Sequence from collections.abc import Mapping, Sequence
from typing import cast from typing import Any, Literal, cast
import numpy.typing as npt
import torch import torch
from transformers import BatchFeature from transformers import BatchFeature
from transformers.video_utils import VideoMetadata from transformers.video_utils import VideoMetadata
@@ -469,10 +470,39 @@ class MiniMaxM3VLMultiModalProcessor(
] ]
# TODO(Isotr0py): Tie with MinimaxVideoProcessor @VIDEO_LOADER_REGISTRY.register(
# after https://github.com/vllm-project/vllm/pull/44126 name="minimax_m3_vl",
@VIDEO_LOADER_REGISTRY.register("minimax_m3_vl") video_processor="MiniMaxM3VLVideoProcessor",
)
class MiniMaxM3VideoBackend(VideoBackend): class MiniMaxM3VideoBackend(VideoBackend):
@classmethod
def load_bytes(
cls,
data: bytes,
num_frames: int = -1,
fps: int = 1,
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal[
"opencv",
"pyav",
"torchcodec",
"pynvvideocodec",
"deepstream",
] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
data,
num_frames=num_frames,
fps=fps,
max_duration=max_duration,
frame_recovery=frame_recovery,
backend=backend,
**kwargs,
)
@classmethod @classmethod
def compute_frames_index_to_sample( def compute_frames_index_to_sample(
cls, cls,
@@ -483,7 +513,6 @@ class MiniMaxM3VideoBackend(VideoBackend):
total_frames = source.total_frames_num total_frames = source.total_frames_num
video_fps = source.original_fps video_fps = source.original_fps
fps = target.fps fps = target.fps
if total_frames <= 0 or video_fps <= 0 or fps <= 0: if total_frames <= 0 or video_fps <= 0 or fps <= 0:
return [0] if total_frames > 0 else [] return [0] if total_frames > 0 else []
@@ -503,8 +532,9 @@ class MiniMaxM3VideoBackend(VideoBackend):
break break
indices.append(target_frame) indices.append(target_frame)
prev_kept_ts = target_frame / video_fps prev_kept_ts = target_frame / video_fps
# Because HF sample_frames includes the last frame,
last_frame_idx = total_frames - 1 # we will use HF as the standard.
last_frame_idx = total_frames
last_ts = last_frame_idx / video_fps last_ts = last_frame_idx / video_fps
if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps: if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps:
indices.append(last_frame_idx) indices.append(last_frame_idx)