This commit is contained in:
@@ -14,7 +14,6 @@ from transformers import AutoVideoProcessor
|
|||||||
from transformers.video_utils import VideoMetadata
|
from transformers.video_utils import VideoMetadata
|
||||||
|
|
||||||
from vllm.assets.base import get_vllm_public_assets
|
from vllm.assets.base import get_vllm_public_assets
|
||||||
from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
|
|
||||||
from vllm.multimodal.video import (
|
from vllm.multimodal.video import (
|
||||||
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
||||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||||
@@ -389,12 +388,6 @@ def test_cosmos3_edge_uses_qwen3_vl_video_backend():
|
|||||||
{"fps": 2},
|
{"fps": 2},
|
||||||
id="qwen2_5_vl",
|
id="qwen2_5_vl",
|
||||||
),
|
),
|
||||||
pytest.param(
|
|
||||||
"MiniMaxAI/MiniMax-M3",
|
|
||||||
MiniMaxM3VideoBackend,
|
|
||||||
None,
|
|
||||||
id="minimax_m3_vl",
|
|
||||||
),
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_video_processor_from_model_repo(
|
def test_video_processor_from_model_repo(
|
||||||
|
|||||||
@@ -3,9 +3,8 @@
|
|||||||
|
|
||||||
import math
|
import math
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
from typing import Any, Literal, cast
|
from typing import cast
|
||||||
|
|
||||||
import numpy.typing as npt
|
|
||||||
import torch
|
import torch
|
||||||
from transformers import BatchFeature
|
from transformers import BatchFeature
|
||||||
from transformers.video_utils import VideoMetadata
|
from transformers.video_utils import VideoMetadata
|
||||||
@@ -470,39 +469,10 @@ class MiniMaxM3VLMultiModalProcessor(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@VIDEO_LOADER_REGISTRY.register(
|
# TODO(Isotr0py): Tie with MinimaxVideoProcessor
|
||||||
name="minimax_m3_vl",
|
# after https://github.com/vllm-project/vllm/pull/44126
|
||||||
video_processor="MiniMaxM3VLVideoProcessor",
|
@VIDEO_LOADER_REGISTRY.register("minimax_m3_vl")
|
||||||
)
|
|
||||||
class MiniMaxM3VideoBackend(VideoBackend):
|
class MiniMaxM3VideoBackend(VideoBackend):
|
||||||
@classmethod
|
|
||||||
def load_bytes(
|
|
||||||
cls,
|
|
||||||
data: bytes,
|
|
||||||
num_frames: int = -1,
|
|
||||||
fps: int = 1,
|
|
||||||
max_duration: int = 300,
|
|
||||||
frame_recovery: bool = False,
|
|
||||||
*,
|
|
||||||
backend: Literal[
|
|
||||||
"opencv",
|
|
||||||
"pyav",
|
|
||||||
"torchcodec",
|
|
||||||
"pynvvideocodec",
|
|
||||||
"deepstream",
|
|
||||||
] = "opencv",
|
|
||||||
**kwargs,
|
|
||||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
|
||||||
return super().load_bytes(
|
|
||||||
data,
|
|
||||||
num_frames=num_frames,
|
|
||||||
fps=fps,
|
|
||||||
max_duration=max_duration,
|
|
||||||
frame_recovery=frame_recovery,
|
|
||||||
backend=backend,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def compute_frames_index_to_sample(
|
def compute_frames_index_to_sample(
|
||||||
cls,
|
cls,
|
||||||
@@ -513,6 +483,7 @@ class MiniMaxM3VideoBackend(VideoBackend):
|
|||||||
total_frames = source.total_frames_num
|
total_frames = source.total_frames_num
|
||||||
video_fps = source.original_fps
|
video_fps = source.original_fps
|
||||||
fps = target.fps
|
fps = target.fps
|
||||||
|
|
||||||
if total_frames <= 0 or video_fps <= 0 or fps <= 0:
|
if total_frames <= 0 or video_fps <= 0 or fps <= 0:
|
||||||
return [0] if total_frames > 0 else []
|
return [0] if total_frames > 0 else []
|
||||||
|
|
||||||
@@ -532,9 +503,8 @@ class MiniMaxM3VideoBackend(VideoBackend):
|
|||||||
break
|
break
|
||||||
indices.append(target_frame)
|
indices.append(target_frame)
|
||||||
prev_kept_ts = target_frame / video_fps
|
prev_kept_ts = target_frame / video_fps
|
||||||
# Because HF sample_frames includes the last frame,
|
|
||||||
# we will use HF as the standard.
|
last_frame_idx = total_frames - 1
|
||||||
last_frame_idx = total_frames
|
|
||||||
last_ts = last_frame_idx / video_fps
|
last_ts = last_frame_idx / video_fps
|
||||||
if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps:
|
if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps:
|
||||||
indices.append(last_frame_idx)
|
indices.append(last_frame_idx)
|
||||||
|
|||||||
Reference in New Issue
Block a user