[Bugfix] Re-land MiniMax M3 default video processor (#50305)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
parent
10e6b40015
commit
f727951d3f
@@ -14,6 +14,7 @@ from transformers import AutoVideoProcessor
|
|||||||
from transformers.video_utils import VideoMetadata
|
from transformers.video_utils import VideoMetadata
|
||||||
|
|
||||||
from vllm.assets.base import get_vllm_public_assets
|
from vllm.assets.base import get_vllm_public_assets
|
||||||
|
from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
|
||||||
from vllm.multimodal.video import (
|
from vllm.multimodal.video import (
|
||||||
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
||||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||||
@@ -388,6 +389,12 @@ def test_cosmos3_edge_uses_qwen3_vl_video_backend():
|
|||||||
{"fps": 2},
|
{"fps": 2},
|
||||||
id="qwen2_5_vl",
|
id="qwen2_5_vl",
|
||||||
),
|
),
|
||||||
|
pytest.param(
|
||||||
|
"MiniMaxAI/MiniMax-M3",
|
||||||
|
MiniMaxM3VideoBackend,
|
||||||
|
None,
|
||||||
|
id="minimax_m3_vl",
|
||||||
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_video_processor_from_model_repo(
|
def test_video_processor_from_model_repo(
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ try:
|
|||||||
if flashinfer_comm is not None
|
if flashinfer_comm is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
except ImportError:
|
except (ImportError, AttributeError):
|
||||||
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
|
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
|
||||||
get_fi_ar_workspace = None # type: ignore[assignment]
|
get_fi_ar_workspace = None # type: ignore[assignment]
|
||||||
_AR_RESIDUAL_RMS_NORM = None
|
_AR_RESIDUAL_RMS_NORM = None
|
||||||
|
|||||||
@@ -3,8 +3,9 @@
|
|||||||
|
|
||||||
import math
|
import math
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
from typing import cast
|
from typing import Any, Literal, cast
|
||||||
|
|
||||||
|
import numpy.typing as npt
|
||||||
import torch
|
import torch
|
||||||
from transformers import BatchFeature
|
from transformers import BatchFeature
|
||||||
from transformers.video_utils import VideoMetadata
|
from transformers.video_utils import VideoMetadata
|
||||||
@@ -469,10 +470,39 @@ class MiniMaxM3VLMultiModalProcessor(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
# TODO(Isotr0py): Tie with MinimaxVideoProcessor
|
@VIDEO_LOADER_REGISTRY.register(
|
||||||
# after https://github.com/vllm-project/vllm/pull/44126
|
name="minimax_m3_vl",
|
||||||
@VIDEO_LOADER_REGISTRY.register("minimax_m3_vl")
|
video_processor="MiniMaxM3VLVideoProcessor",
|
||||||
|
)
|
||||||
class MiniMaxM3VideoBackend(VideoBackend):
|
class MiniMaxM3VideoBackend(VideoBackend):
|
||||||
|
@classmethod
|
||||||
|
def load_bytes(
|
||||||
|
cls,
|
||||||
|
data: bytes,
|
||||||
|
num_frames: int = -1,
|
||||||
|
fps: int = 1,
|
||||||
|
max_duration: int = 300,
|
||||||
|
frame_recovery: bool = False,
|
||||||
|
*,
|
||||||
|
backend: Literal[
|
||||||
|
"opencv",
|
||||||
|
"pyav",
|
||||||
|
"torchcodec",
|
||||||
|
"pynvvideocodec",
|
||||||
|
"deepstream",
|
||||||
|
] = "opencv",
|
||||||
|
**kwargs,
|
||||||
|
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||||
|
return super().load_bytes(
|
||||||
|
data,
|
||||||
|
num_frames=num_frames,
|
||||||
|
fps=fps,
|
||||||
|
max_duration=max_duration,
|
||||||
|
frame_recovery=frame_recovery,
|
||||||
|
backend=backend,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def compute_frames_index_to_sample(
|
def compute_frames_index_to_sample(
|
||||||
cls,
|
cls,
|
||||||
@@ -483,7 +513,6 @@ class MiniMaxM3VideoBackend(VideoBackend):
|
|||||||
total_frames = source.total_frames_num
|
total_frames = source.total_frames_num
|
||||||
video_fps = source.original_fps
|
video_fps = source.original_fps
|
||||||
fps = target.fps
|
fps = target.fps
|
||||||
|
|
||||||
if total_frames <= 0 or video_fps <= 0 or fps <= 0:
|
if total_frames <= 0 or video_fps <= 0 or fps <= 0:
|
||||||
return [0] if total_frames > 0 else []
|
return [0] if total_frames > 0 else []
|
||||||
|
|
||||||
@@ -503,12 +532,6 @@ class MiniMaxM3VideoBackend(VideoBackend):
|
|||||||
break
|
break
|
||||||
indices.append(target_frame)
|
indices.append(target_frame)
|
||||||
prev_kept_ts = target_frame / video_fps
|
prev_kept_ts = target_frame / video_fps
|
||||||
|
|
||||||
last_frame_idx = total_frames - 1
|
|
||||||
last_ts = last_frame_idx / video_fps
|
|
||||||
if indices and indices[-1] != last_frame_idx and last_ts - prev_kept_ts > eps:
|
|
||||||
indices.append(last_frame_idx)
|
|
||||||
|
|
||||||
if not indices:
|
if not indices:
|
||||||
indices = [0]
|
indices = [0]
|
||||||
return indices
|
return indices
|
||||||
|
|||||||
Reference in New Issue
Block a user