Add GLM4.1V model (Draft) (#19331)
Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -57,10 +57,12 @@ which are treated as image embeddings;
|
||||
these are directly passed to the model without HF processing.
|
||||
"""
|
||||
|
||||
VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor"]
|
||||
VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor",
|
||||
tuple[HfVideoItem, dict[str, Any]]]
|
||||
"""
|
||||
A `transformers.image_utils.VideoInput` representing a single video
|
||||
item, which can be passed to a HuggingFace `VideoProcessor`.
|
||||
A `transformers.video_utils.VideoInput` representing a single video item.
|
||||
This can be passed to a HuggingFace `VideoProcessor`
|
||||
with `transformers.video_utils.VideoMetadata`.
|
||||
|
||||
Alternatively, a 3-D tensor or batch of 2-D tensors,
|
||||
which are treated as video embeddings;
|
||||
|
||||
@@ -224,8 +224,14 @@ class ImageEmbeddingItems(EmbeddingItems):
|
||||
|
||||
class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
|
||||
|
||||
def __init__(self, data: Sequence[HfVideoItem]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
data: Sequence[HfVideoItem],
|
||||
metadata: Optional[Union[dict[str, Any],
|
||||
list[Optional[dict[str, Any]]]]] = None,
|
||||
) -> None:
|
||||
super().__init__(data, "video")
|
||||
self.metadata = metadata
|
||||
|
||||
def get_num_frames(self, item_idx: int) -> int:
|
||||
return len(self.get(item_idx))
|
||||
@@ -320,6 +326,7 @@ class MultiModalDataParser:
|
||||
*,
|
||||
target_sr: Optional[float] = None,
|
||||
audio_resample_method: Literal["librosa", "scipy"] = "librosa",
|
||||
video_needs_metadata: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -327,6 +334,7 @@ class MultiModalDataParser:
|
||||
target_sr=target_sr,
|
||||
method=audio_resample_method,
|
||||
)
|
||||
self.video_needs_metadata = video_needs_metadata
|
||||
|
||||
def _is_embeddings(
|
||||
self, data: object
|
||||
@@ -361,6 +369,21 @@ class MultiModalDataParser:
|
||||
|
||||
assert_never(audio)
|
||||
|
||||
def _get_video_with_metadata(
|
||||
self,
|
||||
video: VideoItem,
|
||||
) -> tuple[np.ndarray, Optional[dict[str, Any]]]:
|
||||
if isinstance(video, tuple):
|
||||
return video
|
||||
if isinstance(video, list):
|
||||
return np.array(video), None
|
||||
if isinstance(video, np.ndarray):
|
||||
return video, None
|
||||
if isinstance(video, torch.Tensor):
|
||||
return video.numpy(), None
|
||||
|
||||
assert_never(video)
|
||||
|
||||
def _parse_audio_data(
|
||||
self,
|
||||
data: ModalityData[AudioItem],
|
||||
@@ -433,10 +456,25 @@ class MultiModalDataParser:
|
||||
data_items = [data]
|
||||
elif isinstance(data, (np.ndarray, torch.Tensor)):
|
||||
data_items = [elem for elem in data]
|
||||
elif isinstance(data, tuple) and len(data) == 2:
|
||||
data_items = [data]
|
||||
else:
|
||||
data_items = data
|
||||
|
||||
return VideoProcessorItems(data_items)
|
||||
new_videos = list[tuple[np.ndarray, Optional[dict[str, Any]]]]()
|
||||
metadata_lst: list[Optional[dict[str, Any]]] = []
|
||||
for data_item in data_items:
|
||||
video, metadata = self._get_video_with_metadata(data_item)
|
||||
if self.video_needs_metadata:
|
||||
new_videos.append((video, metadata))
|
||||
metadata_lst.append(metadata)
|
||||
else:
|
||||
new_videos.append(video)
|
||||
|
||||
if not self.video_needs_metadata:
|
||||
metadata = None
|
||||
|
||||
return VideoProcessorItems(new_videos, metadata=metadata_lst)
|
||||
|
||||
def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
|
||||
return {
|
||||
|
||||
@@ -24,6 +24,7 @@ def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
|
||||
dtype=frames.dtype)
|
||||
# lazy import cv2 to avoid bothering users who only use text models
|
||||
import cv2
|
||||
|
||||
for i, frame in enumerate(frames):
|
||||
resized_frame = cv2.resize(frame, (new_width, new_height))
|
||||
resized_frames[i] = resized_frame
|
||||
@@ -92,14 +93,16 @@ class OpenCVVideoBackend(VideoLoader):
|
||||
continue
|
||||
if not vr.isBackendBuiltIn(backend):
|
||||
_, abi, api = vr.getStreamBufferedBackendPluginVersion(backend)
|
||||
if (abi < 1 or (abi == 1 and api < 2)):
|
||||
if abi < 1 or (abi == 1 and api < 2):
|
||||
continue
|
||||
api_pref = backend
|
||||
break
|
||||
return api_pref
|
||||
|
||||
@classmethod
|
||||
def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:
|
||||
def load_bytes(cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1) -> tuple[npt.NDArray, dict]:
|
||||
import cv2
|
||||
|
||||
backend = cls().get_cv2_video_api()
|
||||
@@ -108,6 +111,9 @@ class OpenCVVideoBackend(VideoLoader):
|
||||
raise ValueError("Could not open video stream")
|
||||
|
||||
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
original_fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
duration = total_frames_num / original_fps if original_fps > 0 else 0
|
||||
|
||||
full_read = num_frames == -1 or total_frames_num < num_frames
|
||||
if full_read:
|
||||
num_frames = total_frames_num
|
||||
@@ -125,18 +131,27 @@ class OpenCVVideoBackend(VideoLoader):
|
||||
|
||||
i = 0
|
||||
for idx in range(total_frames_num):
|
||||
ok = cap.grab() # next img
|
||||
ok = cap.grab()
|
||||
if not ok:
|
||||
break
|
||||
if idx in frame_idx: # only decompress needed
|
||||
if idx in frame_idx:
|
||||
ret, frame = cap.retrieve()
|
||||
if ret:
|
||||
frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
i += 1
|
||||
# we expect all frames loaded
|
||||
|
||||
assert i == num_frames, (f"Expected reading {num_frames} frames, "
|
||||
f"but only loaded {i} frames from video.")
|
||||
return frames
|
||||
|
||||
# Use transformers transformers.video_utils.VideoMetadata format
|
||||
metadata = {
|
||||
"total_num_frames": total_frames_num,
|
||||
"fps": original_fps,
|
||||
"duration": duration,
|
||||
"video_backend": "opencv"
|
||||
}
|
||||
|
||||
return frames, metadata
|
||||
|
||||
|
||||
class VideoMediaIO(MediaIO[npt.NDArray]):
|
||||
|
||||
Reference in New Issue
Block a user