Add GLM4.1V model (Draft) (#19331)

Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Yuxuan Zhang
2025-07-01 12:48:26 +00:00
committed by GitHub
co-authored by Isotr0py
parent 650d5dbd04
commit ed70f3c64f
17 changed files with 1946 additions and 16 deletions
+5 -3
View File
@@ -57,10 +57,12 @@ which are treated as image embeddings;
these are directly passed to the model without HF processing.
"""
VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor"]
VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor",
tuple[HfVideoItem, dict[str, Any]]]
"""
A `transformers.image_utils.VideoInput` representing a single video
item, which can be passed to a HuggingFace `VideoProcessor`.
A `transformers.video_utils.VideoInput` representing a single video item.
This can be passed to a HuggingFace `VideoProcessor`
with `transformers.video_utils.VideoMetadata`.
Alternatively, a 3-D tensor or batch of 2-D tensors,
which are treated as video embeddings;
+40 -2
View File
@@ -224,8 +224,14 @@ class ImageEmbeddingItems(EmbeddingItems):
class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
def __init__(self, data: Sequence[HfVideoItem]) -> None:
def __init__(
self,
data: Sequence[HfVideoItem],
metadata: Optional[Union[dict[str, Any],
list[Optional[dict[str, Any]]]]] = None,
) -> None:
super().__init__(data, "video")
self.metadata = metadata
def get_num_frames(self, item_idx: int) -> int:
return len(self.get(item_idx))
@@ -320,6 +326,7 @@ class MultiModalDataParser:
*,
target_sr: Optional[float] = None,
audio_resample_method: Literal["librosa", "scipy"] = "librosa",
video_needs_metadata: bool = False,
) -> None:
super().__init__()
@@ -327,6 +334,7 @@ class MultiModalDataParser:
target_sr=target_sr,
method=audio_resample_method,
)
self.video_needs_metadata = video_needs_metadata
def _is_embeddings(
self, data: object
@@ -361,6 +369,21 @@ class MultiModalDataParser:
assert_never(audio)
def _get_video_with_metadata(
self,
video: VideoItem,
) -> tuple[np.ndarray, Optional[dict[str, Any]]]:
if isinstance(video, tuple):
return video
if isinstance(video, list):
return np.array(video), None
if isinstance(video, np.ndarray):
return video, None
if isinstance(video, torch.Tensor):
return video.numpy(), None
assert_never(video)
def _parse_audio_data(
self,
data: ModalityData[AudioItem],
@@ -433,10 +456,25 @@ class MultiModalDataParser:
data_items = [data]
elif isinstance(data, (np.ndarray, torch.Tensor)):
data_items = [elem for elem in data]
elif isinstance(data, tuple) and len(data) == 2:
data_items = [data]
else:
data_items = data
return VideoProcessorItems(data_items)
new_videos = list[tuple[np.ndarray, Optional[dict[str, Any]]]]()
metadata_lst: list[Optional[dict[str, Any]]] = []
for data_item in data_items:
video, metadata = self._get_video_with_metadata(data_item)
if self.video_needs_metadata:
new_videos.append((video, metadata))
metadata_lst.append(metadata)
else:
new_videos.append(video)
if not self.video_needs_metadata:
metadata = None
return VideoProcessorItems(new_videos, metadata=metadata_lst)
def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
return {
+21 -6
View File
@@ -24,6 +24,7 @@ def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:
dtype=frames.dtype)
# lazy import cv2 to avoid bothering users who only use text models
import cv2
for i, frame in enumerate(frames):
resized_frame = cv2.resize(frame, (new_width, new_height))
resized_frames[i] = resized_frame
@@ -92,14 +93,16 @@ class OpenCVVideoBackend(VideoLoader):
continue
if not vr.isBackendBuiltIn(backend):
_, abi, api = vr.getStreamBufferedBackendPluginVersion(backend)
if (abi < 1 or (abi == 1 and api < 2)):
if abi < 1 or (abi == 1 and api < 2):
continue
api_pref = backend
break
return api_pref
@classmethod
def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray:
def load_bytes(cls,
data: bytes,
num_frames: int = -1) -> tuple[npt.NDArray, dict]:
import cv2
backend = cls().get_cv2_video_api()
@@ -108,6 +111,9 @@ class OpenCVVideoBackend(VideoLoader):
raise ValueError("Could not open video stream")
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
original_fps = cap.get(cv2.CAP_PROP_FPS)
duration = total_frames_num / original_fps if original_fps > 0 else 0
full_read = num_frames == -1 or total_frames_num < num_frames
if full_read:
num_frames = total_frames_num
@@ -125,18 +131,27 @@ class OpenCVVideoBackend(VideoLoader):
i = 0
for idx in range(total_frames_num):
ok = cap.grab() # next img
ok = cap.grab()
if not ok:
break
if idx in frame_idx: # only decompress needed
if idx in frame_idx:
ret, frame = cap.retrieve()
if ret:
frames[i] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
i += 1
# we expect all frames loaded
assert i == num_frames, (f"Expected reading {num_frames} frames, "
f"but only loaded {i} frames from video.")
return frames
# Use transformers transformers.video_utils.VideoMetadata format
metadata = {
"total_num_frames": total_frames_num,
"fps": original_fps,
"duration": duration,
"video_backend": "opencv"
}
return frames, metadata
class VideoMediaIO(MediaIO[npt.NDArray]):