[feat] add GlmgaProcessor specific logits in glm4_1v.py (#43575)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
parent
7bd45da585
commit
212deff2ec
@@ -935,7 +935,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
hf_overrides={"architectures": ["GLM4VForCausalLM"]},
|
hf_overrides={"architectures": ["GLM4VForCausalLM"]},
|
||||||
),
|
),
|
||||||
"Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
|
"Glm4vForConditionalGeneration": _HfExamplesInfo(
|
||||||
|
"zai-org/GLM-4.1V-9B-Thinking",
|
||||||
|
extras={"4.6V": "zai-org/GLM-4.6V-Flash"},
|
||||||
|
),
|
||||||
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
|
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
|
||||||
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
|
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
|
||||||
"zai-org/GLM-OCR",
|
"zai-org/GLM-OCR",
|
||||||
|
|||||||
@@ -36,7 +36,9 @@ import numpy as np
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
|
import transformers
|
||||||
from einops import rearrange
|
from einops import rearrange
|
||||||
|
from packaging.version import Version
|
||||||
from transformers import BatchFeature, Glm4vProcessor
|
from transformers import BatchFeature, Glm4vProcessor
|
||||||
from transformers.models.glm4v.configuration_glm4v import (
|
from transformers.models.glm4v.configuration_glm4v import (
|
||||||
Glm4vTextConfig,
|
Glm4vTextConfig,
|
||||||
@@ -122,6 +124,15 @@ logger = init_logger(__name__)
|
|||||||
# For profile run
|
# For profile run
|
||||||
_MAX_FRAMES_PER_VIDEO = 600
|
_MAX_FRAMES_PER_VIDEO = 600
|
||||||
|
|
||||||
|
TRANSFORMERS_WITH_GA = Version(transformers.__version__) >= Version("5.10.0.dev0")
|
||||||
|
|
||||||
|
|
||||||
|
def _to_video_metadata(metadata: Mapping[str, Any]) -> VideoMetadata:
|
||||||
|
return VideoMetadata(
|
||||||
|
**{k: metadata[k] for k in metadata if k != "do_sample_frames"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# === Vision Inputs === #
|
# === Vision Inputs === #
|
||||||
|
|
||||||
|
|
||||||
@@ -832,6 +843,46 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
|||||||
def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor:
|
def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor:
|
||||||
return self.get_hf_processor(**kwargs).video_processor
|
return self.get_hf_processor(**kwargs).video_processor
|
||||||
|
|
||||||
|
def get_mm_max_tokens_per_item(
|
||||||
|
self,
|
||||||
|
seq_len: int,
|
||||||
|
mm_counts: Mapping[str, int],
|
||||||
|
) -> Mapping[str, int] | None:
|
||||||
|
processor = self.get_hf_processor()
|
||||||
|
if isinstance(processor, Glm4vProcessor):
|
||||||
|
return None
|
||||||
|
|
||||||
|
result: dict[str, int] = {}
|
||||||
|
|
||||||
|
if mm_counts.get("image", 0) > 0:
|
||||||
|
result["image"] = self.get_max_image_tokens()
|
||||||
|
|
||||||
|
if mm_counts.get("video", 0) > 0:
|
||||||
|
video_processor = self.get_video_processor()
|
||||||
|
max_pixels = video_processor.size["longest_edge"]
|
||||||
|
|
||||||
|
vision_config = self.get_hf_config().vision_config
|
||||||
|
temporal_patch_size = vision_config.temporal_patch_size
|
||||||
|
patch_size = vision_config.patch_size
|
||||||
|
merge_size = vision_config.spatial_merge_size
|
||||||
|
|
||||||
|
max_vision_tokens = max_pixels // (
|
||||||
|
temporal_patch_size * patch_size**2 * merge_size**2
|
||||||
|
)
|
||||||
|
|
||||||
|
# GLMGA supports up to 640 frames (max_frames).
|
||||||
|
max_grid_t = 640 // temporal_patch_size
|
||||||
|
|
||||||
|
tokenizer = self.get_tokenizer()
|
||||||
|
max_ts_tokens = max(
|
||||||
|
len(tokenizer.encode(f"{t:.1f} seconds", add_special_tokens=False))
|
||||||
|
for t in range(min(max_grid_t, 300))
|
||||||
|
)
|
||||||
|
|
||||||
|
result["video"] = max_vision_tokens + max_grid_t * (2 + max_ts_tokens) + 2
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
def get_data_parser(self):
|
def get_data_parser(self):
|
||||||
return MultiModalDataParser(
|
return MultiModalDataParser(
|
||||||
video_needs_metadata=True,
|
video_needs_metadata=True,
|
||||||
@@ -1104,12 +1155,83 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
|||||||
selected_timestamps.append(timestamps_list[idx])
|
selected_timestamps.append(timestamps_list[idx])
|
||||||
return selected_timestamps
|
return selected_timestamps
|
||||||
|
|
||||||
|
def _is_glmga_model(self, processor: object) -> bool:
|
||||||
|
"""Detect GLMGA variant via its Glmga sub-processors."""
|
||||||
|
for attr in ("image_processor", "video_processor"):
|
||||||
|
sub = getattr(processor, attr, None)
|
||||||
|
if sub and "Glmga" in type(sub).__name__:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _get_video_second_idx_glmga(
|
||||||
|
self, metadata: dict[str, Any], total_frames: int
|
||||||
|
) -> list[int]:
|
||||||
|
"""Fixed fps=2 frame selection matching GlmgaVideoProcessor.sample_frames."""
|
||||||
|
video_processor = self.get_video_processor()
|
||||||
|
|
||||||
|
video_fps = metadata["fps"]
|
||||||
|
meta_frames = metadata.get("total_num_frames", total_frames)
|
||||||
|
max_frame_idx = meta_frames - 1
|
||||||
|
duration = metadata.get("duration", round(max_frame_idx / video_fps) + 1)
|
||||||
|
|
||||||
|
do_sample_frames = metadata.get("do_sample_frames", True)
|
||||||
|
if not do_sample_frames:
|
||||||
|
frame_indices = metadata["frames_indices"]
|
||||||
|
else:
|
||||||
|
target_fps = 2
|
||||||
|
max_frames = getattr(video_processor, "max_frames", 640)
|
||||||
|
extract_t = int(duration * target_fps)
|
||||||
|
extract_t = min(extract_t, max_frames)
|
||||||
|
|
||||||
|
duration_per_frame = 1 / video_fps
|
||||||
|
timestamps = [i * duration_per_frame for i in range(meta_frames)]
|
||||||
|
|
||||||
|
if meta_frames < extract_t:
|
||||||
|
frame_indices = [
|
||||||
|
math.floor(i * meta_frames / extract_t) for i in range(extract_t)
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
frame_indices = []
|
||||||
|
current_second = 0.0
|
||||||
|
inv_fps = 1 / target_fps
|
||||||
|
for frame_index in range(meta_frames):
|
||||||
|
if timestamps[frame_index] >= current_second:
|
||||||
|
current_second += inv_fps
|
||||||
|
frame_indices.append(frame_index)
|
||||||
|
if current_second >= duration - inv_fps:
|
||||||
|
break
|
||||||
|
|
||||||
|
if len(frame_indices) < extract_t:
|
||||||
|
if len(frame_indices) == 0:
|
||||||
|
start, end = 0, max(meta_frames - 1, 0)
|
||||||
|
else:
|
||||||
|
start, end = frame_indices[0], frame_indices[-1]
|
||||||
|
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
|
||||||
|
elif len(frame_indices) > extract_t:
|
||||||
|
frame_indices = np.linspace(
|
||||||
|
0, meta_frames - 1, extract_t, dtype=int
|
||||||
|
).tolist()
|
||||||
|
|
||||||
|
seen, uniq = set(), []
|
||||||
|
for idx in frame_indices:
|
||||||
|
if idx not in seen:
|
||||||
|
seen.add(idx)
|
||||||
|
uniq.append(idx)
|
||||||
|
|
||||||
|
if len(uniq) & 1:
|
||||||
|
uniq.append(uniq[-1])
|
||||||
|
|
||||||
|
frame_indices = uniq
|
||||||
|
full_second_idxs = [int(idx / video_fps) for idx in frame_indices]
|
||||||
|
timestamps_list = full_second_idxs[::2]
|
||||||
|
return list(timestamps_list)
|
||||||
|
|
||||||
def _construct_video_placeholder(
|
def _construct_video_placeholder(
|
||||||
self,
|
self,
|
||||||
video_array: np.ndarray,
|
video_array: np.ndarray,
|
||||||
metadata: dict[str, Any],
|
metadata: dict[str, Any],
|
||||||
grid_thw: torch.Tensor,
|
grid_thw: torch.Tensor,
|
||||||
) -> str:
|
) -> list[int]:
|
||||||
hf_processor = self.get_hf_processor()
|
hf_processor = self.get_hf_processor()
|
||||||
tokenizer = self.get_tokenizer()
|
tokenizer = self.get_tokenizer()
|
||||||
image_processor = hf_processor.image_processor
|
image_processor = hf_processor.image_processor
|
||||||
@@ -1122,11 +1244,13 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
|||||||
merge_length = image_processor.merge_size**2
|
merge_length = image_processor.merge_size**2
|
||||||
|
|
||||||
assert isinstance(grid_thw, torch.Tensor)
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
timestamps = (
|
|
||||||
self._get_video_second_idx_glm4v(metadata, len(video_array))
|
if isinstance(hf_processor, Glm4vProcessor):
|
||||||
if isinstance(hf_processor, Glm4vProcessor)
|
timestamps = self._get_video_second_idx_glm4v(metadata, len(video_array))
|
||||||
else self._get_video_second_idx_glm46v(metadata, len(video_array))
|
elif self._is_glmga_model(hf_processor):
|
||||||
)
|
timestamps = self._get_video_second_idx_glmga(metadata, len(video_array))
|
||||||
|
else:
|
||||||
|
timestamps = self._get_video_second_idx_glm46v(metadata, len(video_array))
|
||||||
|
|
||||||
timestamp_format = (
|
timestamp_format = (
|
||||||
"{}" if isinstance(hf_processor, Glm4vProcessor) else "{:.1f} seconds"
|
"{}" if isinstance(hf_processor, Glm4vProcessor) else "{:.1f} seconds"
|
||||||
@@ -1139,9 +1263,16 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
|||||||
num_tokens_per_frame = int(H * W) // merge_length
|
num_tokens_per_frame = int(H * W) // merge_length
|
||||||
placeholder = []
|
placeholder = []
|
||||||
placeholder.append(bov_token_id)
|
placeholder.append(bov_token_id)
|
||||||
|
# Glm46VProcessor uses image_token_id for video frame embeddings;
|
||||||
|
# Glm4vProcessor uses video_token_id.
|
||||||
|
frame_embed_token_id = (
|
||||||
|
hf_processor.video_token_id
|
||||||
|
if isinstance(hf_processor, Glm4vProcessor) or not TRANSFORMERS_WITH_GA
|
||||||
|
else hf_processor.image_token_id
|
||||||
|
)
|
||||||
for frame_idx in frames_idx_token:
|
for frame_idx in frames_idx_token:
|
||||||
placeholder.append(boi_token_id)
|
placeholder.append(boi_token_id)
|
||||||
placeholder.extend([hf_processor.video_token_id] * num_tokens_per_frame)
|
placeholder.extend([frame_embed_token_id] * num_tokens_per_frame)
|
||||||
placeholder.append(eoi_token_id)
|
placeholder.append(eoi_token_id)
|
||||||
placeholder.extend(frame_idx)
|
placeholder.extend(frame_idx)
|
||||||
placeholder.append(eov_token_id)
|
placeholder.append(eov_token_id)
|
||||||
@@ -1241,6 +1372,47 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
|
|||||||
|
|
||||||
|
|
||||||
class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||||
|
@staticmethod
|
||||||
|
def _get_direct_path_inputs(
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
mm_kwargs: Mapping[str, object],
|
||||||
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
|
prepared_data = dict(mm_data)
|
||||||
|
prepared_kwargs = dict(mm_kwargs)
|
||||||
|
|
||||||
|
videos = prepared_data.get("videos")
|
||||||
|
if not (isinstance(videos, list) and len(videos) > 0):
|
||||||
|
return prepared_data, prepared_kwargs
|
||||||
|
|
||||||
|
hf_videos = []
|
||||||
|
hf_video_metadata = []
|
||||||
|
for item in videos:
|
||||||
|
if isinstance(item, tuple) and len(item) == 2:
|
||||||
|
video_array, metadata = item
|
||||||
|
hf_videos.append(video_array)
|
||||||
|
if isinstance(metadata, VideoMetadata):
|
||||||
|
hf_video_metadata.append(metadata)
|
||||||
|
elif isinstance(metadata, Mapping):
|
||||||
|
hf_video_metadata.append(_to_video_metadata(metadata))
|
||||||
|
if "do_sample_frames" in metadata:
|
||||||
|
prepared_kwargs["do_sample_frames"] = metadata[
|
||||||
|
"do_sample_frames"
|
||||||
|
]
|
||||||
|
elif metadata is not None:
|
||||||
|
raise TypeError(
|
||||||
|
"Video metadata must be a mapping or VideoMetadata, "
|
||||||
|
f"got {type(metadata)}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
hf_videos.append(item)
|
||||||
|
|
||||||
|
prepared_data["videos"] = hf_videos
|
||||||
|
if hf_video_metadata:
|
||||||
|
prepared_data["video_metadata"] = hf_video_metadata
|
||||||
|
prepared_kwargs["return_metadata"] = True
|
||||||
|
|
||||||
|
return prepared_data, prepared_kwargs
|
||||||
|
|
||||||
def _call_hf_processor(
|
def _call_hf_processor(
|
||||||
self,
|
self,
|
||||||
prompt: str,
|
prompt: str,
|
||||||
@@ -1249,11 +1421,32 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
tok_kwargs: Mapping[str, object],
|
tok_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
|
if not mm_data:
|
||||||
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||||
|
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||||
|
|
||||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||||
|
|
||||||
# GLM-4.1V use `image_token_id` as video placeholder, we need to
|
# Glm46VProcessor and GLMGA handle image/video placeholders together
|
||||||
# replace it with `video_token_id` for video processing. So we
|
# via the direct path. Only Glm4vProcessor (GLM-4.1V) needs the
|
||||||
# separate video processing from image processing.
|
# split-video path because it uses image_token_id as the video
|
||||||
|
# placeholder. The direct path requires transformers >= 5.5.0
|
||||||
|
# (Glm46VProcessor / GlmgaVideoProcessor support).
|
||||||
|
use_direct_path = (
|
||||||
|
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
|
||||||
|
)
|
||||||
|
if use_direct_path:
|
||||||
|
prepared_data, prepared_kwargs = self._get_direct_path_inputs(
|
||||||
|
mm_data, mm_kwargs
|
||||||
|
)
|
||||||
|
return super()._call_hf_processor(
|
||||||
|
prompt=prompt,
|
||||||
|
mm_data=prepared_data,
|
||||||
|
mm_kwargs=prepared_kwargs,
|
||||||
|
tok_kwargs=tok_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
"videos" in mm_data
|
"videos" in mm_data
|
||||||
and isinstance(mm_data["videos"], list)
|
and isinstance(mm_data["videos"], list)
|
||||||
@@ -1272,19 +1465,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
|
|
||||||
video_mm_data = dict()
|
video_mm_data = dict()
|
||||||
video_mm_data["videos"] = [[video_array]]
|
video_mm_data["videos"] = [[video_array]]
|
||||||
|
video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]
|
||||||
unuse_metadata = ["do_sample_frames"]
|
|
||||||
video_mm_data["video_metadata"] = [
|
|
||||||
[
|
|
||||||
VideoMetadata(
|
|
||||||
**{
|
|
||||||
k: metadata[k]
|
|
||||||
for k in metadata
|
|
||||||
if k not in unuse_metadata
|
|
||||||
}
|
|
||||||
)
|
|
||||||
]
|
|
||||||
]
|
|
||||||
|
|
||||||
video_outputs = super()._call_hf_processor(
|
video_outputs = super()._call_hf_processor(
|
||||||
prompt="<|begin_of_video|><|video|><|end_of_video|>",
|
prompt="<|begin_of_video|><|video|><|end_of_video|>",
|
||||||
@@ -1366,6 +1547,22 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
embed_token_id=hf_processor.video_token_id,
|
embed_token_id=hf_processor.video_token_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def get_video_replacement_glm46v(item_idx: int):
|
||||||
|
out_item = out_mm_kwargs["video"][item_idx]
|
||||||
|
grid_thw = out_item["video_grid_thw"].data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
|
|
||||||
|
video, metadata = mm_items["video"][item_idx]
|
||||||
|
placeholder = self.info._construct_video_placeholder(
|
||||||
|
video, metadata, grid_thw
|
||||||
|
)
|
||||||
|
return PromptUpdateDetails.select_token_id(
|
||||||
|
placeholder,
|
||||||
|
embed_token_id=hf_processor.image_token_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
is_glm46v = not isinstance(hf_processor, Glm4vProcessor)
|
||||||
|
|
||||||
return [
|
return [
|
||||||
PromptReplacement(
|
PromptReplacement(
|
||||||
modality="image",
|
modality="image",
|
||||||
@@ -1375,7 +1572,11 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
|||||||
PromptReplacement(
|
PromptReplacement(
|
||||||
modality="video",
|
modality="video",
|
||||||
target="<|begin_of_video|><|video|><|end_of_video|>",
|
target="<|begin_of_video|><|video|><|end_of_video|>",
|
||||||
replacement=get_video_replacement_glm4v,
|
replacement=(
|
||||||
|
get_video_replacement_glm46v
|
||||||
|
if is_glm46v and TRANSFORMERS_WITH_GA
|
||||||
|
else get_video_replacement_glm4v
|
||||||
|
),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1436,7 +1637,7 @@ class Glm4vForConditionalGeneration(
|
|||||||
prefix=maybe_prefix(prefix, "visual"),
|
prefix=maybe_prefix(prefix, "visual"),
|
||||||
)
|
)
|
||||||
|
|
||||||
if config.model_type in ("glm4v", "glm_ocr"):
|
if config.model_type in ("glm4v", "glm_ocr", "glmga"):
|
||||||
architectures = ["Glm4ForCausalLM"]
|
architectures = ["Glm4ForCausalLM"]
|
||||||
elif config.model_type == "glm4v_moe":
|
elif config.model_type == "glm4v_moe":
|
||||||
architectures = ["Glm4MoeForCausalLM"]
|
architectures = ["Glm4MoeForCausalLM"]
|
||||||
@@ -1600,19 +1801,27 @@ class Glm4vForConditionalGeneration(
|
|||||||
hf_config = self.config
|
hf_config = self.config
|
||||||
spatial_merge_size = hf_config.vision_config.spatial_merge_size
|
spatial_merge_size = hf_config.vision_config.spatial_merge_size
|
||||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||||
offset = mm_feature.mm_position.offset
|
embed_ranges = mm_feature.mm_position.extract_embeds_range()
|
||||||
if mm_feature.modality == "image":
|
if mm_feature.modality == "image":
|
||||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||||
|
assert len(embed_ranges) == 1
|
||||||
|
offset, end = embed_ranges[0]
|
||||||
|
assert end - offset + 1 == h * w // spatial_merge_size**2
|
||||||
yield offset, t, h // spatial_merge_size, w // spatial_merge_size
|
yield offset, t, h // spatial_merge_size, w // spatial_merge_size
|
||||||
elif mm_feature.modality == "video":
|
elif mm_feature.modality == "video":
|
||||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||||
yield (
|
llm_grid_h = h // spatial_merge_size
|
||||||
offset,
|
llm_grid_w = w // spatial_merge_size
|
||||||
t,
|
num_tokens_per_frame = llm_grid_h * llm_grid_w
|
||||||
h // spatial_merge_size,
|
|
||||||
w // spatial_merge_size,
|
if len(embed_ranges) == t:
|
||||||
)
|
for offset, end in embed_ranges:
|
||||||
|
assert end - offset + 1 == num_tokens_per_frame
|
||||||
|
yield offset, 1, llm_grid_h, llm_grid_w
|
||||||
|
else:
|
||||||
|
offset = mm_feature.mm_position.offset
|
||||||
|
yield offset, t, llm_grid_h, llm_grid_w
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unsupported modality: {mm_feature.modality}")
|
raise ValueError(f"Unsupported modality: {mm_feature.modality}")
|
||||||
|
|
||||||
|
|||||||
@@ -639,6 +639,107 @@ class DynamicVideoBackend(VideoBackend):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@VIDEO_LOADER_REGISTRY.register("glm4_6v")
|
||||||
|
class GLM4_6VVideoBackend(VideoBackend):
|
||||||
|
@classmethod
|
||||||
|
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
|
||||||
|
# Estimate duration from frame count and fps when the container
|
||||||
|
# does not report it (common for WebM/streaming inputs).
|
||||||
|
if source.duration:
|
||||||
|
return source
|
||||||
|
if source.original_fps > 0:
|
||||||
|
max_frame_idx = source.total_frames_num - 1
|
||||||
|
duration = round(max_frame_idx / source.original_fps) + 1
|
||||||
|
else:
|
||||||
|
duration = 0
|
||||||
|
return VideoSourceMetadata(
|
||||||
|
source.total_frames_num, source.original_fps, duration
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def compute_frames_index_to_sample(
|
||||||
|
cls,
|
||||||
|
source: VideoSourceMetadata,
|
||||||
|
target: VideoTargetMetadata,
|
||||||
|
**kwargs,
|
||||||
|
) -> list[int]:
|
||||||
|
total_frames_num = source.total_frames_num
|
||||||
|
duration = source.duration
|
||||||
|
original_fps = source.original_fps
|
||||||
|
target_fps = target.fps
|
||||||
|
max_frame_idx = source.total_frames_num - 1
|
||||||
|
max_frames = kwargs.get("max_frames", 640)
|
||||||
|
|
||||||
|
duration = duration or round(max_frame_idx / original_fps) + 1
|
||||||
|
|
||||||
|
extract_t = int(duration * target_fps)
|
||||||
|
extract_t = min(extract_t, max_frames)
|
||||||
|
|
||||||
|
duration_per_frame = 1 / original_fps
|
||||||
|
timestamps = [i * duration_per_frame for i in range(total_frames_num)]
|
||||||
|
|
||||||
|
if total_frames_num < extract_t:
|
||||||
|
frame_indices = [
|
||||||
|
math.floor(i * total_frames_num / extract_t) for i in range(extract_t)
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
frame_indices = []
|
||||||
|
current_second = 0.0
|
||||||
|
inv_fps = 1 / target_fps
|
||||||
|
for frame_index in range(total_frames_num):
|
||||||
|
if timestamps[frame_index] >= current_second:
|
||||||
|
current_second += inv_fps
|
||||||
|
frame_indices.append(frame_index)
|
||||||
|
if current_second >= duration - inv_fps:
|
||||||
|
break
|
||||||
|
|
||||||
|
if len(frame_indices) < extract_t:
|
||||||
|
if len(frame_indices) == 0:
|
||||||
|
start, end = 0, max(total_frames_num - 1, 0)
|
||||||
|
else:
|
||||||
|
start, end = frame_indices[0], frame_indices[-1]
|
||||||
|
frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
|
||||||
|
elif len(frame_indices) > extract_t:
|
||||||
|
frame_indices = np.linspace(
|
||||||
|
0, total_frames_num - 1, extract_t, dtype=int
|
||||||
|
).tolist()
|
||||||
|
|
||||||
|
seen, uniq = set(), []
|
||||||
|
for idx in frame_indices:
|
||||||
|
if idx not in seen:
|
||||||
|
seen.add(idx)
|
||||||
|
uniq.append(idx)
|
||||||
|
|
||||||
|
return uniq
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load_bytes(
|
||||||
|
cls,
|
||||||
|
data: bytes,
|
||||||
|
num_frames: int = -1,
|
||||||
|
fps: int = 2,
|
||||||
|
max_duration: int = 300,
|
||||||
|
frame_recovery: bool = False,
|
||||||
|
*,
|
||||||
|
backend: Literal["opencv", "pyav"] = "opencv",
|
||||||
|
**kwargs,
|
||||||
|
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||||
|
frames, metadata = super().load_bytes(
|
||||||
|
data,
|
||||||
|
num_frames=num_frames,
|
||||||
|
fps=fps,
|
||||||
|
max_duration=max_duration,
|
||||||
|
frame_recovery=frame_recovery,
|
||||||
|
backend=backend,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
# Ensure even frame count — matches HF's sample_frames even-padding
|
||||||
|
# and _preprocess temporal_patch_size divisibility check.
|
||||||
|
if frames.shape[0] & 1:
|
||||||
|
frames = np.concatenate([frames, frames[-1:]], axis=0)
|
||||||
|
return frames, metadata
|
||||||
|
|
||||||
|
|
||||||
@VIDEO_LOADER_REGISTRY.register("molmo2")
|
@VIDEO_LOADER_REGISTRY.register("molmo2")
|
||||||
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
|
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
Reference in New Issue
Block a user