Support temporal compression for Nemotron-3-VL videos (#36808)
Signed-off-by: Collin McCarthy <cmccarthy@nvidia.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import math
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import einops
|
||||
@@ -43,6 +44,12 @@ AUDIO_CONTEXT = "<so_embedding>"
|
||||
# MAX_FRAMES = 16
|
||||
DEFAULT_NUM_TILES = 12
|
||||
|
||||
# Configure PIL to handle large images without warnings
|
||||
# This prevents DecompressionBombWarning for legitimate large images
|
||||
Image.MAX_IMAGE_PIXELS = None # Disable the limit entirely
|
||||
# Alternative: Set a specific higher limit
|
||||
# Image.MAX_IMAGE_PIXELS = 300000000 # ~300M pixels
|
||||
|
||||
|
||||
def calculate_timestamps(
|
||||
indices: list[int] | torch.Tensor,
|
||||
@@ -138,19 +145,110 @@ def image_to_pixel_values(
|
||||
return pixel_values
|
||||
|
||||
|
||||
def _compute_aspect_preserving_size(
|
||||
orig_w: int,
|
||||
orig_h: int,
|
||||
target_num_patches: int,
|
||||
patch_size: int,
|
||||
downsample_ratio: float,
|
||||
) -> tuple[int, int]:
|
||||
"""Compute target pixel dimensions that preserve aspect ratio.
|
||||
|
||||
Mirrors Megatron-LM image_processing.py video frame resizing:
|
||||
target area in patch-grid space is *target_num_patches*, distributed
|
||||
according to the source aspect ratio, then snapped to a multiple of
|
||||
the required divisor (2 for pixel-shuffle).
|
||||
"""
|
||||
aspect_wh = orig_w / max(orig_h, 1)
|
||||
ph = round(math.sqrt(target_num_patches / aspect_wh))
|
||||
pw = round(math.sqrt(target_num_patches * aspect_wh))
|
||||
ph = max(ph, 1)
|
||||
pw = max(pw, 1)
|
||||
|
||||
reduction_factor = int(round(1 / downsample_ratio))
|
||||
required_divisor = reduction_factor # 2 for pixel-shuffle
|
||||
if required_divisor > 1:
|
||||
rem_h = ph % required_divisor
|
||||
rem_w = pw % required_divisor
|
||||
ph_up = ph + (required_divisor - rem_h if rem_h else 0)
|
||||
ph_down = ph - rem_h
|
||||
pw_up = pw + (required_divisor - rem_w if rem_w else 0)
|
||||
pw_down = pw - rem_w
|
||||
if ph_up * pw_up <= target_num_patches:
|
||||
ph, pw = ph_up, pw_up
|
||||
else:
|
||||
ph = max(required_divisor, ph_down)
|
||||
pw = max(required_divisor, pw_down)
|
||||
|
||||
return pw * patch_size, ph * patch_size # (width, height) in pixels
|
||||
|
||||
|
||||
def get_video_target_size_and_feature_size(
|
||||
orig_w: int,
|
||||
orig_h: int,
|
||||
target_patches: int,
|
||||
maintain_aspect_ratio: bool,
|
||||
patch_size: int,
|
||||
downsample_ratio: float,
|
||||
) -> tuple[int, int, int]:
|
||||
"""Compute target (width, height) and feature_size for video resize and token count.
|
||||
|
||||
Used by video_to_pixel_values (resize) and get_video_replacement_internvl
|
||||
(seq length calc) so both use the same dimensions.
|
||||
"""
|
||||
if maintain_aspect_ratio:
|
||||
target_w, target_h = _compute_aspect_preserving_size(
|
||||
orig_w=orig_w,
|
||||
orig_h=orig_h,
|
||||
target_num_patches=target_patches,
|
||||
patch_size=patch_size,
|
||||
downsample_ratio=downsample_ratio,
|
||||
)
|
||||
else:
|
||||
reduction_factor = int(round(1 / downsample_ratio))
|
||||
side = int(math.sqrt(target_patches))
|
||||
side = max(reduction_factor, (side // reduction_factor) * reduction_factor)
|
||||
target_w = side * patch_size
|
||||
target_h = side * patch_size
|
||||
|
||||
feature_size = int((target_h // patch_size) * downsample_ratio) * int(
|
||||
(target_w // patch_size) * downsample_ratio
|
||||
)
|
||||
return target_w, target_h, feature_size
|
||||
|
||||
|
||||
def video_to_pixel_values(
|
||||
video: npt.NDArray,
|
||||
*,
|
||||
input_size: int,
|
||||
max_num_tiles: int = 1,
|
||||
use_thumbnail: bool,
|
||||
video_target_num_patches: int | None = None,
|
||||
video_maintain_aspect_ratio: bool = False,
|
||||
patch_size: int = 16,
|
||||
downsample_ratio: float = 0.5,
|
||||
) -> torch.Tensor:
|
||||
assert max_num_tiles == 1, "Video modality always uses one tile"
|
||||
|
||||
# (num_frames, H, W, C) -> (num_frames, C, H, W)
|
||||
video_tensor = torch.from_numpy(video).permute(0, 3, 1, 2)
|
||||
|
||||
if video_tensor.shape[2] != input_size or video_tensor.shape[3] != input_size:
|
||||
if video_target_num_patches is not None:
|
||||
# Resize to target patch count (aspect-preserving or square).
|
||||
orig_h, orig_w = video_tensor.shape[2], video_tensor.shape[3]
|
||||
target_w, target_h, _ = get_video_target_size_and_feature_size(
|
||||
orig_w=orig_w,
|
||||
orig_h=orig_h,
|
||||
target_patches=video_target_num_patches,
|
||||
maintain_aspect_ratio=video_maintain_aspect_ratio,
|
||||
patch_size=patch_size,
|
||||
downsample_ratio=downsample_ratio,
|
||||
)
|
||||
if video_tensor.shape[2] != target_h or video_tensor.shape[3] != target_w:
|
||||
video_tensor = torch.nn.functional.interpolate(
|
||||
video_tensor,
|
||||
size=(target_h, target_w),
|
||||
mode="bicubic",
|
||||
align_corners=False,
|
||||
antialias=True,
|
||||
)
|
||||
elif video_tensor.shape[2] != input_size or video_tensor.shape[3] != input_size:
|
||||
video_tensor = torch.nn.functional.interpolate(
|
||||
video_tensor,
|
||||
size=(input_size, input_size),
|
||||
@@ -645,9 +743,9 @@ class BaseNanoNemotronVLProcessor(ABC):
|
||||
"which should be a single string"
|
||||
)
|
||||
parts = [x for x in re.split(r"(<image>)", text[0]) if x]
|
||||
assert parts.count("<image>") == len(pixel_values_lst), (
|
||||
"the number of <image> tokens in the text should be the "
|
||||
"same as the number of images"
|
||||
assert parts.count("<image>") == len(num_tokens_per_image), (
|
||||
f"Expected {len(num_tokens_per_image)} <image> tokens in text "
|
||||
f"but found {parts.count('<image>')}"
|
||||
)
|
||||
|
||||
for i, (feature_size, num_patches) in enumerate(
|
||||
@@ -706,6 +804,33 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
self.video_token = video_token
|
||||
self.video_pruning_rate = video_pruning_rate
|
||||
|
||||
# Video params live exclusively in vision_config
|
||||
vision_config = getattr(config, "vision_config", config)
|
||||
self.video_temporal_patch_size: int = getattr(
|
||||
vision_config, "video_temporal_patch_size", 1
|
||||
)
|
||||
self.video_maintain_aspect_ratio: bool = getattr(
|
||||
vision_config, "video_maintain_aspect_ratio", False
|
||||
)
|
||||
|
||||
# Resolve video frame target size: exactly one of video_target_num_patches
|
||||
# or video_target_img_size may be set (mirrors Megatron's
|
||||
# DynamicResolutionImageTilingStrategy validation).
|
||||
target_num_patches = getattr(vision_config, "video_target_num_patches", None)
|
||||
target_img_size = getattr(vision_config, "video_target_img_size", None)
|
||||
if target_num_patches is not None and target_img_size is not None:
|
||||
raise ValueError(
|
||||
"Exactly one of video_target_num_patches or "
|
||||
"video_target_img_size must be set, got both"
|
||||
)
|
||||
if target_num_patches is not None:
|
||||
self.video_target_num_patches: int | None = target_num_patches
|
||||
elif target_img_size is not None:
|
||||
base_patches = math.ceil(target_img_size / config.patch_size)
|
||||
self.video_target_num_patches = base_patches * base_patches
|
||||
else:
|
||||
self.video_target_num_patches = None
|
||||
|
||||
self.audio_extractor: ParakeetExtractor | None = None
|
||||
raw_sound_config = getattr(config, "sound_config", None)
|
||||
if raw_sound_config is not None:
|
||||
@@ -721,6 +846,27 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
IMG_CONTEXT, add_special_tokens=False
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def num_video_token(self) -> int:
|
||||
"""Token count per video frame, accounting for video_target_num_patches.
|
||||
|
||||
When video_target_num_patches is set the per-frame feature count
|
||||
differs from the image-based num_image_token. We use a square
|
||||
dummy (1:1) to compute the feature_size because the dummy video is
|
||||
square and the user confirmed that is acceptable.
|
||||
"""
|
||||
if self.video_target_num_patches is not None:
|
||||
_, _, feature_size = get_video_target_size_and_feature_size(
|
||||
orig_w=self.image_size,
|
||||
orig_h=self.image_size,
|
||||
target_patches=self.video_target_num_patches,
|
||||
maintain_aspect_ratio=self.video_maintain_aspect_ratio,
|
||||
patch_size=self.config.patch_size,
|
||||
downsample_ratio=self.config.downsample_ratio,
|
||||
)
|
||||
return feature_size
|
||||
return self.num_image_token
|
||||
|
||||
@property
|
||||
def supports_video(self) -> bool:
|
||||
return self.video_token_id is not None
|
||||
@@ -738,14 +884,15 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
def _videos_to_pixel_values_lst(
|
||||
self,
|
||||
videos: list[npt.NDArray],
|
||||
max_num_tiles: int,
|
||||
) -> list[torch.Tensor]:
|
||||
return [
|
||||
video_to_pixel_values(
|
||||
video,
|
||||
input_size=self.image_size,
|
||||
max_num_tiles=max_num_tiles,
|
||||
use_thumbnail=self.use_thumbnail,
|
||||
video_target_num_patches=self.video_target_num_patches,
|
||||
video_maintain_aspect_ratio=self.video_maintain_aspect_ratio,
|
||||
patch_size=self.config.patch_size,
|
||||
downsample_ratio=self.config.downsample_ratio,
|
||||
)
|
||||
for video in videos
|
||||
]
|
||||
@@ -754,7 +901,6 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
self,
|
||||
text: list[str],
|
||||
videos: list[tuple[npt.NDArray, dict[str, Any]]],
|
||||
max_num_tiles: int,
|
||||
) -> tuple[list[str], dict[str, Any]]:
|
||||
if len(videos) == 0 or not self.supports_video:
|
||||
return text, {}
|
||||
@@ -763,7 +909,6 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
video_metadata_lst = [v[1] for v in videos]
|
||||
pixel_values_lst_video = self._videos_to_pixel_values_lst(
|
||||
videos_lst,
|
||||
max_num_tiles=max_num_tiles,
|
||||
)
|
||||
|
||||
# We use frame duration in milliseconds (as integer) to ensure
|
||||
@@ -788,12 +933,10 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
"frame_duration_ms": torch.tensor(frame_duration_ms_lst),
|
||||
}
|
||||
|
||||
image_size: int = self.config.force_image_size
|
||||
patch_size: int = self.config.patch_size
|
||||
downsample_ratio = self.config.downsample_ratio
|
||||
tokens_in_single_frame = int(
|
||||
(image_size * image_size // patch_size**2) * (downsample_ratio**2)
|
||||
)
|
||||
|
||||
T = self.video_temporal_patch_size
|
||||
|
||||
for pixel_values, video_metadata, frames_indices, frame_duration_ms in zip(
|
||||
pixel_values_lst_video,
|
||||
@@ -802,23 +945,28 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
frame_duration_ms_lst,
|
||||
):
|
||||
num_frames = pixel_values.shape[0]
|
||||
frame_h, frame_w = pixel_values.shape[-2], pixel_values.shape[-1]
|
||||
tokens_in_single_frame = int(
|
||||
(frame_h * frame_w // patch_size**2) * (downsample_ratio**2)
|
||||
)
|
||||
num_tubelets = math.ceil(num_frames / T) if T > 1 else num_frames
|
||||
|
||||
if self.video_pruning_rate is not None and self.video_pruning_rate > 0.0:
|
||||
# Start of EVS-specific code
|
||||
num_tokens = compute_retained_tokens_count(
|
||||
tokens_per_frame=tokens_in_single_frame,
|
||||
num_frames=num_frames,
|
||||
num_frames=num_tubelets,
|
||||
q=self.video_pruning_rate,
|
||||
)
|
||||
|
||||
# Here we just need placeholders that won't actually be replaced -
|
||||
# we just need to make sure the total number of tokens is correct
|
||||
# assign all tokens to the first frame
|
||||
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
|
||||
tokens_per_frame = [num_tokens] + [0] * (num_tubelets - 1)
|
||||
|
||||
# End of EVS-specific code
|
||||
else:
|
||||
tokens_per_frame = [tokens_in_single_frame] * num_frames
|
||||
tokens_per_frame = [tokens_in_single_frame] * num_tubelets
|
||||
|
||||
video_repl = self.get_video_repl(
|
||||
tokens_per_frame=tokens_per_frame,
|
||||
@@ -828,6 +976,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
img_start_token_ids=self._img_start_token_ids,
|
||||
img_end_token_ids=self._img_end_token_ids,
|
||||
img_context_token_ids=self._img_context_token_ids,
|
||||
video_temporal_patch_size=T,
|
||||
)
|
||||
|
||||
# video_repl.full is a list of token IDs
|
||||
@@ -908,7 +1057,6 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
text, video_inputs = self._preprocess_video(
|
||||
text=text,
|
||||
videos=videos,
|
||||
max_num_tiles=1,
|
||||
)
|
||||
|
||||
text, audio_inputs = self._preprocess_audio(
|
||||
@@ -962,6 +1110,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
img_start_token_ids: list[int],
|
||||
img_end_token_ids: list[int],
|
||||
img_context_token_ids: list[int],
|
||||
video_temporal_patch_size: int = 1,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
"""
|
||||
Build prompt replacement for a video.
|
||||
@@ -981,31 +1130,60 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
- EVS real (called from get_real_video_repl_for_evs) - different value per frame
|
||||
Args:
|
||||
tokens_per_frame (list[int]): number of tokens per frame
|
||||
frames_indices (list[int]): frame indices
|
||||
(one per tubelet when T > 1)
|
||||
frames_indices (list[int]): orig. frame indices
|
||||
(one per frame, before tubelet subsampling)
|
||||
frame_duration_ms (int): duration of each frame in milliseconds
|
||||
tokenizer (HfTokenizer): tokenizer to use for tokenizing frame separators
|
||||
tokenizer (TokenizerLike): tokenizer to use for tokenizing frame separators
|
||||
img_start_token_ids (list[int]): pre-tokenized IMG_START tokens
|
||||
img_end_token_ids (list[int]): pre-tokenized IMG_END tokens
|
||||
img_context_token_ids (list[int]): pre-tokenized IMG_CONTEXT tokens
|
||||
video_temporal_patch_size (int): temporal patch size for videos
|
||||
"""
|
||||
# TODO: Add support of frame_duration_ms to be None
|
||||
# At preprocessing step we should allow absent / metadata without
|
||||
# frames_indices field.
|
||||
timestamps_enabled = frame_duration_ms is not None
|
||||
T = video_temporal_patch_size
|
||||
num_frames = len(frames_indices)
|
||||
|
||||
if timestamps_enabled:
|
||||
if T > 1 and timestamps_enabled:
|
||||
all_timestamps = calculate_timestamps(frames_indices, frame_duration_ms)
|
||||
|
||||
frame_separators = []
|
||||
for group_idx, i in enumerate(range(0, num_frames, T)):
|
||||
group_frames = []
|
||||
for j in range(T): # Every frame in the group
|
||||
frame_idx = i + j
|
||||
if frame_idx < num_frames:
|
||||
# Valid idx (haven't padded to mult. of T yet)
|
||||
ts = all_timestamps[frame_idx]
|
||||
frame_str = "Frame" if j == 0 else "frame"
|
||||
group_frames.append(
|
||||
f"{frame_str} {frame_idx + 1} sampled at {ts:.2f} seconds"
|
||||
)
|
||||
if group_frames:
|
||||
# Join by `and` if there are >1 frame, otherwise no `and`
|
||||
# Prepend \n to match training format (except first group)
|
||||
sep = " and ".join(group_frames) + ": "
|
||||
if group_idx > 0:
|
||||
sep = "\n" + sep
|
||||
frame_separators.append(sep)
|
||||
elif timestamps_enabled:
|
||||
timestamps = calculate_timestamps(frames_indices, frame_duration_ms)
|
||||
|
||||
assert len(timestamps) == len(tokens_per_frame), (
|
||||
"timestamps and tokens_per_frame must have the same length"
|
||||
)
|
||||
frame_separators = [
|
||||
f"Frame {i + 1} sampled at {timestamp:.2f} seconds: "
|
||||
("\n" if i > 0 else "")
|
||||
+ f"Frame {i + 1} sampled at {timestamp:.2f} seconds: "
|
||||
for i, timestamp in enumerate(timestamps)
|
||||
]
|
||||
else:
|
||||
frame_separators = [
|
||||
f"Frame {i + 1}: " for i, _ in enumerate(tokens_per_frame)
|
||||
("\n" if i > 0 else "") + f"Frame {i + 1}: "
|
||||
for i, _ in enumerate(tokens_per_frame)
|
||||
]
|
||||
|
||||
# Tokenize frame separator independently
|
||||
|
||||
Reference in New Issue
Block a user