Support temporal compression for Nemotron-3-VL videos (#36808)
Signed-off-by: Collin McCarthy <cmccarthy@nvidia.com>
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
# --------------------------------------------------------
|
||||
|
||||
import copy
|
||||
import math
|
||||
import warnings
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
@@ -77,6 +78,7 @@ from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.radio import RadioConfig
|
||||
from vllm.transformers_utils.processors.internvl import get_internvl_target_ratios
|
||||
from vllm.transformers_utils.processors.nano_nemotron_vl import (
|
||||
AUDIO_CONTEXT,
|
||||
IMG_CONTEXT,
|
||||
@@ -85,7 +87,7 @@ from vllm.transformers_utils.processors.nano_nemotron_vl import (
|
||||
BaseNanoNemotronVLProcessor,
|
||||
DynamicResolutionImageTiler,
|
||||
NanoNemotronVLProcessor,
|
||||
get_internvl_target_ratios,
|
||||
get_video_target_size_and_feature_size,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -295,10 +297,13 @@ class NanoNemotronVLProcessingInfo(BaseNanoNemotronVLProcessingInfo):
|
||||
max_videos = mm_counts.get("video", 0)
|
||||
|
||||
processor = self.get_hf_processor() # we get the CustomProcessor here
|
||||
T = processor.video_temporal_patch_size
|
||||
|
||||
max_image_tokens = self.get_max_image_tokens() * max_images
|
||||
max_total_frames = (seq_len - max_image_tokens) // processor.num_image_token
|
||||
max_frames_per_video = max_total_frames // max(max_videos, 1)
|
||||
tokens_per_tubelet = processor.num_video_token
|
||||
max_total_tubelets = (seq_len - max_image_tokens) // tokens_per_tubelet
|
||||
max_tubelets_per_video = max_total_tubelets // max(max_videos, 1)
|
||||
max_frames_per_video = max_tubelets_per_video * T
|
||||
return max(max_frames_per_video, 1)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:
|
||||
@@ -589,28 +594,49 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
video_num_patches = []
|
||||
|
||||
def get_video_replacement_internvl(item_idx: int):
|
||||
feature_size = hf_processor.num_image_token
|
||||
video, metadata = mm_items["video"][item_idx]
|
||||
patch_size = hf_processor.config.patch_size
|
||||
downsample_ratio = hf_processor.config.downsample_ratio
|
||||
target_patches = hf_processor.video_target_num_patches
|
||||
|
||||
if target_patches is not None and video is not None and video.shape[0] > 0:
|
||||
orig_h, orig_w = video.shape[1], video.shape[2]
|
||||
_, _, feature_size = get_video_target_size_and_feature_size(
|
||||
orig_w=orig_w,
|
||||
orig_h=orig_h,
|
||||
target_patches=target_patches,
|
||||
maintain_aspect_ratio=hf_processor.video_maintain_aspect_ratio,
|
||||
patch_size=patch_size,
|
||||
downsample_ratio=downsample_ratio,
|
||||
)
|
||||
else:
|
||||
feature_size = hf_processor.num_image_token
|
||||
num_patches = video_num_patches[item_idx]
|
||||
if num_patches is not None:
|
||||
assert isinstance(num_patches, int)
|
||||
|
||||
T = hf_processor.video_temporal_patch_size
|
||||
if T > 1 and num_patches is not None:
|
||||
num_tubelets = math.ceil(num_patches / T)
|
||||
else:
|
||||
num_tubelets = num_patches
|
||||
|
||||
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
|
||||
if video_pruning_rate is not None and video_pruning_rate > 0.0:
|
||||
# Start of EVS-specific code
|
||||
num_tokens = compute_retained_tokens_count(
|
||||
tokens_per_frame=feature_size,
|
||||
num_frames=num_patches,
|
||||
num_frames=num_tubelets,
|
||||
q=video_pruning_rate,
|
||||
)
|
||||
# Here we just need placeholders that won't actually be replaced -
|
||||
# we just need to make sure the total number of tokens is correct
|
||||
# assign all tokens to the first frame
|
||||
tokens_per_frame = [num_tokens] + [0] * (num_patches - 1)
|
||||
tokens_per_frame = [num_tokens] + [0] * (num_tubelets - 1)
|
||||
|
||||
# End of EVS-specific code
|
||||
else:
|
||||
tokens_per_frame = [feature_size] * num_patches
|
||||
tokens_per_frame = [feature_size] * num_tubelets
|
||||
|
||||
frame_duration_ms = int(1000 / metadata["fps"])
|
||||
return hf_processor.get_video_repl(
|
||||
@@ -621,6 +647,7 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
img_start_token_ids=hf_processor._img_start_token_ids,
|
||||
img_end_token_ids=hf_processor._img_end_token_ids,
|
||||
img_context_token_ids=hf_processor._img_context_token_ids,
|
||||
video_temporal_patch_size=T,
|
||||
)
|
||||
|
||||
if self.info.supports_video:
|
||||
@@ -745,15 +772,39 @@ class NanoNemotronVLDummyInputsBuilder(
|
||||
if self.info.supports_video:
|
||||
config = self.info.get_hf_config()
|
||||
image_size: int = config.force_image_size
|
||||
processor = self.info.get_hf_processor()
|
||||
|
||||
# When video_target_num_patches is set the per-frame pixel
|
||||
# resolution can exceed image_size. Use the actual target
|
||||
# dimensions so that profiling sees the correct upper bound.
|
||||
if processor.video_target_num_patches is not None:
|
||||
target_w, target_h, _ = get_video_target_size_and_feature_size(
|
||||
orig_w=image_size,
|
||||
orig_h=image_size,
|
||||
target_patches=processor.video_target_num_patches,
|
||||
maintain_aspect_ratio=processor.video_maintain_aspect_ratio,
|
||||
patch_size=config.patch_size,
|
||||
downsample_ratio=config.downsample_ratio,
|
||||
)
|
||||
video_width, video_height = target_w, target_h
|
||||
else:
|
||||
video_width, video_height = image_size, image_size
|
||||
|
||||
target_num_frames = self.info.get_num_frames_with_most_features(
|
||||
seq_len, mm_counts
|
||||
)
|
||||
mm_config = self.info.ctx.get_mm_config()
|
||||
if num_frames := mm_config.media_io_kwargs.get("video", {}).get(
|
||||
"num_frames"
|
||||
):
|
||||
assert num_frames > 0
|
||||
target_num_frames = num_frames
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
video_overrides = mm_options.get("video")
|
||||
dummy_video = {
|
||||
"video": self._get_dummy_videos(
|
||||
width=image_size,
|
||||
height=image_size,
|
||||
width=video_width,
|
||||
height=video_height,
|
||||
num_frames=target_num_frames,
|
||||
num_videos=num_videos,
|
||||
overrides=video_overrides,
|
||||
@@ -790,6 +841,9 @@ class NanoNemotronVLDummyInputsBuilder(
|
||||
class NemotronH_Nano_VL_V2(
|
||||
nn.Module, HasInnerState, IsHybrid, SupportsMultiModal, SupportsMultiModalPruning
|
||||
):
|
||||
requires_sequential_video_encoding = True
|
||||
"""Temporarily needed for dynamic res video w/ conv3d, doesn't support bs>1 yet"""
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
@@ -817,6 +871,11 @@ class NemotronH_Nano_VL_V2(
|
||||
self.image_tag_type = config.image_tag_type
|
||||
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
||||
|
||||
vision_config = getattr(config, "vision_config", config)
|
||||
self.video_temporal_patch_size: int = getattr(
|
||||
vision_config, "video_temporal_patch_size", 1
|
||||
)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
@@ -838,11 +897,12 @@ class NemotronH_Nano_VL_V2(
|
||||
|
||||
mlp1 = nn.Sequential(
|
||||
RMSNorm(
|
||||
hidden_size=vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
|
||||
hidden_size=vit_hidden_size
|
||||
* int(round(1 / self.downsample_ratio)) ** 2,
|
||||
eps=1e-5,
|
||||
),
|
||||
nn.Linear(
|
||||
vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
|
||||
vit_hidden_size * int(round(1 / self.downsample_ratio)) ** 2,
|
||||
vision_projection_hidden_size,
|
||||
bias=False,
|
||||
),
|
||||
@@ -958,19 +1018,37 @@ class NemotronH_Nano_VL_V2(
|
||||
vit_embeds = self.mlp1(vit_embeds)
|
||||
return vit_embeds
|
||||
|
||||
def extract_feature(self, pixel_values: torch.Tensor):
|
||||
def extract_feature(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
num_frames: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Process images in a micro-batch of at most 128 frames per call
|
||||
# This is done on purpose to ensure peak GPU ram usage of huge batch
|
||||
# (namely for really long videos with EVS ON) won't cause any problems
|
||||
# as we don't support chunked prefill for video media
|
||||
micro_batch_size = 128
|
||||
n = pixel_values.shape[0]
|
||||
# This is done on purpose to ensure peak GPU ram usage of huge batch
|
||||
# (namely for really long videos with EVS ON) won't cause any problems
|
||||
# as we don't support chunked prefill for video media
|
||||
# When num_frames is provided and temporal_patch_size > 1, consecutive
|
||||
# frames are grouped into tubelets — the batch size must be a multiple
|
||||
# of T so chunk boundaries don't split a tubelet.
|
||||
N, _C, H, W = pixel_values.shape
|
||||
|
||||
T = self.video_temporal_patch_size if num_frames is not None else 1
|
||||
micro_batch_size = 128 - (128 % T)
|
||||
patch_size = self.patch_size
|
||||
H_patches = H // patch_size
|
||||
W_patches = W // patch_size
|
||||
|
||||
vit_embeds_list = []
|
||||
for i in range(0, n, micro_batch_size):
|
||||
_, vit_embeds = self.vision_model(pixel_values[i : i + micro_batch_size])
|
||||
for i in range(0, N, micro_batch_size):
|
||||
chunk = pixel_values[i : i + micro_batch_size]
|
||||
if num_frames is not None and T > 1:
|
||||
_, vit_embeds = self.vision_model(chunk, num_frames=chunk.shape[0])
|
||||
else:
|
||||
_, vit_embeds = self.vision_model(chunk)
|
||||
vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
|
||||
h = w = int(vit_embeds.shape[1] ** 0.5)
|
||||
vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
|
||||
vit_embeds = vit_embeds.reshape(
|
||||
vit_embeds.shape[0], H_patches, W_patches, -1
|
||||
)
|
||||
vit_embeds = self.pixel_shuffle(
|
||||
vit_embeds, scale_factor=self.downsample_ratio
|
||||
)
|
||||
@@ -1042,16 +1120,21 @@ class NemotronH_Nano_VL_V2(
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Process video input and create final embeddings with video content
|
||||
and indicator tokens."""
|
||||
# Get video embeddings using the same processing as images
|
||||
video_embeddings = self._process_image_input(video_input)
|
||||
T = self.video_temporal_patch_size
|
||||
|
||||
if T > 1:
|
||||
video_embeddings = self._extract_video_embeddings_temporal(video_input)
|
||||
else:
|
||||
video_embeddings = self._process_image_input(video_input)
|
||||
|
||||
final_video_embeddings: tuple[torch.Tensor, ...] = ()
|
||||
|
||||
image_rows = image_cols = self.config.force_image_size
|
||||
downsample_ratio = self.config.downsample_ratio
|
||||
patch_size = self.config.patch_size
|
||||
rows = int(image_rows * downsample_ratio // patch_size)
|
||||
cols = int(image_cols * downsample_ratio // patch_size)
|
||||
pixel_values = video_input["pixel_values_flat"]
|
||||
frame_h, frame_w = pixel_values.shape[-2], pixel_values.shape[-1]
|
||||
rows = int(frame_h * downsample_ratio // patch_size)
|
||||
cols = int(frame_w * downsample_ratio // patch_size)
|
||||
video_pruning_rate = self.video_pruning_rate
|
||||
video_num_frames = video_input["num_patches"].tolist()
|
||||
video_frames_indices = video_input["frames_indices"].split(video_num_frames)
|
||||
@@ -1062,13 +1145,14 @@ class NemotronH_Nano_VL_V2(
|
||||
num_frames = video_num_frames[i]
|
||||
frames_indices = video_frames_indices[i].tolist()
|
||||
frame_duration_ms = video_input["frame_duration_ms"][i].item()
|
||||
assert single_video_embeddings.shape[0] % num_frames == 0
|
||||
num_tubelets = math.ceil(num_frames / T) if T > 1 else num_frames
|
||||
assert single_video_embeddings.shape[0] % num_tubelets == 0
|
||||
|
||||
if video_pruning_rate is not None and video_pruning_rate > 0.0:
|
||||
# Start of EVS-specific code
|
||||
retention_mask = compute_retention_mask(
|
||||
single_video_embeddings,
|
||||
video_size_thw=(num_frames, rows, cols),
|
||||
video_size_thw=(num_tubelets, rows, cols),
|
||||
spatial_merge_size=1,
|
||||
q=video_pruning_rate,
|
||||
)
|
||||
@@ -1077,14 +1161,14 @@ class NemotronH_Nano_VL_V2(
|
||||
single_video_embeddings = single_video_embeddings[retention_mask]
|
||||
|
||||
# calculate the actual number of retained tokens per frame
|
||||
retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)
|
||||
retention_mask_thw = retention_mask.reshape(num_tubelets, rows, cols)
|
||||
num_tokens_per_frame = (
|
||||
retention_mask_thw.sum(dim=(1, 2)).long().tolist()
|
||||
)
|
||||
# End of EVS-specific code
|
||||
else:
|
||||
feature_size = single_video_embeddings.shape[0] // num_frames
|
||||
num_tokens_per_frame = [feature_size] * num_frames
|
||||
feature_size = single_video_embeddings.shape[0] // num_tubelets
|
||||
num_tokens_per_frame = [feature_size] * num_tubelets
|
||||
|
||||
final_video_embeddings += (
|
||||
self._create_final_video_embeddings(
|
||||
@@ -1092,11 +1176,36 @@ class NemotronH_Nano_VL_V2(
|
||||
num_tokens_per_frame,
|
||||
frames_indices,
|
||||
frame_duration_ms,
|
||||
video_temporal_patch_size=T,
|
||||
),
|
||||
)
|
||||
|
||||
return final_video_embeddings
|
||||
|
||||
def _extract_video_embeddings_temporal(
|
||||
self, video_input: NanoNemotronVLVideoPixelInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Extract per-video embeddings with temporal compression.
|
||||
|
||||
Each video is processed separately through extract_feature with
|
||||
num_frames, which uses the fixed-resolution temporal path in RADIO
|
||||
(no attention mask, flash attention).
|
||||
"""
|
||||
pixel_values = video_input["pixel_values_flat"]
|
||||
num_frames_per_video = video_input["num_patches"].tolist()
|
||||
hidden_size = self.config.text_config.hidden_size
|
||||
|
||||
results: list[torch.Tensor] = []
|
||||
frame_offset = 0
|
||||
for nf in num_frames_per_video:
|
||||
video_frames = pixel_values[frame_offset : frame_offset + nf]
|
||||
frame_offset += nf
|
||||
|
||||
vit_embeds = self.extract_feature(video_frames, num_frames=nf)
|
||||
results.append(vit_embeds.view(-1, hidden_size))
|
||||
|
||||
return tuple(results)
|
||||
|
||||
def _process_audio_input(
|
||||
self, audio_input: NanoNemotronVLAudioFeatureInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
@@ -1134,6 +1243,7 @@ class NemotronH_Nano_VL_V2(
|
||||
num_tokens_per_frame: list[int],
|
||||
frames_indices: list[int],
|
||||
frame_duration_ms: int,
|
||||
video_temporal_patch_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Create final embeddings that combine video embeddings with
|
||||
text embeddings of indicator tokens.
|
||||
@@ -1161,6 +1271,7 @@ class NemotronH_Nano_VL_V2(
|
||||
img_start_token_ids=self._img_start_token_ids,
|
||||
img_end_token_ids=self._img_end_token_ids,
|
||||
img_context_token_ids=self._img_context_token_ids,
|
||||
video_temporal_patch_size=video_temporal_patch_size,
|
||||
)
|
||||
|
||||
# video_repl.full is a list of token IDs
|
||||
@@ -1207,8 +1318,27 @@ class NemotronH_Nano_VL_V2(
|
||||
else:
|
||||
frames_indices = torch.cat([f.flatten() for f in frames_indices], dim=0)
|
||||
|
||||
frame_duration_ms = frame_duration_ms.flatten()
|
||||
expected_h = expected_w = self.config.force_image_size
|
||||
if torch.is_tensor(frame_duration_ms):
|
||||
frame_duration_ms = frame_duration_ms.flatten()
|
||||
else:
|
||||
frame_duration_ms = torch.cat(
|
||||
[f.flatten() for f in frame_duration_ms], dim=0
|
||||
)
|
||||
|
||||
if (
|
||||
torch.is_tensor(pixel_values_flat_video)
|
||||
and pixel_values_flat_video.ndim == 5
|
||||
):
|
||||
# batched._reduce_data stacked same-shape videos into
|
||||
# [num_videos, nf, 3, H, W]; unstack back to a list so the
|
||||
# same-H,W cat path below handles it uniformly.
|
||||
pixel_values_flat_video = list(pixel_values_flat_video)
|
||||
|
||||
if not torch.is_tensor(pixel_values_flat_video):
|
||||
pixel_values_flat_video = torch.cat(pixel_values_flat_video, dim=0)
|
||||
|
||||
expected_h = pixel_values_flat_video.shape[-2]
|
||||
expected_w = pixel_values_flat_video.shape[-1]
|
||||
num_frames = video_num_patches[0].item()
|
||||
resolve_bindings = {"h": expected_h, "w": expected_w, "f": num_frames}
|
||||
|
||||
@@ -1361,8 +1491,7 @@ class NemotronH_Nano_VL_V2(
|
||||
|
||||
self.language_model.load_weights(llm_weights)
|
||||
self.vision_model.load_weights(vision_weights)
|
||||
if self.sound_encoder is not None:
|
||||
assert len(sound_weights) > 0
|
||||
if self.sound_encoder is not None and len(sound_weights) > 0:
|
||||
self.sound_encoder.load_weights(sound_weights)
|
||||
|
||||
def get_vit_model_from_radio_config(self, hf_config):
|
||||
@@ -1375,12 +1504,23 @@ class NemotronH_Nano_VL_V2(
|
||||
image_size = preferred_resolution[0] if preferred_resolution else 224
|
||||
patch_size = getattr(hf_config_vision, "patch_size", 16)
|
||||
|
||||
# video_temporal_patch_size and separate_video_embedder are
|
||||
# top-level vision_config attributes, not inside args.
|
||||
video_temporal_patch_size = getattr(
|
||||
hf_config_vision, "video_temporal_patch_size", 1
|
||||
)
|
||||
separate_video_embedder = getattr(
|
||||
hf_config_vision, "separate_video_embedder", True
|
||||
)
|
||||
|
||||
radio_config = RadioConfig(
|
||||
model_name=model_name,
|
||||
image_size=image_size,
|
||||
patch_size=patch_size,
|
||||
norm_mean=hf_config.norm_mean,
|
||||
norm_std=hf_config.norm_std,
|
||||
video_temporal_patch_size=video_temporal_patch_size,
|
||||
separate_video_embedder=separate_video_embedder,
|
||||
**hf_config_vision.args,
|
||||
)
|
||||
|
||||
|
||||
@@ -123,6 +123,8 @@ class ViTPatchGenerator(nn.Module):
|
||||
register_multiple: int | None = None,
|
||||
num_registers: int | None = None,
|
||||
patch_bias: bool = False,
|
||||
temporal_patch_size: int = 1,
|
||||
separate_video_embedder: bool = True,
|
||||
device=None,
|
||||
dtype=None,
|
||||
):
|
||||
@@ -148,6 +150,7 @@ class ViTPatchGenerator(nn.Module):
|
||||
self.patch_size = patch_size
|
||||
self.abs_pos = abs_pos
|
||||
self.embed_dim = embed_dim
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
|
||||
self.num_rows = max_input_dims[0] // patch_size
|
||||
self.num_cols = max_input_dims[1] // patch_size
|
||||
@@ -160,6 +163,21 @@ class ViTPatchGenerator(nn.Module):
|
||||
patch_size, embed_dim, bias=patch_bias, **factory
|
||||
)
|
||||
|
||||
if temporal_patch_size > 1:
|
||||
if not separate_video_embedder:
|
||||
raise NotImplementedError(
|
||||
"Only separate_video_embedder=True is supported for"
|
||||
" temporal compression (temporal_patch_size > 1)"
|
||||
)
|
||||
self.video_embedder = ViTPatchLinear(
|
||||
patch_size,
|
||||
embed_dim,
|
||||
bias=patch_bias,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
**factory,
|
||||
)
|
||||
self._video_embedder_loaded = False
|
||||
|
||||
if abs_pos:
|
||||
scale = embed_dim**-0.5
|
||||
self.pos_embed = nn.Parameter(
|
||||
@@ -196,6 +214,60 @@ class ViTPatchGenerator(nn.Module):
|
||||
return patches, pos_enc
|
||||
return patches
|
||||
|
||||
def forward_video(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Process video frames with temporal compression.
|
||||
|
||||
Groups T consecutive frames into tubelets before embedding.
|
||||
|
||||
Args:
|
||||
x: [num_frames, 3, H, W] tensor of video frames
|
||||
|
||||
Returns:
|
||||
Embedded patches with temporal compression applied.
|
||||
"""
|
||||
if not self._video_embedder_loaded:
|
||||
raise ValueError(
|
||||
"Temporal compression (video_temporal_patch_size > 1) requires "
|
||||
"video_embedder weights, but they were never loaded. "
|
||||
"Ensure the checkpoint was trained with temporal compression."
|
||||
)
|
||||
T = self.temporal_patch_size
|
||||
input_size = x.shape[2:]
|
||||
|
||||
patches = self.im_to_patches(x) # [N, num_patches, 3*P*P]
|
||||
num_frames, num_spatial, feat_dim = patches.shape
|
||||
|
||||
# Pad to a multiple of T by repeating the last frame so that
|
||||
# all tubelets have exactly T frames.
|
||||
num_pad_frames = (-num_frames) % T
|
||||
if num_pad_frames > 0:
|
||||
last_frame_dup = patches[-1:].expand(num_pad_frames, -1, -1)
|
||||
patches = torch.cat([patches, last_frame_dup], dim=0)
|
||||
|
||||
# Group T frames per tubelet: for each spatial position, concatenate
|
||||
# features across T consecutive frames; order follows Megatron training
|
||||
num_frames_padded = patches.shape[0]
|
||||
num_tublets = num_frames_padded // T
|
||||
patches = rearrange(
|
||||
patches,
|
||||
"(tubelets frames) spatial feat -> tubelets spatial (frames feat)",
|
||||
tubelets=num_tublets,
|
||||
frames=T,
|
||||
spatial=num_spatial,
|
||||
feat=feat_dim,
|
||||
)
|
||||
|
||||
patches = self.video_embedder(patches)
|
||||
|
||||
patches, pos_enc = self.apply_pos_enc(patches, input_size=input_size)
|
||||
|
||||
patches = self.cls_token(patches)
|
||||
|
||||
patches = self.patch_normalizer(patches)
|
||||
if self.return_pos_enc:
|
||||
return patches, pos_enc
|
||||
return patches
|
||||
|
||||
def apply_pos_enc_dynamic(
|
||||
self, patches: torch.Tensor, imgs_sizes: list[tuple[int, int]]
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
@@ -381,66 +453,21 @@ class ViTPatchGenerator(nn.Module):
|
||||
return pos_embed
|
||||
|
||||
if self.cpe_mode:
|
||||
if self.training:
|
||||
min_scale = math.sqrt(0.1)
|
||||
scale = (
|
||||
torch.rand(batch_size, 1, 1, device=pos_embed.device)
|
||||
* (1 - min_scale)
|
||||
+ min_scale
|
||||
)
|
||||
aspect_min = math.log(3 / 4)
|
||||
aspect_max = -aspect_min
|
||||
aspect = torch.exp(
|
||||
torch.rand(batch_size, 1, 1, device=pos_embed.device)
|
||||
* (aspect_max - aspect_min)
|
||||
+ aspect_min
|
||||
)
|
||||
max_dim = max(input_dims)
|
||||
pos_embed = F.interpolate(
|
||||
pos_embed.float(),
|
||||
size=(max_dim, max_dim),
|
||||
align_corners=False,
|
||||
mode="bilinear",
|
||||
).to(pos_embed.dtype)
|
||||
|
||||
scale_x = scale * aspect
|
||||
scale_y = scale * (1 / aspect)
|
||||
scale_xy = torch.stack([scale_x, scale_y], dim=-1).clamp_(0, 1)
|
||||
|
||||
pos_xy = torch.rand(batch_size, 1, 1, 2, device=pos_embed.device) * (
|
||||
1 - scale_xy
|
||||
)
|
||||
|
||||
lin_x = torch.linspace(
|
||||
0, 1, steps=input_dims[1], device=pos_embed.device
|
||||
)[None, None].expand(batch_size, input_dims[0], -1)
|
||||
lin_y = torch.linspace(
|
||||
0, 1, steps=input_dims[0], device=pos_embed.device
|
||||
)[None, :, None].expand(batch_size, -1, input_dims[1])
|
||||
|
||||
lin_xy = torch.stack([lin_x, lin_y], dim=-1)
|
||||
|
||||
grid_xy = lin_xy * scale_xy + pos_xy
|
||||
|
||||
# Convert to [-1, 1] range
|
||||
grid_xy.mul_(2).sub_(1)
|
||||
|
||||
pos_embed = F.grid_sample(
|
||||
pos_embed.float().expand(batch_size, -1, -1, -1),
|
||||
grid=grid_xy,
|
||||
mode="bilinear",
|
||||
padding_mode="zeros",
|
||||
align_corners=True,
|
||||
).to(pos_embed.dtype)
|
||||
else:
|
||||
max_dim = max(input_dims)
|
||||
pos_embed = F.interpolate(
|
||||
pos_embed.float(),
|
||||
size=(max_dim, max_dim),
|
||||
align_corners=True,
|
||||
mode="bilinear",
|
||||
).to(pos_embed.dtype)
|
||||
|
||||
pos_embed = window_select(pos_embed)
|
||||
pos_embed = window_select(pos_embed)
|
||||
else:
|
||||
pos_embed = window_select(pos_embed)
|
||||
|
||||
if pos_embed.shape[-2:] != input_dims:
|
||||
pos_embed = F.interpolate(
|
||||
pos_embed.float(), size=input_dims, align_corners=True, mode="bilinear"
|
||||
pos_embed.float(), size=input_dims, align_corners=False, mode="bilinear"
|
||||
).to(pos_embed.dtype)
|
||||
|
||||
pos_embed = pos_embed.flatten(2).permute(0, 2, 1)
|
||||
@@ -473,9 +500,19 @@ class Im2Patches(nn.Module):
|
||||
|
||||
|
||||
class ViTPatchLinear(nn.Linear):
|
||||
def __init__(self, patch_size: int, embed_dim: int, bias: bool = False, **factory):
|
||||
super().__init__(3 * (patch_size**2), embed_dim, bias=bias, **factory)
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int,
|
||||
embed_dim: int,
|
||||
bias: bool = False,
|
||||
temporal_patch_size: int = 1,
|
||||
**factory,
|
||||
):
|
||||
super().__init__(
|
||||
3 * temporal_patch_size * (patch_size**2), embed_dim, bias=bias, **factory
|
||||
)
|
||||
self.patch_size = patch_size
|
||||
self.temporal_patch_size = temporal_patch_size
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
@@ -560,6 +597,7 @@ class RadioInternVisionModel(nn.Module):
|
||||
max_img_size = int(
|
||||
round(config.cpe_max_size / config.patch_size) * config.patch_size
|
||||
)
|
||||
self.temporal_patch_size = config.video_temporal_patch_size
|
||||
unique_teachers = set(t["name"] for t in config.teachers)
|
||||
self.patch_generator = ViTPatchGenerator(
|
||||
config.patch_size,
|
||||
@@ -569,6 +607,8 @@ class RadioInternVisionModel(nn.Module):
|
||||
cls_token=True,
|
||||
num_cls_tokens=len(unique_teachers) if config.cls_token_per_teacher else 1,
|
||||
register_multiple=config.register_multiple,
|
||||
temporal_patch_size=self.temporal_patch_size,
|
||||
separate_video_embedder=config.separate_video_embedder,
|
||||
)
|
||||
|
||||
self.encoder = RadioVisionEncoder(
|
||||
@@ -593,33 +633,68 @@ class RadioInternVisionModel(nn.Module):
|
||||
def inter_image_mask_metadata(
|
||||
self, imgs_sizes: list[tuple[int, int]], device: torch.device
|
||||
) -> MaskMetadata:
|
||||
"""Build mask metadata from image pixel sizes. Adds num_skip to each
|
||||
sequence length (cls/register tokens) to match patch generator output."""
|
||||
patch_size = self.patch_generator.patch_size
|
||||
num_skip = self.patch_generator.num_skip
|
||||
|
||||
seq_lens = calc_seq_lens(imgs_sizes, patch_size)
|
||||
adjusted = [s + num_skip for s in seq_lens]
|
||||
return self._inter_image_mask_metadata_from_seq_lens(adjusted, device=device)
|
||||
|
||||
def _inter_image_mask_metadata_from_seq_lens(
|
||||
self, seq_lens: list[int], device: torch.device
|
||||
) -> MaskMetadata:
|
||||
"""Build mask metadata from actual sequence lengths (already including
|
||||
cls/register tokens, i.e. patch_count + num_skip per item).
|
||||
Use inter_image_mask_metadata() when you only have imgs_sizes."""
|
||||
assert len(seq_lens) > 0
|
||||
cu_seqlens = torch.tensor(
|
||||
list(accumulate(adjusted, initial=0)), dtype=torch.int32, device=device
|
||||
list(accumulate(seq_lens, initial=0)), dtype=torch.int32, device=device
|
||||
)
|
||||
# Keep max_seqlen on CPU to avoid .item() sync
|
||||
# See: https://github.com/vllm-project/vllm/blob/20b6b01/vllm/v1/attention/ops/vit_attn_wrappers.py#L48
|
||||
max_seqlen = torch.tensor(max(adjusted), dtype=torch.int32)
|
||||
max_seqlen = torch.tensor(max(seq_lens), dtype=torch.int32)
|
||||
return MaskMetadata(cu_seqlens=cu_seqlens, max_seqlen=max_seqlen)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
imgs_sizes: list[tuple[int, int]] | None = None,
|
||||
num_frames: int | None = None,
|
||||
) -> torch.FloatTensor:
|
||||
hidden_states = self.patch_generator(x, imgs_sizes=imgs_sizes)
|
||||
T = self.temporal_patch_size
|
||||
|
||||
# Build packed-sequence metadata for MMEncoderAttention when needed.
|
||||
mask_meta = None
|
||||
if imgs_sizes is not None:
|
||||
assert len(imgs_sizes) > 0
|
||||
# Dynamic resolution: process each image as an independent sequence.
|
||||
mask_meta = self.inter_image_mask_metadata(
|
||||
imgs_sizes, device=hidden_states.device
|
||||
packed_batch_size = None # Original batch size before packing
|
||||
|
||||
if num_frames is not None and T > 1:
|
||||
# Conv3d video: all tubelets have the same sequence length.
|
||||
# Pack [num_tubelets, seq_per_tubelet, hidden] → [1, total, hidden]
|
||||
hidden_states = self.patch_generator.forward_video(x)
|
||||
packed_batch_size, seq_per_tubelet, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(1, -1, hidden_dim)
|
||||
mask_meta = self._inter_image_mask_metadata_from_seq_lens(
|
||||
[seq_per_tubelet] * packed_batch_size, device=hidden_states.device
|
||||
)
|
||||
else:
|
||||
# Images for any model, or video for non-conv3d model
|
||||
hidden_states = self.patch_generator(x, imgs_sizes=imgs_sizes)
|
||||
if imgs_sizes is not None and len(imgs_sizes) > 1:
|
||||
# Dynamic resolution w/ > 1 image, create attn mask
|
||||
mask_meta = self.inter_image_mask_metadata(
|
||||
imgs_sizes, device=hidden_states.device
|
||||
)
|
||||
|
||||
encoder_outputs = self.encoder(inputs_embeds=hidden_states, mask_meta=mask_meta)
|
||||
|
||||
# Unpack back to original batch shape if we packed for video
|
||||
if packed_batch_size is not None:
|
||||
encoder_outputs = encoder_outputs.reshape(
|
||||
packed_batch_size, seq_per_tubelet, -1
|
||||
)
|
||||
|
||||
return encoder_outputs
|
||||
|
||||
|
||||
@@ -663,8 +738,13 @@ class RadioModel(nn.Module):
|
||||
pixel_embeds: torch.Tensor | None = None,
|
||||
*,
|
||||
imgs_sizes: list[tuple[int, int]] | None = None,
|
||||
num_frames: int | None = None,
|
||||
) -> tuple[torch.FloatTensor, torch.FloatTensor]:
|
||||
y = self.model(pixel_values, imgs_sizes=imgs_sizes)
|
||||
y = self.model(
|
||||
pixel_values,
|
||||
imgs_sizes=imgs_sizes,
|
||||
num_frames=num_frames,
|
||||
)
|
||||
return self._extract_final(y, imgs_sizes=imgs_sizes)
|
||||
|
||||
def load_weights(self, weights) -> set[str]:
|
||||
@@ -714,6 +794,9 @@ class RadioModel(nn.Module):
|
||||
weight_loader(param, weight)
|
||||
loaded_params.add(vllm_key)
|
||||
|
||||
if "model.patch_generator.video_embedder.weight" in loaded_params:
|
||||
self.model.patch_generator._video_embedder_loaded = True
|
||||
|
||||
return loaded_params
|
||||
|
||||
def _extract_final(
|
||||
|
||||
Reference in New Issue
Block a user