Optimize nemotron VL image/video preprocessing (#40283)

Signed-off-by: milesial <milesial@users.noreply.github.com>
Co-authored-by: milesial <milesial@users.noreply.github.com>
This commit is contained in:
Netanel Haber
2026-04-19 15:06:20 +00:00
committed by GitHub
co-authored by milesial
parent 45232a454e
commit 982beae809
@@ -8,7 +8,6 @@
# -------------------------------------------------------- # --------------------------------------------------------
import math import math
import warnings
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Sequence from collections.abc import Sequence
from dataclasses import dataclass from dataclasses import dataclass
@@ -26,7 +25,7 @@ from transformers import BatchFeature, PretrainedConfig, TensorType
from vllm.model_executor.models.parakeet import ParakeetExtractor from vllm.model_executor.models.parakeet import ParakeetExtractor
from vllm.multimodal.evs import compute_retained_tokens_count from vllm.multimodal.evs import compute_retained_tokens_count
from vllm.multimodal.inputs import AudioItem from vllm.multimodal.inputs import AudioItem
from vllm.multimodal.processing.processor import PromptUpdateDetails, _seq2tokens from vllm.multimodal.processing.processor import PromptUpdateDetails
from vllm.tokenizers.hf import HfTokenizer from vllm.tokenizers.hf import HfTokenizer
from .internvl import calculate_internvl_targets, get_internvl_target_ratios from .internvl import calculate_internvl_targets, get_internvl_target_ratios
@@ -63,42 +62,50 @@ def calculate_timestamps(
return timestamps return timestamps
def input_conditioner(x: torch.Tensor, norm_mean: torch.Tensor, norm_std: torch.Tensor): @torch.compile(dynamic=True)
return (x - norm_mean) / norm_std def _bicubic_resize_and_normalize(
tensor: torch.Tensor,
size: tuple[int, int] | None = None,
def _bicubic_from_ndarray( norm_mean: torch.Tensor | None = None,
array: npt.NDArray[Any], *, size: tuple[int, int] norm_std: torch.Tensor | None = None,
dtype: torch.dtype = torch.float32,
) -> torch.Tensor: ) -> torch.Tensor:
""" """Permute NHWC→NCHW, optional bicubic resize, rescale + normalize.
Convert a 4D NHWC ndarray to NCHW and interpolate with bicubic.
Suppresses PyTorch's non-writable NumPy warning because interpolate copies,
and torch.from_numpy(array) is discarded at the end of function scope.
"""
with warnings.catch_warnings(): Input must be a raw 4-D **NHWC** tensor.
msg = "The given NumPy array is not writ.*"
# Apparently, different versions of PyTorch use writable or writeable. *size*: target ``(H, W)``; skips interpolation when ``None``.
warnings.filterwarnings("ignore", message=msg, category=UserWarning) *norm_mean* / *norm_std*: when both provided, fused
tensor = torch.from_numpy(array) ``(x/255 - mean) / std`` + dtype cast; otherwise ``x/255`` + cast.
assert tensor.ndim == 4, f"{tensor.ndim=}" """
tensor = tensor.permute(0, 3, 1, 2) tensor = tensor.permute(0, 3, 1, 2).to(dtype=torch.float32)
return ( if size is not None:
torch.nn.functional.interpolate( tensor = torch.nn.functional.interpolate(
tensor, size=size, mode="bicubic", align_corners=False, antialias=True tensor, size=size, mode="bicubic", align_corners=False, antialias=True
) )
/ 255.0 if norm_mean is not None and norm_std is not None:
return ((tensor / 255.0 - norm_mean) / norm_std).to(dtype=dtype).contiguous()
return (tensor / 255.0).to(dtype=dtype).contiguous()
def _pil_to_nhwc_tensor(image: Image.Image) -> torch.Tensor:
"""Convert a PIL image to a 4-D NHWC tensor suitable for compiled ops."""
array = np.asarray(
image.convert("RGB") if image.mode != "RGB" else image, dtype=np.uint8
) )
return torch.from_numpy(np.expand_dims(array, axis=0))
def dynamic_preprocess( def dynamic_preprocess(
image, image: Image.Image,
*, *,
image_size=512, image_size: int = 512,
max_num_tiles=12, max_num_tiles: int = 12,
use_thumbnail=True, use_thumbnail: bool = True,
idx=0, norm_mean: torch.Tensor | None = None,
): norm_std: torch.Tensor | None = None,
dtype: torch.dtype = torch.float32,
) -> torch.Tensor:
orig_width, orig_height = image.size orig_width, orig_height = image.size
target_ratios = get_internvl_target_ratios(1, max_num_tiles) target_ratios = get_internvl_target_ratios(1, max_num_tiles)
@@ -111,13 +118,15 @@ def dynamic_preprocess(
use_thumbnail=False, use_thumbnail=False,
) )
image = np.asarray( tensor = _pil_to_nhwc_tensor(image)
image.convert("RGB") if image.mode != "RGB" else image, dtype=np.uint8
resized_img = _bicubic_resize_and_normalize(
tensor,
size=(target_height, target_width),
norm_mean=norm_mean,
norm_std=norm_std,
dtype=dtype,
) )
image = np.expand_dims(image, axis=0)
resized_img = _bicubic_from_ndarray(image, size=(target_height, target_width))
B, C, H, W = resized_img.shape B, C, H, W = resized_img.shape
hp, wp = H // image_size, W // image_size hp, wp = H // image_size, W // image_size
patches = ( patches = (
@@ -127,30 +136,16 @@ def dynamic_preprocess(
) )
if use_thumbnail and patches.shape[0] > 1: if use_thumbnail and patches.shape[0] > 1:
thumb = _bicubic_from_ndarray(image, size=(image_size, image_size)) thumb = _bicubic_resize_and_normalize(
tensor,
size=(image_size, image_size),
norm_mean=norm_mean,
norm_std=norm_std,
dtype=dtype,
)
patches = torch.cat([patches, thumb], dim=0) patches = torch.cat([patches, thumb], dim=0)
return list(patches) return patches
def image_to_pixel_values(
image: Image.Image,
*,
input_size: int,
max_num: int,
use_thumbnail: bool,
idx: int,
) -> torch.Tensor:
images = dynamic_preprocess(
image,
image_size=input_size,
max_num_tiles=max_num,
use_thumbnail=use_thumbnail,
idx=idx,
)
pixel_values = torch.stack(images)
return pixel_values
def _compute_aspect_preserving_size( def _compute_aspect_preserving_size(
@@ -233,14 +228,16 @@ def video_to_pixel_values(
video_maintain_aspect_ratio: bool = False, video_maintain_aspect_ratio: bool = False,
patch_size: int = 16, patch_size: int = 16,
downsample_ratio: float = 0.5, downsample_ratio: float = 0.5,
norm_mean: torch.Tensor | None = None,
norm_std: torch.Tensor | None = None,
dtype: torch.dtype = torch.float32,
) -> torch.Tensor: ) -> torch.Tensor:
# (num_frames, H, W, C) -> (num_frames, C, H, W) """Convert video ndarray (T, H, W, C) to normalized pixel tensor (T, C, H, W)."""
video_tensor = torch.from_numpy(video).permute(0, 3, 1, 2) orig_h, orig_w = video.shape[1], video.shape[2]
size: tuple[int, int] | None = None
if video_target_num_patches is not None: if video_target_num_patches is not None:
# Resize to target patch count (aspect-preserving or square). tw, th, _ = get_video_target_size_and_feature_size(
orig_h, orig_w = video_tensor.shape[2], video_tensor.shape[3]
target_w, target_h, _ = get_video_target_size_and_feature_size(
orig_w=orig_w, orig_w=orig_w,
orig_h=orig_h, orig_h=orig_h,
target_patches=video_target_num_patches, target_patches=video_target_num_patches,
@@ -248,14 +245,13 @@ def video_to_pixel_values(
patch_size=patch_size, patch_size=patch_size,
downsample_ratio=downsample_ratio, downsample_ratio=downsample_ratio,
) )
if video_tensor.shape[2] != target_h or video_tensor.shape[3] != target_w: if orig_h != th or orig_w != tw:
return _bicubic_from_ndarray(video, size=(target_h, target_w)) size = (th, tw)
elif video_tensor.shape[2] != input_size or video_tensor.shape[3] != input_size: elif orig_h != input_size or orig_w != input_size:
return _bicubic_from_ndarray(video, size=(input_size, input_size)) size = (input_size, input_size)
video_tensor = video_tensor / 255.0 tensor = torch.from_numpy(video)
return _bicubic_resize_and_normalize(tensor, size, norm_mean, norm_std, dtype)
return video_tensor
class DynamicResolutionImageTiler: class DynamicResolutionImageTiler:
@@ -343,6 +339,7 @@ class DynamicResolutionImageTiler:
self, self,
text_prompt_length: int, text_prompt_length: int,
images: list[Image.Image], images: list[Image.Image],
dtype: torch.dtype = torch.float32,
) -> tuple[list[torch.Tensor], list[int]]: ) -> tuple[list[torch.Tensor], list[int]]:
num_tokens_available = self.max_num_tokens_available(text_prompt_length) num_tokens_available = self.max_num_tokens_available(text_prompt_length)
params_per_image = self.compute_params(images, num_tokens_available) params_per_image = self.compute_params(images, num_tokens_available)
@@ -350,7 +347,7 @@ class DynamicResolutionImageTiler:
feature_sizes = [] feature_sizes = []
images = [] images = []
for param in params_per_image: for param in params_per_image:
for t in self.apply_params(param): for t in self.apply_params(param, dtype=dtype):
assert t.ndim == 3, f"{t.ndim=}: expected 3 dim tensor" assert t.ndim == 3, f"{t.ndim=}: expected 3 dim tensor"
images.append(t) images.append(t)
feature_sizes.append(param.num_embeddings) feature_sizes.append(param.num_embeddings)
@@ -363,17 +360,23 @@ class DynamicResolutionImageTiler:
num_embeddings: int num_embeddings: int
patch_size: tuple[int, int] patch_size: tuple[int, int]
def apply_params(self, params: DynamicResolutionParams) -> list[torch.Tensor]: def apply_params(
self,
params: DynamicResolutionParams,
dtype: torch.dtype = torch.float32,
) -> list[torch.Tensor]:
target_size = ( target_size = (
params.patch_size[1] * self._patch_size, params.patch_size[1] * self._patch_size,
params.patch_size[0] * self._patch_size, params.patch_size[0] * self._patch_size,
) )
image = np.asarray( tensor = _pil_to_nhwc_tensor(params.media)
params.media.convert("RGB") if params.media.mode != "RGB" else params.media, resized_img = _bicubic_resize_and_normalize(
dtype=np.uint8, tensor,
size=target_size,
norm_mean=self.norm_mean,
norm_std=self.norm_std,
dtype=dtype,
) )
image = np.expand_dims(image, axis=0)
resized_img = _bicubic_from_ndarray(image, size=target_size)
return list(resized_img) return list(resized_img)
def process_media( def process_media(
@@ -619,6 +622,7 @@ class BaseNanoNemotronVLProcessor(ABC):
norm_mean=config.norm_mean, norm_mean=config.norm_mean,
norm_std=config.norm_std, norm_std=config.norm_std,
) )
self.dtype: torch.dtype = getattr(config, "dtype", torch.float32)
@staticmethod @staticmethod
def use_dynamic_resolution(config: PretrainedConfig) -> bool: def use_dynamic_resolution(config: PretrainedConfig) -> bool:
@@ -662,14 +666,16 @@ class BaseNanoNemotronVLProcessor(ABC):
max_num_tiles: int, max_num_tiles: int,
) -> list[torch.Tensor]: ) -> list[torch.Tensor]:
return [ return [
image_to_pixel_values( dynamic_preprocess(
image, image,
input_size=self.image_size, image_size=self.image_size,
max_num=max_num_tiles, max_num_tiles=max_num_tiles,
use_thumbnail=self.use_thumbnail, use_thumbnail=self.use_thumbnail,
idx=idx, norm_mean=self.norm_mean,
norm_std=self.norm_std,
dtype=self.dtype,
) )
for idx, image in enumerate(images) for image in images
] ]
def _preprocess_image( def _preprocess_image(
@@ -690,23 +696,22 @@ class BaseNanoNemotronVLProcessor(ABC):
pixel_values_lst, num_tokens_per_image = tiler._images_to_pixel_values_lst( pixel_values_lst, num_tokens_per_image = tiler._images_to_pixel_values_lst(
text_prompt_length=text_prompt_length, text_prompt_length=text_prompt_length,
images=images, images=images,
dtype=self.dtype,
) )
imgs_sizes = [(pv.shape[-2], pv.shape[-1]) for pv in pixel_values_lst] imgs_sizes = [(pv.shape[-2], pv.shape[-1]) for pv in pixel_values_lst]
normalized = [
input_conditioner(img, tiler.norm_mean, tiler.norm_std)
for img in pixel_values_lst
]
image_num_patches = torch.tensor([1] * len(num_tokens_per_image)) image_num_patches = torch.tensor([1] * len(num_tokens_per_image))
image_inputs = { image_inputs = {
"pixel_values_flat": normalized, "pixel_values_flat": pixel_values_lst,
"imgs_sizes": imgs_sizes, "imgs_sizes": imgs_sizes,
"num_tokens_per_image": num_tokens_per_image, "num_tokens_per_image": num_tokens_per_image,
} }
else: else:
pixel_values_lst = self._images_to_pixel_values_lst(images, max_num_tiles) pixel_values_lst = self._images_to_pixel_values_lst(images, max_num_tiles)
image_num_patches = torch.tensor([len(item) for item in pixel_values_lst]) image_num_patches = torch.tensor([len(item) for item in pixel_values_lst])
pixel_values_flat = input_conditioner( pixel_values_flat = (
torch.cat(pixel_values_lst), self.norm_mean, self.norm_std torch.cat(pixel_values_lst)
if len(pixel_values_lst) > 1
else pixel_values_lst[0]
) )
image_inputs = { image_inputs = {
"pixel_values_flat": pixel_values_flat, "pixel_values_flat": pixel_values_flat,
@@ -863,6 +868,8 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
def _videos_to_pixel_values_lst( def _videos_to_pixel_values_lst(
self, self,
videos: list[npt.NDArray], videos: list[npt.NDArray],
*,
dtype: torch.dtype = torch.float32,
) -> list[torch.Tensor]: ) -> list[torch.Tensor]:
return [ return [
video_to_pixel_values( video_to_pixel_values(
@@ -872,6 +879,9 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
video_maintain_aspect_ratio=self.video_maintain_aspect_ratio, video_maintain_aspect_ratio=self.video_maintain_aspect_ratio,
patch_size=self.config.patch_size, patch_size=self.config.patch_size,
downsample_ratio=self.config.downsample_ratio, downsample_ratio=self.config.downsample_ratio,
norm_mean=self.norm_mean,
norm_std=self.norm_std,
dtype=dtype,
) )
for video in videos for video in videos
] ]
@@ -886,8 +896,10 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
videos_lst = [v[0] for v in videos] videos_lst = [v[0] for v in videos]
video_metadata_lst = [v[1] for v in videos] video_metadata_lst = [v[1] for v in videos]
pixel_values_lst_video = self._videos_to_pixel_values_lst( pixel_values_lst_video = self._videos_to_pixel_values_lst(
videos_lst, videos_lst,
dtype=self.dtype,
) )
# We use frame duration in milliseconds (as integer) to ensure # We use frame duration in milliseconds (as integer) to ensure
@@ -903,10 +915,15 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
metadata["frames_indices"] for metadata in video_metadata_lst metadata["frames_indices"] for metadata in video_metadata_lst
] ]
video_num_patches = torch.tensor([len(item) for item in pixel_values_lst_video]) video_num_patches = torch.tensor([len(item) for item in pixel_values_lst_video])
# Normalization already fused into resize above.
# Skip the torch.cat copy when there is exactly one video
if len(pixel_values_lst_video) == 1:
pixel_values_flat = pixel_values_lst_video[0]
else:
pixel_values_flat = torch.cat(pixel_values_lst_video)
video_inputs = { video_inputs = {
"pixel_values_flat_video": input_conditioner( "pixel_values_flat_video": pixel_values_flat,
torch.cat(pixel_values_lst_video), self.norm_mean, self.norm_std
),
"video_num_patches": video_num_patches, "video_num_patches": video_num_patches,
"frames_indices": frames_indices_lst, "frames_indices": frames_indices_lst,
"frame_duration_ms": torch.tensor(frame_duration_ms_lst), "frame_duration_ms": torch.tensor(frame_duration_ms_lst),
@@ -1168,20 +1185,21 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
for i, _ in enumerate(tokens_per_frame) for i, _ in enumerate(tokens_per_frame)
] ]
# Tokenize frame separator independently # Batch-tokenize all frame separators at once — the HuggingFace
frame_separators_tokenized = [ # tokenizers Rust backend parallelizes batch encoding across threads.
_seq2tokens(tokenizer, sep) for sep in frame_separators batch_encoded = tokenizer(
] frame_separators,
add_special_tokens=False,
return_attention_mask=False,
)
frame_separators_tokenized: list[list[int]] = batch_encoded["input_ids"]
# Tokenize each component independently to avoid tokenizer merging tokens # Tokenize each component independently to avoid tokenizer merging tokens
# across boundaries. This ensures consistent tokenization regardless of # across boundaries. This ensures consistent tokenization regardless of
# num_tokens_per_frame values. # num_tokens_per_frame values.
all_token_ids = [] all_token_ids = []
for i, num_tokens in enumerate(tokens_per_frame): for i, num_tokens in enumerate(tokens_per_frame):
frame_sep_token_ids = frame_separators_tokenized[i] all_token_ids.extend(frame_separators_tokenized[i])
all_token_ids.extend(frame_sep_token_ids)
# Add pre-tokenized special tokens
all_token_ids.extend(img_start_token_ids) all_token_ids.extend(img_start_token_ids)
all_token_ids.extend(img_context_token_ids * num_tokens) all_token_ids.extend(img_context_token_ids * num_tokens)
all_token_ids.extend(img_end_token_ids) all_token_ids.extend(img_end_token_ids)