# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -------------------------------------------------------- # Adapted from # https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/internvl.py # under Apache-2.0 License # LICENSE is in root directory. # -------------------------------------------------------- import math import warnings from collections.abc import Iterable, Mapping, Sequence from functools import cached_property from io import BytesIO from typing import Annotated, Literal, TypeAlias, TypedDict, cast import torch import torch.nn as nn from transformers import BatchFeature, PreTrainedConfig from vllm import envs from vllm.config import VllmConfig from vllm.config.multimodal import ( MultiModalDummyOptions, VideoDummyOptions, ) from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.logger import init_logger from vllm.lora.layers.base import BaseLayerWithLoRA from vllm.model_executor.layers.activation import ReLUSquaredActivation from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( HasInnerState, IsHybrid, MultiModalEmbeddings, SupportsEagle3, SupportsLoRA, SupportsMultiModal, SupportsMultiModalPruning, ) from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM from vllm.model_executor.models.parakeet import ParakeetExtractor, ProjectedParakeet from vllm.model_executor.models.radio import RadioModel, calc_seq_lens from vllm.model_executor.models.utils import ( WeightsMapper, init_vllm_registered_model, maybe_prefix, ) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( AudioItem, BatchedTensorInputs, MultiModalFieldConfig, MultiModalKwargsItems, VideoItem, ) from vllm.multimodal.media.audio import load_audio_pyav from vllm.multimodal.parse import ( AudioProcessorItems, ImageEmbeddingItems, ImageProcessorItems, ImageSize, MultiModalDataItems, MultiModalDataParser, VideoProcessorItems, ) from vllm.multimodal.processing import ( BaseDummyInputsBuilder, ProcessorInputs, TimingContext, ) from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, HFMultiModalInputs, PromptReplacement, PromptUpdate, cached_encode, ) from vllm.multimodal.video_prune.evs import ( compute_retained_tokens_count, compute_retention_mask, ) from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.configs.radio import RadioConfig from vllm.transformers_utils.processors.internvl import get_internvl_target_ratios from vllm.transformers_utils.processors.nano_nemotron_vl import ( AUDIO_CONTEXT, IMG_CONTEXT, IMG_END, IMG_START, BaseNanoNemotronVLProcessor, DynamicResolutionImageTiler, NanoNemotronVLProcessor, get_video_target_size_and_feature_size, ) from vllm.utils.tensor_schema import TensorSchema, TensorShape from .utils import _merge_multimodal_embeddings logger = init_logger(__name__) MAX_AUDIO_LEN_S = 10 * 60 # 10 minutes class NanoNemotronVLAudioFeatureInputs(TensorSchema): """Dimensions: - c: Number of audio clips (possibly flattened across audio items) - b: Number of original audio items - t: Audio feature length - f: Feature size (mel bins) """ type: Literal["audio_features"] = "audio_features" input_audio_features: Annotated[torch.Tensor, TensorShape("c", "t", "f")] feature_attention_mask: Annotated[torch.Tensor, TensorShape("c", "t")] audio_num_clips: list[int] | torch.Tensor class NanoNemotronVLImagePixelInputs(TensorSchema): """Dimensions: - bn: Batch size * number of images - bnp: Batch size * number of images * (1 + num_patches) - c: Number of channels (3) - h: Height of each image patch - w: Width of each image patch """ type: Literal["pixel_values"] = "pixel_values" pixel_values_flat: Annotated[torch.Tensor, TensorShape("bnp", 3, "h", "w")] num_patches: Annotated[torch.Tensor, TensorShape("bn")] class NanoNemotronVLImagePixelInputsDynamic(TensorSchema): """Dynamic-resolution image inputs. imgs_sizes: per-image (height, width) in pixels. num_tokens_per_image: per-image number of embedding tokens (post downsample). """ type: Literal["pixel_values_dynamic"] = "pixel_values_dynamic" pixel_values_flat: Annotated[torch.Tensor, TensorShape("bn", "h", "w")] imgs_sizes: list[tuple[int, int]] num_tokens_per_image: list[int] class NanoNemotronVLImageEmbeddingInputs(TensorSchema): """Dimensions: - n: Number of images - f: Total image feature size - h: Hidden size (must match the hidden size of language model backbone) """ type: Literal["image_embeds"] data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("n", "f", "h")] NanoNemotronVLImageInputs: TypeAlias = ( NanoNemotronVLImagePixelInputs | NanoNemotronVLImagePixelInputsDynamic | NanoNemotronVLImageEmbeddingInputs ) class NanoNemotronVLVideoPixelInputs(TensorSchema): """Dimensions: - bvf: Batch size * number of videos * num_frames - bn: Batch size * number of videos - f: Number of frames - c: Number of channels (3) - h: Height of each video frame - w: Width of each video frame """ type: Literal["pixel_values_videos"] pixel_values_flat: Annotated[torch.Tensor, TensorShape("bvf", 3, "h", "w")] num_patches: Annotated[torch.Tensor, TensorShape("bn")] frames_indices: Annotated[torch.Tensor, TensorShape("bvf")] frame_duration_ms: Annotated[torch.Tensor, TensorShape("bn")] class NanoNemotronVLVideoEmbeddingInputs(TensorSchema): """Dimensions: - n: Number of videos - f: Total video feature size - h: Hidden size (must match the hidden size of language model backbone) """ type: Literal["video_embeds"] data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("n", "f", "h")] NanoNemotronVLVideoInputs: TypeAlias = ( NanoNemotronVLVideoPixelInputs | NanoNemotronVLVideoEmbeddingInputs ) class NanoNemotronVLMultiModalInputs(TypedDict, total=False): images: NanoNemotronVLImageInputs | None videos: NanoNemotronVLVideoInputs | None audios: NanoNemotronVLAudioFeatureInputs | None class NanoNemotronVLProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor: # Not accepted by NanoNemotronVLProcessor.__init__ kwargs.pop("truncation", None) return self.ctx.init_processor( NanoNemotronVLProcessor, config=self.get_hf_config(), tokenizer=self.get_tokenizer(), video_token=self.get_video_token(), video_pruning_rate=self.get_video_pruning_rate(), max_model_len=self.ctx.model_config.max_model_len, **kwargs, ) @cached_property def is_dynamic_tiler(self) -> bool: return BaseNanoNemotronVLProcessor.use_dynamic_resolution(self.get_hf_config()) @property def supports_video(self): return True @property def supports_audio(self) -> bool: return self.sound_config is not None def get_video_token(self) -> str | None: return IMG_CONTEXT def get_video_pruning_rate(self) -> float | None: return self.ctx.get_mm_config().video_pruning_rate @property def sound_config(self) -> PreTrainedConfig | None: return getattr(self.get_hf_config(), "sound_config", None) def get_default_tok_params(self) -> TokenizeParams: return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_supported_mm_limits(self) -> Mapping[str, int | None]: image_limit = {"image": None} video_limit = {"video": None} if self.supports_video else {} audio_limit = {"audio": None} if self.supports_audio else {} return {**image_limit, **video_limit, **audio_limit} def get_data_parser(self): target_sr = None target_channels = None if self.sound_config: target_sr = self.sound_config.sampling_rate target_channels = 1 return MultiModalDataParser( video_needs_metadata=True, target_sr=target_sr, target_channels=target_channels, expected_hidden_size=self._get_expected_hidden_size(), ) def get_image_size_with_most_features(self, max_num_tiles: int) -> ImageSize: processor = self.get_hf_processor() base_size = processor.image_size target_ratios = get_internvl_target_ratios(1, max_num_tiles) largest_feature_size, largest_feature_pinpoint = 0, None for wr, hr in target_ratios: width, height = base_size * wr, base_size * hr feat_size = processor.get_num_image_tokens( image_width=width, image_height=height, max_num_tiles=max_num_tiles ) if feat_size > largest_feature_size: largest_feature_size = feat_size largest_feature_pinpoint = ImageSize(width=width, height=height) if largest_feature_size == 0 or largest_feature_pinpoint is None: raise ValueError("Cannot have a largest feature size of 0!") return largest_feature_pinpoint def get_max_image_tokens(self) -> int: processor = self.get_hf_processor() # Use default max_num_tiles for max tokens calculation max_num_tiles = processor.max_num_tiles target_width, target_height = self.get_image_size_with_most_features( max_num_tiles ) return processor.get_num_image_tokens( image_width=target_width, image_height=target_height, max_num_tiles=max_num_tiles, ) def get_dummy_image_size_and_max_tokens( self, mm_counts: Mapping[str, int] ) -> tuple[tuple[int, int], int]: processor = self.get_hf_processor() num_images = mm_counts.get("image", 0) if tiler := processor.dynamic_tiler: budget = tiler.max_num_tokens_available(text_prompt_length=num_images) target_width, target_height = ( tiler.width_and_height_for_max_num_tokens_available(budget) ) return ( (target_width, target_height), tiler._get_num_embeddings(target_width, target_height), ) max_num_tiles = processor.max_num_tiles target_width, target_height = self.get_image_size_with_most_features( max_num_tiles ) return ( (target_width, target_height), processor.get_num_image_tokens( image_width=target_width, image_height=target_height, max_num_tiles=max_num_tiles, ), ) def get_num_frames_with_most_features( self, seq_len: int, mm_counts: Mapping[str, int], ) -> int: max_images = mm_counts.get("image", 0) max_videos = mm_counts.get("video", 0) processor = self.get_hf_processor() # we get the CustomProcessor here T = processor.video_temporal_patch_size max_image_tokens = self.get_max_image_tokens() * max_images tokens_per_tubelet = processor.num_video_token max_total_tubelets = (seq_len - max_image_tokens) // tokens_per_tubelet max_tubelets_per_video = max_total_tubelets // max(max_videos, 1) max_frames_per_video = max_tubelets_per_video * T return max(max_frames_per_video, 1) def get_mm_max_tokens_per_item( self, seq_len: int, mm_counts: Mapping[str, int] ) -> Mapping[str, int]: mm_max_tokens: dict[str, int] = {} if mm_counts.get("image", 0) > 0: _, mm_max_tokens["image"] = self.get_dummy_image_size_and_max_tokens( mm_counts ) if mm_counts.get("video", 0) > 0: assert self.supports_video mm_max_tokens["video"] = seq_len if mm_counts.get("audio", 0) > 0: assert self.supports_audio mm_max_tokens["audio"] = seq_len return mm_max_tokens class NanoNemotronVLMultiModalProcessor( BaseMultiModalProcessor[NanoNemotronVLProcessingInfo] ): def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) def _get_hf_mm_inputs( self, mm_items: MultiModalDataItems, hf_kwargs: Mapping[str, object], ) -> HFMultiModalInputs: hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) # NanoNemotronVLProcessor expects "audios" instead of "audio" hf_data = hf_inputs.hf_data if "audio" in hf_data: hf_data["audios"] = hf_data.pop("audio") return hf_inputs def _get_image_fields_config(self, hf_inputs: BatchFeature): if self.info.is_dynamic_tiler: pixel_values_flat = MultiModalFieldConfig.batched("image") else: image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0)) pixel_values_flat = MultiModalFieldConfig.flat_from_sizes( "image", image_num_patches ) return dict( pixel_values_flat=pixel_values_flat, image_num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True), image_embeds=MultiModalFieldConfig.batched("image"), num_tokens_per_image=MultiModalFieldConfig.batched( "image", keep_on_cpu=True ), imgs_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True), ) def _get_video_fields_config(self, hf_inputs: BatchFeature): video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0)) return dict( pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes( "video", video_num_patches ), video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True), frames_indices=MultiModalFieldConfig.batched("video", keep_on_cpu=True), frame_duration_ms=MultiModalFieldConfig.batched("video", keep_on_cpu=True), ) def _get_audio_fields_config(self, hf_inputs: BatchFeature): audio_num_clips = torch.as_tensor( hf_inputs.get("audio_num_clips", torch.empty(0)) ) return dict( input_audio_features=MultiModalFieldConfig.flat_from_sizes( "audio", audio_num_clips ), feature_attention_mask=MultiModalFieldConfig.flat_from_sizes( "audio", audio_num_clips ), audio_num_clips=MultiModalFieldConfig.batched("audio", keep_on_cpu=True), ) def _get_mm_fields_config( self, hf_inputs: BatchFeature, hf_processor_mm_kwargs: Mapping[str, object], ) -> Mapping[str, MultiModalFieldConfig]: fields = self._get_image_fields_config(hf_inputs) if self.info.supports_video: fields |= self._get_video_fields_config(hf_inputs) if self.info.supports_audio: fields |= self._get_audio_fields_config(hf_inputs) return fields def _get_prompt_repl_image( self, mm_items: MultiModalDataItems, hf_processor: NanoNemotronVLProcessor, out_mm_data: BatchedTensorInputs, ): tokenizer = self.info.get_tokenizer() vocab = tokenizer.get_vocab() if "image_num_patches" in out_mm_data: image_num_patches = out_mm_data["image_num_patches"] assert isinstance(image_num_patches, torch.Tensor) image_num_patches = image_num_patches.tolist() elif "image_embeds" in out_mm_data: # to compute num_patches (similar to Qwen2-VL) image_num_patches = [None] * len(out_mm_data["image_embeds"]) else: image_num_patches = [] def get_image_replacement(item_idx: int): images = mm_items.get_items( "image", (ImageEmbeddingItems, ImageProcessorItems) ) if isinstance(images, ImageEmbeddingItems): feature_size = images.get_feature_size(item_idx) elif self.info.is_dynamic_tiler: num_tokens_per_image = out_mm_data["num_tokens_per_image"] assert isinstance(num_tokens_per_image, list) feature_size = num_tokens_per_image[item_idx] else: assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) max_num_tiles = hf_processor.max_num_tiles feature_size = hf_processor.get_num_image_tokens( image_width=image_size.width, image_height=image_size.height, max_num_tiles=max_num_tiles, ) num_patches = None local_image_num_patches = image_num_patches if isinstance(local_image_num_patches, torch.Tensor): local_image_num_patches = local_image_num_patches.tolist() if isinstance(local_image_num_patches, (list, tuple)) and item_idx < len( local_image_num_patches ): num_patches = int(local_image_num_patches[item_idx]) return hf_processor.get_image_repl(feature_size, num_patches) return PromptReplacement( modality="image", target=[vocab[""]], replacement=get_image_replacement, ) def _get_prompt_repl_video( self, mm_items: MultiModalDataItems, hf_processor: NanoNemotronVLProcessor, out_mm_data: BatchedTensorInputs, ): tokenizer = self.info.get_tokenizer() if "video_num_patches" in out_mm_data: video_num_patches = out_mm_data["video_num_patches"] assert isinstance(video_num_patches, torch.Tensor) video_num_patches = video_num_patches.tolist() else: video_num_patches = [] def get_video_replacement(item_idx: int): video, metadata = mm_items["video"][item_idx] patch_size = hf_processor.config.patch_size downsample_ratio = hf_processor.config.downsample_ratio target_patches = hf_processor.video_target_num_patches if target_patches is not None and video is not None and video.shape[0] > 0: orig_h, orig_w = video.shape[1], video.shape[2] _, _, feature_size = get_video_target_size_and_feature_size( orig_w=orig_w, orig_h=orig_h, target_patches=target_patches, maintain_aspect_ratio=hf_processor.video_maintain_aspect_ratio, patch_size=patch_size, downsample_ratio=downsample_ratio, ) else: feature_size = hf_processor.num_image_token num_patches = video_num_patches[item_idx] if num_patches is not None: assert isinstance(num_patches, int) T = hf_processor.video_temporal_patch_size num_tubelets: int | None if T > 1 and num_patches is not None: num_tubelets = math.ceil(num_patches / T) else: num_tubelets = num_patches assert num_tubelets is not None video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate if video_pruning_rate is not None and video_pruning_rate > 0.0: # Start of EVS-specific code num_tokens = compute_retained_tokens_count( tokens_per_frame=feature_size, num_frames=num_tubelets, q=video_pruning_rate, ) # Here we just need placeholders that won't actually be replaced - # we just need to make sure the total number of tokens is correct # assign all tokens to the first frame tokens_per_frame = [num_tokens] + [0] * (num_tubelets - 1) # End of EVS-specific code else: tokens_per_frame = [feature_size] * num_tubelets frame_duration_ms = int(1000 / metadata["fps"]) return hf_processor.get_video_repl( tokens_per_frame=tokens_per_frame, frames_indices=metadata["frames_indices"], frame_duration_ms=frame_duration_ms, tokenizer=hf_processor.tokenizer, img_start_token_ids=hf_processor._img_start_token_ids, img_end_token_ids=hf_processor._img_end_token_ids, img_context_token_ids=hf_processor._img_context_token_ids, video_temporal_patch_size=T, ) return PromptReplacement( modality="video", target=cached_encode(tokenizer, "