diff --git a/tests/models/multimodal/test_nano_nemotron_vl.py b/tests/models/multimodal/test_nano_nemotron_vl.py index 0aef83a827..f4ca19ed96 100644 --- a/tests/models/multimodal/test_nano_nemotron_vl.py +++ b/tests/models/multimodal/test_nano_nemotron_vl.py @@ -5,6 +5,7 @@ from unittest.mock import patch import numpy as np import pytest +import torch from vllm import envs from vllm.model_executor.models.nano_nemotron_vl import ( @@ -14,6 +15,17 @@ from vllm.model_executor.models.nano_nemotron_vl import ( from vllm.multimodal.parse import MultiModalDataItems, VideoProcessorItems +@pytest.mark.parametrize("input_key", ["image_embeds", "video_embeds"]) +def test_precomputed_multimodal_embeddings(input_key: str): + model = object.__new__(NemotronH_Nano_VL_V2) + embeds = torch.randn(2, 4, 8) + + outputs = model.embed_multimodal(**{input_key: embeds}) + + assert len(outputs) == len(embeds) + assert all(torch.equal(output, embed) for output, embed in zip(outputs, embeds)) + + class _TextOnlyMultiModalConfig: def get_limit_per_prompt(self, modality: str) -> int: return 0 diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index ac6e254cb2..2edf56871d 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -1103,7 +1103,8 @@ class NemotronH_Nano_VL_V2( def _parse_and_validate_image_input( self, **kwargs: object ) -> NanoNemotronVLImageInputs | None: - if image_embeds := kwargs.pop("image_embeds", None): + image_embeds = kwargs.pop("image_embeds", None) + if image_embeds is not None: return NanoNemotronVLImageEmbeddingInputs( type="image_embeds", data=image_embeds, @@ -1341,7 +1342,7 @@ class NemotronH_Nano_VL_V2( def _parse_and_validate_video_input( self, **kwargs: object - ) -> NanoNemotronVLVideoPixelInputs | None: + ) -> NanoNemotronVLVideoInputs | None: pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None) video_num_patches = kwargs.pop("video_num_patches", None) video_embeds = kwargs.pop("video_embeds", None) @@ -1408,7 +1409,10 @@ class NemotronH_Nano_VL_V2( and "images" not in modalities ): modalities["images"] = self._parse_and_validate_image_input(**kwargs) - if input_key in ("pixel_values_flat_video",) and "videos" not in modalities: + if ( + input_key in ("pixel_values_flat_video", "video_embeds") + and "videos" not in modalities + ): modalities["videos"] = self._parse_and_validate_video_input(**kwargs) if ( input_key @@ -1450,7 +1454,10 @@ class NemotronH_Nano_VL_V2( multimodal_embeddings += tuple(image_embeddings) if modality == "videos": video_input = modalities["videos"] - video_embeddings = self._process_video_input(video_input) + if video_input["type"] == "video_embeds": + video_embeddings = video_input["data"] + else: + video_embeddings = self._process_video_input(video_input) multimodal_embeddings += tuple(video_embeddings) if modality == "audios": audio_input = modalities["audios"]