From 67c89fe40ac50b9d232087f07748bfcd25b54885 Mon Sep 17 00:00:00 2001 From: KaivalyaMDabhadkar Date: Wed, 13 May 2026 00:17:55 +0530 Subject: [PATCH] [Model][Bugfix] Fix Step3-VL image_embeds input path (#42333) Signed-off-by: Kaivalya Dabhadkar --- .../processing/test_step3_vl_image_embeds.py | 58 +++++++++++++++++++ vllm/model_executor/models/step3_vl.py | 24 ++++---- 2 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 tests/models/multimodal/processing/test_step3_vl_image_embeds.py diff --git a/tests/models/multimodal/processing/test_step3_vl_image_embeds.py b/tests/models/multimodal/processing/test_step3_vl_image_embeds.py new file mode 100644 index 0000000000..a43c93c074 --- /dev/null +++ b/tests/models/multimodal/processing/test_step3_vl_image_embeds.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for Step3-VL precomputed image embedding inputs.""" + +import pytest +import torch + +from vllm.model_executor.models.step3_vl import ( + Step3VLForConditionalGeneration, + Step3VLImageEmbeddingInputs, +) + + +class _FakeStep3VL: + @staticmethod + def _process_image_features(image_features: torch.Tensor) -> torch.Tensor: + return image_features + + +def test_image_embedding_inputs_construction(): + """Step3VLImageEmbeddingInputs should store embeddings in the data field.""" + image_embeds = torch.randn(2, 16, 64) + + inputs = Step3VLImageEmbeddingInputs( + type="image_embeds", + data=image_embeds, + ) + + assert inputs["type"] == "image_embeds" + assert torch.equal(inputs["data"], image_embeds) + assert torch.equal(inputs.data, image_embeds) + + +def test_image_embedding_inputs_validation_rejects_wrong_rank(): + """Validation should reject tensors with wrong rank.""" + with pytest.raises(ValueError, match="rank"): + Step3VLImageEmbeddingInputs( + type="image_embeds", + data=torch.randn(16, 64), + ) + + +def test_process_image_embeds_does_not_require_pixel_input_fields(): + """The image_embeds branch should not reference patch pixel metadata.""" + image_embeds = torch.randn(2, 4, 8) + image_input = Step3VLImageEmbeddingInputs( + type="image_embeds", + data=image_embeds, + ) + + outputs = Step3VLForConditionalGeneration._process_image_input( + _FakeStep3VL(), + image_input, + ) + + assert len(outputs) == 2 + assert torch.equal(outputs[0], image_embeds[0]) + assert torch.equal(outputs[1], image_embeds[1]) diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index dc4b429619..8caa124552 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -592,7 +592,7 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) if image_embeds is not None: return Step3VLImageEmbeddingInputs( type="image_embeds", - image_embeds=image_embeds.to(self.dtype), + data=image_embeds.to(self.dtype), ) raise AssertionError("This line should be unreachable.") @@ -615,15 +615,19 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) self, image_input: Step3VLImageInputs ) -> tuple[torch.Tensor, ...]: if image_input["type"] == "image_embeds": - image_features = image_input["image_embeds"] - else: - image_features = self._get_vision_model_output(image_input["pixel_values"]) - patch_image_features = ( - self._get_vision_model_output(image_input["patch_pixel_values"]) - if len(image_input["patch_pixel_values"]) > 0 - else None - ) - num_patches = image_input["num_patches"] + image_features = image_input["data"] + return [ + image_features[i].view(-1, image_features.shape[-1]) + for i in range(image_features.shape[0]) + ] + + image_features = self._get_vision_model_output(image_input["pixel_values"]) + patch_image_features = ( + self._get_vision_model_output(image_input["patch_pixel_values"]) + if len(image_input["patch_pixel_values"]) > 0 + else None + ) + num_patches = image_input["num_patches"] image_features = self._process_image_features(image_features) patch_image_features = (