[Model][Bugfix] Fix Step3-VL image_embeds input path (#42333)

Signed-off-by: Kaivalya Dabhadkar <kdabhadkar@nvidia.com>
This commit is contained in:
KaivalyaMDabhadkar
2026-05-12 18:47:55 +00:00
committed by GitHub
parent d9b4990783
commit 67c89fe40a
2 changed files with 72 additions and 10 deletions
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Step3-VL precomputed image embedding inputs."""
import pytest
import torch
from vllm.model_executor.models.step3_vl import (
Step3VLForConditionalGeneration,
Step3VLImageEmbeddingInputs,
)
class _FakeStep3VL:
@staticmethod
def _process_image_features(image_features: torch.Tensor) -> torch.Tensor:
return image_features
def test_image_embedding_inputs_construction():
"""Step3VLImageEmbeddingInputs should store embeddings in the data field."""
image_embeds = torch.randn(2, 16, 64)
inputs = Step3VLImageEmbeddingInputs(
type="image_embeds",
data=image_embeds,
)
assert inputs["type"] == "image_embeds"
assert torch.equal(inputs["data"], image_embeds)
assert torch.equal(inputs.data, image_embeds)
def test_image_embedding_inputs_validation_rejects_wrong_rank():
"""Validation should reject tensors with wrong rank."""
with pytest.raises(ValueError, match="rank"):
Step3VLImageEmbeddingInputs(
type="image_embeds",
data=torch.randn(16, 64),
)
def test_process_image_embeds_does_not_require_pixel_input_fields():
"""The image_embeds branch should not reference patch pixel metadata."""
image_embeds = torch.randn(2, 4, 8)
image_input = Step3VLImageEmbeddingInputs(
type="image_embeds",
data=image_embeds,
)
outputs = Step3VLForConditionalGeneration._process_image_input(
_FakeStep3VL(),
image_input,
)
assert len(outputs) == 2
assert torch.equal(outputs[0], image_embeds[0])
assert torch.equal(outputs[1], image_embeds[1])
+14 -10
View File
@@ -592,7 +592,7 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
if image_embeds is not None:
return Step3VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds.to(self.dtype),
data=image_embeds.to(self.dtype),
)
raise AssertionError("This line should be unreachable.")
@@ -615,15 +615,19 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
self, image_input: Step3VLImageInputs
) -> tuple[torch.Tensor, ...]:
if image_input["type"] == "image_embeds":
image_features = image_input["image_embeds"]
else:
image_features = self._get_vision_model_output(image_input["pixel_values"])
patch_image_features = (
self._get_vision_model_output(image_input["patch_pixel_values"])
if len(image_input["patch_pixel_values"]) > 0
else None
)
num_patches = image_input["num_patches"]
image_features = image_input["data"]
return [
image_features[i].view(-1, image_features.shape[-1])
for i in range(image_features.shape[0])
]
image_features = self._get_vision_model_output(image_input["pixel_values"])
patch_image_features = (
self._get_vision_model_output(image_input["patch_pixel_values"])
if len(image_input["patch_pixel_values"]) > 0
else None
)
num_patches = image_input["num_patches"]
image_features = self._process_image_features(image_features)
patch_image_features = (