[Model][Bugfix] Fix Step3-VL image_embeds input path (#42333)
Signed-off-by: Kaivalya Dabhadkar <kdabhadkar@nvidia.com>
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Step3-VL precomputed image embedding inputs."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.step3_vl import (
|
||||
Step3VLForConditionalGeneration,
|
||||
Step3VLImageEmbeddingInputs,
|
||||
)
|
||||
|
||||
|
||||
class _FakeStep3VL:
|
||||
@staticmethod
|
||||
def _process_image_features(image_features: torch.Tensor) -> torch.Tensor:
|
||||
return image_features
|
||||
|
||||
|
||||
def test_image_embedding_inputs_construction():
|
||||
"""Step3VLImageEmbeddingInputs should store embeddings in the data field."""
|
||||
image_embeds = torch.randn(2, 16, 64)
|
||||
|
||||
inputs = Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
data=image_embeds,
|
||||
)
|
||||
|
||||
assert inputs["type"] == "image_embeds"
|
||||
assert torch.equal(inputs["data"], image_embeds)
|
||||
assert torch.equal(inputs.data, image_embeds)
|
||||
|
||||
|
||||
def test_image_embedding_inputs_validation_rejects_wrong_rank():
|
||||
"""Validation should reject tensors with wrong rank."""
|
||||
with pytest.raises(ValueError, match="rank"):
|
||||
Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
data=torch.randn(16, 64),
|
||||
)
|
||||
|
||||
|
||||
def test_process_image_embeds_does_not_require_pixel_input_fields():
|
||||
"""The image_embeds branch should not reference patch pixel metadata."""
|
||||
image_embeds = torch.randn(2, 4, 8)
|
||||
image_input = Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
data=image_embeds,
|
||||
)
|
||||
|
||||
outputs = Step3VLForConditionalGeneration._process_image_input(
|
||||
_FakeStep3VL(),
|
||||
image_input,
|
||||
)
|
||||
|
||||
assert len(outputs) == 2
|
||||
assert torch.equal(outputs[0], image_embeds[0])
|
||||
assert torch.equal(outputs[1], image_embeds[1])
|
||||
@@ -592,7 +592,7 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
|
||||
if image_embeds is not None:
|
||||
return Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds.to(self.dtype),
|
||||
data=image_embeds.to(self.dtype),
|
||||
)
|
||||
|
||||
raise AssertionError("This line should be unreachable.")
|
||||
@@ -615,15 +615,19 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
|
||||
self, image_input: Step3VLImageInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
if image_input["type"] == "image_embeds":
|
||||
image_features = image_input["image_embeds"]
|
||||
else:
|
||||
image_features = self._get_vision_model_output(image_input["pixel_values"])
|
||||
patch_image_features = (
|
||||
self._get_vision_model_output(image_input["patch_pixel_values"])
|
||||
if len(image_input["patch_pixel_values"]) > 0
|
||||
else None
|
||||
)
|
||||
num_patches = image_input["num_patches"]
|
||||
image_features = image_input["data"]
|
||||
return [
|
||||
image_features[i].view(-1, image_features.shape[-1])
|
||||
for i in range(image_features.shape[0])
|
||||
]
|
||||
|
||||
image_features = self._get_vision_model_output(image_input["pixel_values"])
|
||||
patch_image_features = (
|
||||
self._get_vision_model_output(image_input["patch_pixel_values"])
|
||||
if len(image_input["patch_pixel_values"]) > 0
|
||||
else None
|
||||
)
|
||||
num_patches = image_input["num_patches"]
|
||||
|
||||
image_features = self._process_image_features(image_features)
|
||||
patch_image_features = (
|
||||
|
||||
Reference in New Issue
Block a user