[Model][Bugfix] Fix Step3-VL image_embeds input path (#42333)
Signed-off-by: Kaivalya Dabhadkar <kdabhadkar@nvidia.com>
This commit is contained in:
@@ -592,7 +592,7 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
|
||||
if image_embeds is not None:
|
||||
return Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds.to(self.dtype),
|
||||
data=image_embeds.to(self.dtype),
|
||||
)
|
||||
|
||||
raise AssertionError("This line should be unreachable.")
|
||||
@@ -615,15 +615,19 @@ class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
|
||||
self, image_input: Step3VLImageInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
if image_input["type"] == "image_embeds":
|
||||
image_features = image_input["image_embeds"]
|
||||
else:
|
||||
image_features = self._get_vision_model_output(image_input["pixel_values"])
|
||||
patch_image_features = (
|
||||
self._get_vision_model_output(image_input["patch_pixel_values"])
|
||||
if len(image_input["patch_pixel_values"]) > 0
|
||||
else None
|
||||
)
|
||||
num_patches = image_input["num_patches"]
|
||||
image_features = image_input["data"]
|
||||
return [
|
||||
image_features[i].view(-1, image_features.shape[-1])
|
||||
for i in range(image_features.shape[0])
|
||||
]
|
||||
|
||||
image_features = self._get_vision_model_output(image_input["pixel_values"])
|
||||
patch_image_features = (
|
||||
self._get_vision_model_output(image_input["patch_pixel_values"])
|
||||
if len(image_input["patch_pixel_values"]) > 0
|
||||
else None
|
||||
)
|
||||
num_patches = image_input["num_patches"]
|
||||
|
||||
image_features = self._process_image_features(image_features)
|
||||
patch_image_features = (
|
||||
|
||||
Reference in New Issue
Block a user