[Mypy] Fix mypy typing for L models (#54177)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias
|
||||
from typing import Annotated, Literal, Protocol, TypeAlias, TypedDict
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
|
||||
)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -123,7 +123,12 @@ LlavaOnevisionMultiInputs: TypeAlias = (
|
||||
|
||||
|
||||
class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
|
||||
video_token_index: Final[int]
|
||||
video_token_index: int
|
||||
|
||||
|
||||
class LlavaOnevisionInputsByModality(TypedDict, total=False):
|
||||
image: LlavaOnevisionImageInputs | None
|
||||
video: LlavaOnevisionVideoPixelInputs | None
|
||||
|
||||
|
||||
class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
|
||||
@@ -288,6 +293,7 @@ class LlavaOnevisionDummyInputsBuilder(
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
video_overrides = mm_options.get("video")
|
||||
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
@@ -417,6 +423,7 @@ class LlavaOnevisionMultiModalProcessor(
|
||||
if isinstance(videos, VideoEmbeddingItems):
|
||||
num_video_tokens = videos.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(videos, VideoProcessorItems)
|
||||
image_size = videos.get_frame_size(item_idx)
|
||||
num_video_tokens = self.info.get_num_video_tokens(
|
||||
image_width=image_size.width,
|
||||
@@ -570,8 +577,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
},
|
||||
)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
def _parse_and_validate_multimodal_inputs(
|
||||
self, **kwargs: object
|
||||
) -> LlavaOnevisionInputsByModality:
|
||||
mm_input_by_modality = LlavaOnevisionInputsByModality()
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -749,7 +758,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
self,
|
||||
image_input: LlavaOnevisionImageInputs,
|
||||
) -> torch.Tensor | list[torch.Tensor]:
|
||||
if image_input["type"] == "image_embeds":
|
||||
if isinstance(image_input, LlavaOnevisionImageEmbeddingInputs):
|
||||
return image_input["data"]
|
||||
|
||||
patch_embeddings = self._process_image_pixels(image_input)
|
||||
@@ -860,12 +869,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
# NOTE: It is important to iterate over the keys in this dictionary
|
||||
# to preserve the order of the modalities.
|
||||
for modality in mm_input_by_modality:
|
||||
multimodal_input = mm_input_by_modality[modality]
|
||||
if modality == "image":
|
||||
image_embeddings = self._process_image_input(multimodal_input)
|
||||
image_input = mm_input_by_modality["image"]
|
||||
assert image_input is not None
|
||||
image_embeddings = self._process_image_input(image_input)
|
||||
multimodal_embeddings += tuple(image_embeddings)
|
||||
if modality == "video":
|
||||
video_embeddings = self._process_video_pixels(multimodal_input)
|
||||
video_input = mm_input_by_modality["video"]
|
||||
assert video_input is not None
|
||||
video_embeddings = self._process_video_pixels(video_input)
|
||||
multimodal_embeddings += tuple(video_embeddings)
|
||||
|
||||
return multimodal_embeddings
|
||||
|
||||
Reference in New Issue
Block a user