[Mypy] Fix mypy typing for L models (#54177)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-04 07:50:43 -04:00
committed by GitHub
parent 3ff4f02dfe
commit c615b1fd67
19 changed files with 255 additions and 89 deletions
+21 -9
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Final, Literal, Protocol, TypeAlias
from typing import Annotated, Literal, Protocol, TypeAlias, TypedDict
import torch
import torch.nn as nn
@@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -123,7 +123,12 @@ LlavaOnevisionMultiInputs: TypeAlias = (
class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
video_token_index: Final[int]
video_token_index: int
class LlavaOnevisionInputsByModality(TypedDict, total=False):
image: LlavaOnevisionImageInputs | None
video: LlavaOnevisionVideoPixelInputs | None
class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
@@ -288,6 +293,7 @@ class LlavaOnevisionDummyInputsBuilder(
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
@@ -417,6 +423,7 @@ class LlavaOnevisionMultiModalProcessor(
if isinstance(videos, VideoEmbeddingItems):
num_video_tokens = videos.get_feature_size(item_idx)
else:
assert isinstance(videos, VideoProcessorItems)
image_size = videos.get_frame_size(item_idx)
num_video_tokens = self.info.get_num_video_tokens(
image_width=image_size.width,
@@ -570,8 +577,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
},
)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
def _parse_and_validate_multimodal_inputs(
self, **kwargs: object
) -> LlavaOnevisionInputsByModality:
mm_input_by_modality = LlavaOnevisionInputsByModality()
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -749,7 +758,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
self,
image_input: LlavaOnevisionImageInputs,
) -> torch.Tensor | list[torch.Tensor]:
if image_input["type"] == "image_embeds":
if isinstance(image_input, LlavaOnevisionImageEmbeddingInputs):
return image_input["data"]
patch_embeddings = self._process_image_pixels(image_input)
@@ -860,12 +869,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
# NOTE: It is important to iterate over the keys in this dictionary
# to preserve the order of the modalities.
for modality in mm_input_by_modality:
multimodal_input = mm_input_by_modality[modality]
if modality == "image":
image_embeddings = self._process_image_input(multimodal_input)
image_input = mm_input_by_modality["image"]
assert image_input is not None
image_embeddings = self._process_image_input(image_input)
multimodal_embeddings += tuple(image_embeddings)
if modality == "video":
video_embeddings = self._process_video_pixels(multimodal_input)
video_input = mm_input_by_modality["video"]
assert video_input is not None
video_embeddings = self._process_video_pixels(video_input)
multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings