[Mypy] Fix mypy typing for model interfaces and H/I models (#54079)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-08-28 16:49:19 -04:00
committed by GitHub
co-authored by Wentao Ye
parent 74850f9f66
commit 21fa2c5a2a
23 changed files with 222 additions and 137 deletions
+31 -19
View File
@@ -7,7 +7,7 @@
# Licensed under The MIT License [see LICENSE for details]
# --------------------------------------------------------
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, TypeAlias
from typing import Annotated, Literal, TypeAlias, TypedDict
import regex as re
import torch
@@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.interns1_vit import InternS1VisionModel
@@ -148,6 +148,11 @@ class InternS1VideoEmbeddingInputs(TensorSchema):
InternS1VideoInputs: TypeAlias = InternS1VideoPixelInputs | InternS1VideoEmbeddingInputs
class InternS1MultiModalInputs(TypedDict, total=False):
images: InternS1ImageInputs | None
videos: InternS1VideoInputs | None
def resolve_interns1_min_max_num(
min_dynamic_patch: int,
max_dynamic_patch: int,
@@ -312,6 +317,7 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
@@ -474,22 +480,24 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
video_token = hf_processor.video_token
tokenizer = self.info.get_tokenizer()
video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False)
assert len(video_token_id) == 1
video_token_id = video_token_id[0]
video_token_ids = cached_encode(
tokenizer, video_token, add_special_tokens=False
)
assert len(video_token_ids) == 1
video_token_id = video_token_ids[0]
out_mm_data = out_mm_kwargs.get_data()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor)
video_num_patches = video_num_patches.tolist()
video_num_patches_tensor = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches_tensor, torch.Tensor)
video_num_patches: list[int] = video_num_patches_tensor.tolist()
else:
video_num_patches = []
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
image_num_patches = image_num_patches.tolist()
image_num_patches_tensor = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches_tensor, torch.Tensor)
image_num_patches: list[int] = image_num_patches_tensor.tolist()
else:
image_num_patches = []
@@ -602,8 +610,8 @@ class InternS1ForConditionalGeneration(
prefix=maybe_prefix(prefix, "language_model"),
)
self.img_context_token_id = None
self.video_context_token_id = None
self.img_context_token_id: int | None = None
self.video_context_token_id: int | None = None
self.visual_token_mask = None
self.make_empty_intermediate_tensors = (
@@ -756,8 +764,10 @@ class InternS1ForConditionalGeneration(
]
return image_embeds.split(image_feature_sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {}
def _parse_and_validate_multimodal_inputs(
self, **kwargs: object
) -> InternS1MultiModalInputs:
modalities: InternS1MultiModalInputs = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -789,12 +799,14 @@ class InternS1ForConditionalGeneration(
for modality in modalities:
if modality == "images":
image_input = modalities["images"]
image_embeddings = self._process_vision_input(image_input)
multimodal_embeddings += tuple(image_embeddings)
if image_input is not None:
image_embeddings = self._process_vision_input(image_input)
multimodal_embeddings += tuple(image_embeddings)
if modality == "videos":
video_input = modalities["videos"]
video_embeddings = self._process_vision_input(video_input)
multimodal_embeddings += tuple(video_embeddings)
if video_input is not None:
video_embeddings = self._process_vision_input(video_input)
multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings