[Mypy] Fix mypy typing for model interfaces and H/I models (#54079)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-08-28 16:49:19 -04:00
committed by GitHub
co-authored by Wentao Ye
parent 74850f9f66
commit 21fa2c5a2a
23 changed files with 222 additions and 137 deletions
+29 -16
View File
@@ -10,14 +10,14 @@
from abc import abstractmethod
from collections.abc import Iterable, Mapping, Sequence
from functools import cached_property
from typing import Annotated, Any, Literal, TypeAlias, TypeVar
from typing import Annotated, Any, Literal, TypeAlias, TypedDict, TypeVar
import torch
import torch.nn as nn
from transformers import BatchFeature, PretrainedConfig
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
@@ -128,6 +128,11 @@ class InternVLVideoEmbeddingInputs(TensorSchema):
InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs
class InternVLMultiModalInputs(TypedDict, total=False):
images: InternVLImageInputs | None
videos: InternVLVideoInputs | None
class BaseInternVLProcessingInfo(BaseProcessingInfo):
"""Basic image-only ProcessingInfo for InternVL-style models."""
@@ -286,6 +291,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(images, ImageEmbeddingItems):
feature_size = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx)
feature_size = self.info.get_num_image_tokens(
image_width=image_size.width,
@@ -433,6 +439,9 @@ class InternVLDummyInputsBuilder(
)
num_videos = mm_counts.get("video", 0)
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(
video_overrides, VideoDummyOptions
)
dummy_video = {
"video": self._get_dummy_videos(
width=image_size,
@@ -504,9 +513,9 @@ class InternVLMultiModalProcessor(
tokenizer = self.info.get_tokenizer()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor)
video_num_patches = video_num_patches.tolist()
video_num_patches_tensor = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches_tensor, torch.Tensor)
video_num_patches: list[int] = video_num_patches_tensor.tolist()
else:
video_num_patches = []
@@ -591,7 +600,7 @@ class InternVLChatModel(
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config
self.multimodal_config = multimodal_config
@@ -621,15 +630,15 @@ class InternVLChatModel(
prefix=maybe_prefix(prefix, "language_model"),
)
self.img_context_token_id = None
self.video_context_token_id = None
self.img_context_token_id: int | None = None
self.video_context_token_id: int | None = None
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
def _patch_quant_config(
self, config: PretrainedConfig, quant_config: QuantizationConfig
self, config: PretrainedConfig, quant_config: QuantizationConfig | None
):
# the awq models from OpenGVLab missing `modules_to_not_convert`
# patch the quant_config to add `modules_to_not_convert` back
@@ -743,7 +752,7 @@ class InternVLChatModel(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> InternVLVideoPixelInputs | None:
) -> InternVLVideoInputs | None:
pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
video_num_patches = kwargs.pop("video_num_patches", None)
video_embeds = kwargs.pop("image_embeds", None)
@@ -804,8 +813,10 @@ class InternVLChatModel(
]
return image_embeds.split(image_feature_sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {}
def _parse_and_validate_multimodal_inputs(
self, **kwargs: object
) -> InternVLMultiModalInputs:
modalities: InternVLMultiModalInputs = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -834,12 +845,14 @@ class InternVLChatModel(
for modality in modalities:
if modality == "images":
image_input = modalities["images"]
image_embeddings = self._process_vision_input(image_input)
multimodal_embeddings += tuple(image_embeddings)
if image_input is not None:
image_embeddings = self._process_vision_input(image_input)
multimodal_embeddings += tuple(image_embeddings)
if modality == "videos":
video_input = modalities["videos"]
video_embeddings = self._process_vision_input(video_input)
multimodal_embeddings += tuple(video_embeddings)
if video_input is not None:
video_embeddings = self._process_vision_input(video_input)
multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings