[Mypy] Fix mypy typing for model interfaces and H/I models (#54079)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Wentao Ye
parent
74850f9f66
commit
21fa2c5a2a
@@ -10,14 +10,14 @@
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from functools import cached_property
|
||||
from typing import Annotated, Any, Literal, TypeAlias, TypeVar
|
||||
from typing import Annotated, Any, Literal, TypeAlias, TypedDict, TypeVar
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import BatchFeature, PretrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
@@ -128,6 +128,11 @@ class InternVLVideoEmbeddingInputs(TensorSchema):
|
||||
InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs
|
||||
|
||||
|
||||
class InternVLMultiModalInputs(TypedDict, total=False):
|
||||
images: InternVLImageInputs | None
|
||||
videos: InternVLVideoInputs | None
|
||||
|
||||
|
||||
class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||
"""Basic image-only ProcessingInfo for InternVL-style models."""
|
||||
|
||||
@@ -286,6 +291,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
feature_size = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
image_size = images.get_image_size(item_idx)
|
||||
feature_size = self.info.get_num_image_tokens(
|
||||
image_width=image_size.width,
|
||||
@@ -433,6 +439,9 @@ class InternVLDummyInputsBuilder(
|
||||
)
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
video_overrides = mm_options.get("video")
|
||||
assert video_overrides is None or isinstance(
|
||||
video_overrides, VideoDummyOptions
|
||||
)
|
||||
dummy_video = {
|
||||
"video": self._get_dummy_videos(
|
||||
width=image_size,
|
||||
@@ -504,9 +513,9 @@ class InternVLMultiModalProcessor(
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "video_num_patches" in out_mm_data:
|
||||
video_num_patches = out_mm_data["video_num_patches"]
|
||||
assert isinstance(video_num_patches, torch.Tensor)
|
||||
video_num_patches = video_num_patches.tolist()
|
||||
video_num_patches_tensor = out_mm_data["video_num_patches"]
|
||||
assert isinstance(video_num_patches_tensor, torch.Tensor)
|
||||
video_num_patches: list[int] = video_num_patches_tensor.tolist()
|
||||
else:
|
||||
video_num_patches = []
|
||||
|
||||
@@ -591,7 +600,7 @@ class InternVLChatModel(
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
@@ -621,15 +630,15 @@ class InternVLChatModel(
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
)
|
||||
|
||||
self.img_context_token_id = None
|
||||
self.video_context_token_id = None
|
||||
self.img_context_token_id: int | None = None
|
||||
self.video_context_token_id: int | None = None
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _patch_quant_config(
|
||||
self, config: PretrainedConfig, quant_config: QuantizationConfig
|
||||
self, config: PretrainedConfig, quant_config: QuantizationConfig | None
|
||||
):
|
||||
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
||||
# patch the quant_config to add `modules_to_not_convert` back
|
||||
@@ -743,7 +752,7 @@ class InternVLChatModel(
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
) -> InternVLVideoPixelInputs | None:
|
||||
) -> InternVLVideoInputs | None:
|
||||
pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
|
||||
video_num_patches = kwargs.pop("video_num_patches", None)
|
||||
video_embeds = kwargs.pop("image_embeds", None)
|
||||
@@ -804,8 +813,10 @@ class InternVLChatModel(
|
||||
]
|
||||
return image_embeds.split(image_feature_sizes)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = {}
|
||||
def _parse_and_validate_multimodal_inputs(
|
||||
self, **kwargs: object
|
||||
) -> InternVLMultiModalInputs:
|
||||
modalities: InternVLMultiModalInputs = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -834,12 +845,14 @@ class InternVLChatModel(
|
||||
for modality in modalities:
|
||||
if modality == "images":
|
||||
image_input = modalities["images"]
|
||||
image_embeddings = self._process_vision_input(image_input)
|
||||
multimodal_embeddings += tuple(image_embeddings)
|
||||
if image_input is not None:
|
||||
image_embeddings = self._process_vision_input(image_input)
|
||||
multimodal_embeddings += tuple(image_embeddings)
|
||||
if modality == "videos":
|
||||
video_input = modalities["videos"]
|
||||
video_embeddings = self._process_vision_input(video_input)
|
||||
multimodal_embeddings += tuple(video_embeddings)
|
||||
if video_input is not None:
|
||||
video_embeddings = self._process_vision_input(video_input)
|
||||
multimodal_embeddings += tuple(video_embeddings)
|
||||
|
||||
return multimodal_embeddings
|
||||
|
||||
|
||||
Reference in New Issue
Block a user