[Mypy] Fix mypy typing for L models (#54177)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-04 07:50:43 -04:00
committed by GitHub
parent 3ff4f02dfe
commit c615b1fd67
19 changed files with 255 additions and 89 deletions
+55 -18
View File
@@ -24,12 +24,15 @@ from __future__ import annotations
import hashlib
import importlib
from collections.abc import Callable, Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence, Sized
from functools import lru_cache
from typing import (
Annotated,
Any,
Literal,
SupportsIndex,
SupportsInt,
TypedDict,
)
import numpy as np
@@ -45,6 +48,7 @@ from transformers.dynamic_module_utils import (
)
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from typing_extensions import Buffer
from vllm.compilation.decorators import (
should_torch_compile_mm_encoder,
@@ -730,6 +734,11 @@ LlavaOnevision2ImageInputs = (
)
class LlavaOnevision2InputsByModality(TypedDict, total=False):
images: LlavaOnevision2ImageInputs | None
videos: LlavaOnevision2VideoPixelInputs | None
class LlavaOnevision2VisionRotaryEmbedding(nn.Module):
"""3-D rotary frequency constructor with 4:6:6 (T:H:W) split.
@@ -1468,7 +1477,7 @@ class LlavaOnevision2DummyInputsBuilder(
n_img = mm_counts.get("image", 0)
n_vid = mm_counts.get("video", 0)
w, h = self.info.get_image_size_with_most_features()
out: MultiModalDataDict = {}
out: dict[str, object] = {}
if n_img:
out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img)
if n_vid:
@@ -1586,6 +1595,8 @@ class LlavaOnevision2MultiModalProcessor(
# array, or tensor, and ``and <array>`` would raise on the ambiguous
# truth value of a multi-element array.
_videos = mm_data.get("videos")
if _videos is not None and not isinstance(_videos, Sized):
raise TypeError(f"object of type '{type(_videos).__name__}' has no len()")
videos_present = _videos is not None and len(_videos) > 0
codec_video_paths = (
@@ -1666,15 +1677,24 @@ class LlavaOnevision2MultiModalProcessor(
# provided by the backend's ``compute_frames_index_to_sample``; SSRF /
# local-file gating is enforced by the connector before decoding.
if videos_present:
timestamp_decimals = int(
hf_processor_mm_kwargs.get(
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
)
timestamp_decimals = hf_processor_mm_kwargs.get(
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
)
if not isinstance(
timestamp_decimals, (str, Buffer, SupportsInt, SupportsIndex)
):
raise TypeError(
"int() argument must be a string, a bytes-like object "
f"or a real number, not '{type(timestamp_decimals).__name__}'"
)
timestamp_decimals = int(timestamp_decimals)
per_video_frames: list[list[Image.Image]] = []
per_video_timestamps: list[list[float]] = []
for item in mm_data["videos"]:
videos = mm_data["videos"]
if not isinstance(videos, Iterable):
raise TypeError(f"Unsupported videos input: {type(videos)}")
for item in videos:
pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item)
per_video_frames.append(pil_frames)
per_video_timestamps.append(timestamps)
@@ -1902,18 +1922,21 @@ class LlavaOnevision2MultiModalProcessor(
def get_image_replacement(item_idx: int):
out_item = out_mm_kwargs["image"][item_idx]
grid_thw = out_item["image_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
n = int(grid_thw.prod(-1).sum()) // merge_length
return [image_pad_id] * n
def get_video_replacement(item_idx: int):
out_item = out_mm_kwargs["video"][item_idx]
grid_thw = out_item["video_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
is_codec_field = out_item.get("video_is_codec")
is_codec = (
bool(int(is_codec_field.data.item()))
if is_codec_field is not None
else False
)
if is_codec_field is not None:
is_codec_data = is_codec_field.data
assert isinstance(is_codec_data, torch.Tensor)
is_codec = bool(int(is_codec_data.item()))
else:
is_codec = False
tokens: list[int] = []
if is_codec:
@@ -1924,6 +1947,8 @@ class LlavaOnevision2MultiModalProcessor(
# run, emit ``<sec seconds><|vision_start|><pad*N><|vision_end|>\n``.
patch_positions = out_item["patch_positions_videos"].data
fps_t = out_item["codec_fps"].data
assert isinstance(patch_positions, torch.Tensor)
assert isinstance(fps_t, torch.Tensor)
fps = float(int(fps_t.item())) / 1_000_000_000.0
runs = _codec_timestamp_runs(
patch_positions, fps, image_processor.merge_size
@@ -1938,6 +1963,7 @@ class LlavaOnevision2MultiModalProcessor(
tokens.extend(newline_ids)
else:
timestamps = out_item["frame_timestamps"].data
assert isinstance(timestamps, torch.Tensor)
T_total = int(grid_thw.shape[0])
for t in range(T_total):
sec = float(timestamps[t].item())
@@ -2121,7 +2147,7 @@ class LlavaOnevision2ForConditionalGeneration(
grid_thw = image_input["image_grid_thw"]
assert grid_thw.ndim == 2
if image_input["type"] == "image_embeds":
if isinstance(image_input, LlavaOnevision2ImageEmbeddingInputs):
image_embeds = image_input["image_embeds"]
else:
image_embeds = self.visual(
@@ -2166,8 +2192,13 @@ class LlavaOnevision2ForConditionalGeneration(
)
if isinstance(video_num_frames, list):
video_num_frames = torch.cat([v.flatten() for v in video_num_frames])
else:
elif isinstance(video_num_frames, torch.Tensor):
video_num_frames = video_num_frames.flatten()
else:
raise TypeError(
"video_num_frames must be a tensor or a list of tensors, "
f"got {type(video_num_frames)}"
)
return LlavaOnevision2VideoPixelInputs(
type="pixel_values_videos",
pixel_values_videos=pixel_values_videos,
@@ -2202,8 +2233,10 @@ class LlavaOnevision2ForConditionalGeneration(
cursor += n
return video_embeds.split(sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {}
def _parse_and_validate_multimodal_inputs(
self, **kwargs: object
) -> LlavaOnevision2InputsByModality:
modalities = LlavaOnevision2InputsByModality()
for key in kwargs:
if key in ("pixel_values", "image_embeds") and "images" not in modalities:
modalities["images"] = self._parse_and_validate_image_input(**kwargs)
@@ -2221,9 +2254,13 @@ class LlavaOnevision2ForConditionalGeneration(
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
for modality in modalities:
if modality == "images":
multimodal_embeddings += self._process_image_input(modalities["images"])
image_input = modalities["images"]
assert image_input is not None
multimodal_embeddings += self._process_image_input(image_input)
elif modality == "videos":
multimodal_embeddings += self._process_video_input(modalities["videos"])
video_input = modalities["videos"]
assert video_input is not None
multimodal_embeddings += self._process_video_input(video_input)
return multimodal_embeddings
def get_input_embeddings(