[Mypy] Fix typing for M models (#54262)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-01 11:26:00 -04:00
committed by GitHub
parent d9eb4e344f
commit 76f3249fbd
26 changed files with 512 additions and 200 deletions
+41 -9
View File
@@ -22,6 +22,7 @@ from transformers import (
)
from transformers.image_utils import ImageInput
from transformers.video_utils import VideoMetadata
from typing_extensions import TypedDict
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -157,6 +158,11 @@ class Molmo2VideoInputs(TensorSchema):
num_video_tokens: Annotated[torch.Tensor, TensorShape("nv")]
class Molmo2MultiModalInputs(TypedDict, total=False):
images: Molmo2ImageInputs | None
videos: Molmo2VideoInputs | None
@dataclass
class VitConfig:
"""Config for a vision transformer"""
@@ -474,7 +480,7 @@ class Molmo2VisionTransformer(nn.Module):
super().__init__()
scale = config.hidden_size**-0.5
self.num_prefix_tokens: int = 0 # no class embeddings
self.patch_num = config.image_num_patch
self.patch_num: tuple[int, int] = config.image_num_patch
self.positional_embedding = nn.Parameter(
torch.randn(config.image_num_pos, config.hidden_size) * scale,
)
@@ -490,7 +496,7 @@ class Molmo2VisionTransformer(nn.Module):
prefix=maybe_prefix(prefix, "transformer"),
)
def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor:
def add_pos_emb(self, x: torch.Tensor, patch_num: tuple[int, int]) -> torch.Tensor:
pos_emb = self.positional_embedding
pos_emb = pos_emb.reshape(
@@ -522,7 +528,7 @@ class Molmo2VisionTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
patch_num: int | None = None,
patch_num: tuple[int, int] | None = None,
) -> list[torch.Tensor]:
"""
: param x: (batch_size, num_patch, n_pixels)
@@ -652,6 +658,7 @@ class ImagePoolingAttention(nn.Module):
if self.use_pytorch_sdpa:
output = self.forward_sdpa(xq, xk, xv, attn_mask)
else:
assert self.attn is not None
output = self.attn(xq, xk, xv)
output, _ = self.o_proj(output)
@@ -954,12 +961,15 @@ class Molmo2Attention(nn.Module):
q: torch.Tensor,
k: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
assert self.q_norm is not None
assert self.k_norm is not None
if self.tp_size > 1:
q = tensor_model_parallel_all_gather(q.contiguous())
k = tensor_model_parallel_all_gather(k.contiguous())
q = self.q_norm(q)
k = self.k_norm(k)
if self.tp_size > 1:
assert self.tp_rank is not None
splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size)
q = splitter(q)[self.tp_rank]
k = splitter(k)[self.tp_rank]
@@ -1911,10 +1921,10 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
)
videos = [v.copy() for v in videos]
video_items = []
video_items: list[VideoItem] = []
for video in videos:
video_num_frames = video.shape[0]
video_metadata = {
video_metadata: dict[str, int | float | str | bool | list[int]] = {
"fps": 2.0,
"duration": video_num_frames / 2.0,
"total_num_frames": video_num_frames,
@@ -1972,6 +1982,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
image_processor = hf_processor.image_processor
if videos := mm_data.pop("videos", []):
assert isinstance(videos, Sequence)
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
pixel_values_videos_lst = []
@@ -1983,7 +1994,9 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
num_video_tokens_lst = []
for item in videos:
assert isinstance(item, tuple) and len(item) == 2
video_array, metadata = item
assert isinstance(metadata, Mapping)
# NOTE: metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is
@@ -2240,6 +2253,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def get_video_replacement_molmo2(item_idx: int):
video, metadata = mm_items["video"][item_idx]
do_sample_frames = hf_processor_mm_kwargs.get("do_sample_frames")
assert do_sample_frames is None or isinstance(do_sample_frames, bool)
timestamps = self.info._get_video_second_idx(metadata, do_sample_frames)
ncols, nrows = self.info.get_base_grid_size(video_processor)
@@ -2408,10 +2422,17 @@ class Molmo2ForConditionalGeneration(
image_tokens = kwargs.pop("image_tokens", None)
num_image_tokens = kwargs.pop("num_image_tokens", None)
assert isinstance(token_pooling, torch.Tensor)
assert isinstance(num_pooled_patches, torch.Tensor)
assert isinstance(num_patches, torch.Tensor)
assert isinstance(image_tokens, torch.Tensor)
assert isinstance(num_image_tokens, torch.Tensor)
accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist()
patch_offset = 0
new_token_pooling = token_pooling.clone()
for i, n in enumerate(num_pooled_patches):
for i, n_tensor in enumerate(num_pooled_patches):
n = int(n_tensor)
cur_slice = token_pooling[patch_offset : patch_offset + n]
index_offset = int(accum_patches[i])
new_token_pooling[patch_offset : patch_offset + n] = torch.where(
@@ -2443,10 +2464,17 @@ class Molmo2ForConditionalGeneration(
video_tokens = kwargs.pop("video_tokens", None)
num_video_tokens = kwargs.pop("num_video_tokens", None)
assert isinstance(token_pooling, torch.Tensor)
assert isinstance(num_pooled_patches, torch.Tensor)
assert isinstance(num_patches, torch.Tensor)
assert isinstance(video_tokens, torch.Tensor)
assert isinstance(num_video_tokens, torch.Tensor)
accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist()
patch_offset = 0
new_token_pooling = token_pooling.clone()
for i, n in enumerate(num_pooled_patches):
for i, n_tensor in enumerate(num_pooled_patches):
n = int(n_tensor)
cur_slice = token_pooling[patch_offset : patch_offset + n]
index_offset = int(accum_patches[i])
new_token_pooling[patch_offset : patch_offset + n] = torch.where(
@@ -2464,8 +2492,10 @@ class Molmo2ForConditionalGeneration(
num_video_tokens=num_video_tokens,
)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {}
def _parse_and_validate_multimodal_inputs(
self, **kwargs: object
) -> Molmo2MultiModalInputs:
modalities: Molmo2MultiModalInputs = {}
for input_key in kwargs:
if input_key in ("pixel_values",) and "images" not in modalities:
@@ -2544,10 +2574,12 @@ class Molmo2ForConditionalGeneration(
for modality in modalities:
if modality == "images":
image_input = modalities["images"]
assert image_input is not None
image_embeddings = self._process_image_input(image_input)
multimodal_embeddings += image_embeddings
if modality == "videos":
video_input = modalities["videos"]
assert video_input is not None
video_embeddings = self._process_video_input(video_input)
multimodal_embeddings += video_embeddings