[Mypy] Fix typing for M models (#54262)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -22,6 +22,7 @@ from transformers import (
|
||||
)
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.video_utils import VideoMetadata
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -157,6 +158,11 @@ class Molmo2VideoInputs(TensorSchema):
|
||||
num_video_tokens: Annotated[torch.Tensor, TensorShape("nv")]
|
||||
|
||||
|
||||
class Molmo2MultiModalInputs(TypedDict, total=False):
|
||||
images: Molmo2ImageInputs | None
|
||||
videos: Molmo2VideoInputs | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VitConfig:
|
||||
"""Config for a vision transformer"""
|
||||
@@ -474,7 +480,7 @@ class Molmo2VisionTransformer(nn.Module):
|
||||
super().__init__()
|
||||
scale = config.hidden_size**-0.5
|
||||
self.num_prefix_tokens: int = 0 # no class embeddings
|
||||
self.patch_num = config.image_num_patch
|
||||
self.patch_num: tuple[int, int] = config.image_num_patch
|
||||
self.positional_embedding = nn.Parameter(
|
||||
torch.randn(config.image_num_pos, config.hidden_size) * scale,
|
||||
)
|
||||
@@ -490,7 +496,7 @@ class Molmo2VisionTransformer(nn.Module):
|
||||
prefix=maybe_prefix(prefix, "transformer"),
|
||||
)
|
||||
|
||||
def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor:
|
||||
def add_pos_emb(self, x: torch.Tensor, patch_num: tuple[int, int]) -> torch.Tensor:
|
||||
pos_emb = self.positional_embedding
|
||||
|
||||
pos_emb = pos_emb.reshape(
|
||||
@@ -522,7 +528,7 @@ class Molmo2VisionTransformer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
patch_num: int | None = None,
|
||||
patch_num: tuple[int, int] | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""
|
||||
: param x: (batch_size, num_patch, n_pixels)
|
||||
@@ -652,6 +658,7 @@ class ImagePoolingAttention(nn.Module):
|
||||
if self.use_pytorch_sdpa:
|
||||
output = self.forward_sdpa(xq, xk, xv, attn_mask)
|
||||
else:
|
||||
assert self.attn is not None
|
||||
output = self.attn(xq, xk, xv)
|
||||
|
||||
output, _ = self.o_proj(output)
|
||||
@@ -954,12 +961,15 @@ class Molmo2Attention(nn.Module):
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self.q_norm is not None
|
||||
assert self.k_norm is not None
|
||||
if self.tp_size > 1:
|
||||
q = tensor_model_parallel_all_gather(q.contiguous())
|
||||
k = tensor_model_parallel_all_gather(k.contiguous())
|
||||
q = self.q_norm(q)
|
||||
k = self.k_norm(k)
|
||||
if self.tp_size > 1:
|
||||
assert self.tp_rank is not None
|
||||
splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size)
|
||||
q = splitter(q)[self.tp_rank]
|
||||
k = splitter(k)[self.tp_rank]
|
||||
@@ -1911,10 +1921,10 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
|
||||
)
|
||||
videos = [v.copy() for v in videos]
|
||||
|
||||
video_items = []
|
||||
video_items: list[VideoItem] = []
|
||||
for video in videos:
|
||||
video_num_frames = video.shape[0]
|
||||
video_metadata = {
|
||||
video_metadata: dict[str, int | float | str | bool | list[int]] = {
|
||||
"fps": 2.0,
|
||||
"duration": video_num_frames / 2.0,
|
||||
"total_num_frames": video_num_frames,
|
||||
@@ -1972,6 +1982,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
image_processor = hf_processor.image_processor
|
||||
|
||||
if videos := mm_data.pop("videos", []):
|
||||
assert isinstance(videos, Sequence)
|
||||
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
|
||||
|
||||
pixel_values_videos_lst = []
|
||||
@@ -1983,7 +1994,9 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
num_video_tokens_lst = []
|
||||
|
||||
for item in videos:
|
||||
assert isinstance(item, tuple) and len(item) == 2
|
||||
video_array, metadata = item
|
||||
assert isinstance(metadata, Mapping)
|
||||
|
||||
# NOTE: metadata.frames_indices indicates
|
||||
# the sampled frames indices of pre-sampled videos, which is
|
||||
@@ -2240,6 +2253,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
|
||||
def get_video_replacement_molmo2(item_idx: int):
|
||||
video, metadata = mm_items["video"][item_idx]
|
||||
do_sample_frames = hf_processor_mm_kwargs.get("do_sample_frames")
|
||||
assert do_sample_frames is None or isinstance(do_sample_frames, bool)
|
||||
|
||||
timestamps = self.info._get_video_second_idx(metadata, do_sample_frames)
|
||||
ncols, nrows = self.info.get_base_grid_size(video_processor)
|
||||
@@ -2408,10 +2422,17 @@ class Molmo2ForConditionalGeneration(
|
||||
image_tokens = kwargs.pop("image_tokens", None)
|
||||
num_image_tokens = kwargs.pop("num_image_tokens", None)
|
||||
|
||||
assert isinstance(token_pooling, torch.Tensor)
|
||||
assert isinstance(num_pooled_patches, torch.Tensor)
|
||||
assert isinstance(num_patches, torch.Tensor)
|
||||
assert isinstance(image_tokens, torch.Tensor)
|
||||
assert isinstance(num_image_tokens, torch.Tensor)
|
||||
|
||||
accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist()
|
||||
patch_offset = 0
|
||||
new_token_pooling = token_pooling.clone()
|
||||
for i, n in enumerate(num_pooled_patches):
|
||||
for i, n_tensor in enumerate(num_pooled_patches):
|
||||
n = int(n_tensor)
|
||||
cur_slice = token_pooling[patch_offset : patch_offset + n]
|
||||
index_offset = int(accum_patches[i])
|
||||
new_token_pooling[patch_offset : patch_offset + n] = torch.where(
|
||||
@@ -2443,10 +2464,17 @@ class Molmo2ForConditionalGeneration(
|
||||
video_tokens = kwargs.pop("video_tokens", None)
|
||||
num_video_tokens = kwargs.pop("num_video_tokens", None)
|
||||
|
||||
assert isinstance(token_pooling, torch.Tensor)
|
||||
assert isinstance(num_pooled_patches, torch.Tensor)
|
||||
assert isinstance(num_patches, torch.Tensor)
|
||||
assert isinstance(video_tokens, torch.Tensor)
|
||||
assert isinstance(num_video_tokens, torch.Tensor)
|
||||
|
||||
accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist()
|
||||
patch_offset = 0
|
||||
new_token_pooling = token_pooling.clone()
|
||||
for i, n in enumerate(num_pooled_patches):
|
||||
for i, n_tensor in enumerate(num_pooled_patches):
|
||||
n = int(n_tensor)
|
||||
cur_slice = token_pooling[patch_offset : patch_offset + n]
|
||||
index_offset = int(accum_patches[i])
|
||||
new_token_pooling[patch_offset : patch_offset + n] = torch.where(
|
||||
@@ -2464,8 +2492,10 @@ class Molmo2ForConditionalGeneration(
|
||||
num_video_tokens=num_video_tokens,
|
||||
)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = {}
|
||||
def _parse_and_validate_multimodal_inputs(
|
||||
self, **kwargs: object
|
||||
) -> Molmo2MultiModalInputs:
|
||||
modalities: Molmo2MultiModalInputs = {}
|
||||
|
||||
for input_key in kwargs:
|
||||
if input_key in ("pixel_values",) and "images" not in modalities:
|
||||
@@ -2544,10 +2574,12 @@ class Molmo2ForConditionalGeneration(
|
||||
for modality in modalities:
|
||||
if modality == "images":
|
||||
image_input = modalities["images"]
|
||||
assert image_input is not None
|
||||
image_embeddings = self._process_image_input(image_input)
|
||||
multimodal_embeddings += image_embeddings
|
||||
if modality == "videos":
|
||||
video_input = modalities["videos"]
|
||||
assert video_input is not None
|
||||
video_embeddings = self._process_video_input(video_input)
|
||||
multimodal_embeddings += video_embeddings
|
||||
|
||||
|
||||
Reference in New Issue
Block a user