[Mypy] Fix typing for M models (#54262)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-01 11:26:00 -04:00
committed by GitHub
parent d9eb4e344f
commit 76f3249fbd
26 changed files with 512 additions and 200 deletions
+21 -7
View File
@@ -45,7 +45,11 @@ from transformers import BatchFeature
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.distributed import (
divide,
get_pp_group,
@@ -241,12 +245,16 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
* int(video_processor.downsample_factor)
* video_grid
)
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
width=image_width,
height=image_height,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
overrides=image_overrides,
),
"video": self._get_dummy_videos(
width=video_size,
@@ -255,7 +263,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
seq_len, mm_counts
),
num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"),
overrides=video_overrides,
),
}
@@ -370,7 +378,7 @@ class MuseGlimmerMultiModalProcessor(
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
fields = {}
fields: dict[str, MultiModalFieldConfig] = {}
if "image_pixel_values" in hf_inputs:
fields.update(
image_pixel_values=MultiModalFieldConfig.batched("image"),
@@ -410,7 +418,9 @@ class MuseGlimmerMultiModalProcessor(
def image_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["image"][item_idx]
num_tokens = int(out_item["image_feature_sizes"].data)
feature_size = out_item["image_feature_sizes"].data
assert isinstance(feature_size, torch.Tensor)
num_tokens = int(feature_size.item())
replacement = (
[image_start_id] + [config.image_token_id] * num_tokens + [image_end_id]
)
@@ -427,7 +437,9 @@ class MuseGlimmerMultiModalProcessor(
"MuseGlimmer video must contain at least one frame group"
)
num_tokens = int(out_item["video_feature_sizes"].data)
feature_size = out_item["video_feature_sizes"].data
assert isinstance(feature_size, torch.Tensor)
num_tokens = int(feature_size.item())
tokens_per_group, remainder = divmod(num_tokens, num_groups)
if remainder:
raise ValueError(
@@ -1477,6 +1489,8 @@ class MuseGlimmerForCausalLM(
prefix=maybe_prefix(prefix, "model"),
)
self.vision_encoder: MuseGlimmerVisionEncoder | None
self.vision_adapter: MuseGlimmerVisionAdapter | None
if self.has_vision:
with self._mark_tower_model(vllm_config, {"image", "video"}):
self.vision_encoder = MuseGlimmerVisionEncoder(
@@ -1616,7 +1630,7 @@ class MuseGlimmerForCausalLM(
image_input = self._parse_and_validate_image_input(**kwargs)
video_input = self._parse_and_validate_video_input(**kwargs)
embeddings: MultiModalEmbeddings = []
embeddings: list[torch.Tensor] = []
for key in kwargs:
if key == "image_pixel_values" and image_input is not None:
embeddings.extend(self._process_image_input(image_input))