[Mypy] Fix typing for M models (#54262)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -45,7 +45,11 @@ from transformers import BatchFeature
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.multimodal import (
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
get_pp_group,
|
||||
@@ -241,12 +245,16 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
|
||||
* int(video_processor.downsample_factor)
|
||||
* video_grid
|
||||
)
|
||||
image_overrides = mm_options.get("image")
|
||||
video_overrides = mm_options.get("video")
|
||||
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
|
||||
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
width=image_width,
|
||||
height=image_height,
|
||||
num_images=mm_counts.get("image", 0),
|
||||
overrides=mm_options.get("image"),
|
||||
overrides=image_overrides,
|
||||
),
|
||||
"video": self._get_dummy_videos(
|
||||
width=video_size,
|
||||
@@ -255,7 +263,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
|
||||
seq_len, mm_counts
|
||||
),
|
||||
num_videos=mm_counts.get("video", 0),
|
||||
overrides=mm_options.get("video"),
|
||||
overrides=video_overrides,
|
||||
),
|
||||
}
|
||||
|
||||
@@ -370,7 +378,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
fields = {}
|
||||
fields: dict[str, MultiModalFieldConfig] = {}
|
||||
if "image_pixel_values" in hf_inputs:
|
||||
fields.update(
|
||||
image_pixel_values=MultiModalFieldConfig.batched("image"),
|
||||
@@ -410,7 +418,9 @@ class MuseGlimmerMultiModalProcessor(
|
||||
|
||||
def image_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
num_tokens = int(out_item["image_feature_sizes"].data)
|
||||
feature_size = out_item["image_feature_sizes"].data
|
||||
assert isinstance(feature_size, torch.Tensor)
|
||||
num_tokens = int(feature_size.item())
|
||||
replacement = (
|
||||
[image_start_id] + [config.image_token_id] * num_tokens + [image_end_id]
|
||||
)
|
||||
@@ -427,7 +437,9 @@ class MuseGlimmerMultiModalProcessor(
|
||||
"MuseGlimmer video must contain at least one frame group"
|
||||
)
|
||||
|
||||
num_tokens = int(out_item["video_feature_sizes"].data)
|
||||
feature_size = out_item["video_feature_sizes"].data
|
||||
assert isinstance(feature_size, torch.Tensor)
|
||||
num_tokens = int(feature_size.item())
|
||||
tokens_per_group, remainder = divmod(num_tokens, num_groups)
|
||||
if remainder:
|
||||
raise ValueError(
|
||||
@@ -1477,6 +1489,8 @@ class MuseGlimmerForCausalLM(
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
)
|
||||
|
||||
self.vision_encoder: MuseGlimmerVisionEncoder | None
|
||||
self.vision_adapter: MuseGlimmerVisionAdapter | None
|
||||
if self.has_vision:
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
self.vision_encoder = MuseGlimmerVisionEncoder(
|
||||
@@ -1616,7 +1630,7 @@ class MuseGlimmerForCausalLM(
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
video_input = self._parse_and_validate_video_input(**kwargs)
|
||||
|
||||
embeddings: MultiModalEmbeddings = []
|
||||
embeddings: list[torch.Tensor] = []
|
||||
for key in kwargs:
|
||||
if key == "image_pixel_values" and image_input is not None:
|
||||
embeddings.extend(self._process_image_input(image_input))
|
||||
|
||||
Reference in New Issue
Block a user