[Mypy] Fix typing for M models (#54262)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -41,7 +41,7 @@ from transformers.dynamic_module_utils import (
|
||||
get_class_from_dynamic_module,
|
||||
resolve_trust_remote_code,
|
||||
)
|
||||
from typing_extensions import TypeVar
|
||||
from typing_extensions import TypedDict, TypeVar
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import (
|
||||
@@ -97,7 +97,7 @@ from vllm.transformers_utils.processor import (
|
||||
cached_get_image_processor,
|
||||
)
|
||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
from vllm.utils.collection_utils import flatten_2d_lists
|
||||
from vllm.utils.collection_utils import flatten_2d_lists, is_list_of
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
@@ -386,6 +386,7 @@ class Resampler4_5(Resampler2_5):
|
||||
for i in range(bs):
|
||||
tgt_h, tgt_w = tgt_sizes[i]
|
||||
if temporal_pos_emb:
|
||||
assert temporal_ids_flatten is not None
|
||||
if temporal_ids_flatten[i] == -1:
|
||||
pos_embed_temporal.append(
|
||||
torch.zeros(self.embed_dim, dtype=dtype, device=device)
|
||||
@@ -549,6 +550,11 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
|
||||
return len(self.get(index)["video_image_sizes"])
|
||||
|
||||
|
||||
class MiniCPMVMultiModalInputs(TypedDict, total=False):
|
||||
images: MiniCPMVImageInputs | None
|
||||
videos: MiniCPMVImageInputs | None
|
||||
|
||||
|
||||
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
|
||||
# The per-item sizes are what size the placeholder range.
|
||||
embedding_fields = {
|
||||
@@ -833,6 +839,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
video_overrides = mm_options.get("video")
|
||||
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
|
||||
|
||||
# Convert video overrides to image overrides for per-frame image generation,
|
||||
# and apply num_frames override to num_video_frames.
|
||||
@@ -842,6 +849,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
|
||||
num_video_frames = min(num_video_frames, video_overrides.num_frames)
|
||||
if video_overrides.width or video_overrides.height:
|
||||
video_frame_overrides = ImageDummyOptions(
|
||||
count=video_overrides.count,
|
||||
width=video_overrides.width,
|
||||
height=video_overrides.height,
|
||||
)
|
||||
@@ -1081,6 +1089,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
images = mm_items.get_items(
|
||||
"image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
|
||||
)
|
||||
assert isinstance(
|
||||
images, (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
|
||||
)
|
||||
|
||||
image_size = images.get_image_size(item_idx)
|
||||
|
||||
@@ -1097,6 +1108,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
videos = mm_items.get_items(
|
||||
"video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
|
||||
)
|
||||
assert isinstance(
|
||||
videos, (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
|
||||
)
|
||||
|
||||
frame_size = videos.get_frame_size(item_idx)
|
||||
num_frames = videos.get_num_frames(item_idx)
|
||||
@@ -1199,6 +1213,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
config = vllm_config.model_config.hf_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
assert multimodal_config is not None
|
||||
quant_config = vllm_config.quant_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
super().__init__()
|
||||
@@ -1261,10 +1276,15 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
image_embeds=image_embeds,
|
||||
)
|
||||
|
||||
assert is_list_of(pixel_values, list, check="all")
|
||||
pixel_values_list: list[list[torch.Tensor]] = []
|
||||
for pixel_value in pixel_values:
|
||||
assert is_list_of(pixel_value, torch.Tensor, check="all")
|
||||
pixel_values_list.append(pixel_value)
|
||||
tgt_sizes = kwargs.pop("tgt_sizes")
|
||||
|
||||
num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
|
||||
pixel_values_flat = flatten_bn(pixel_values)
|
||||
num_slices_flat = torch.tensor([len(ps) for ps in pixel_values_list])
|
||||
pixel_values_flat = flatten_2d_lists(pixel_values_list)
|
||||
tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
|
||||
|
||||
return MiniCPMVImagePixelInputs(
|
||||
@@ -1274,8 +1294,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
num_slices=num_slices_flat,
|
||||
)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = {}
|
||||
def _parse_and_validate_multimodal_inputs(
|
||||
self, **kwargs: object
|
||||
) -> MiniCPMVMultiModalInputs:
|
||||
kwargs.pop("modality", None)
|
||||
modalities: MiniCPMVMultiModalInputs = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -1304,12 +1327,13 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["image_embeds"]
|
||||
|
||||
assert isinstance(image_input, MiniCPMVImagePixelInputs)
|
||||
image_features_flat = self.get_vision_hidden_states(image_input)
|
||||
|
||||
num_slices = image_input["num_slices"]
|
||||
return [e.flatten(0, 1) for e in image_features_flat.split(num_slices.tolist())]
|
||||
|
||||
def _process_multimodal_inputs(self, modalities: dict):
|
||||
def _process_multimodal_inputs(self, modalities: MiniCPMVMultiModalInputs):
|
||||
# The result multimodal_embeddings is tuple of tensors, with each
|
||||
# tensor corresponding to a multimodal data item (image or video).
|
||||
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
|
||||
@@ -1319,10 +1343,12 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
for modality in modalities:
|
||||
if modality == "images":
|
||||
image_input = modalities["images"]
|
||||
assert image_input is not None
|
||||
image_embeddings = self._process_vision_input(image_input)
|
||||
multimodal_embeddings += tuple(image_embeddings)
|
||||
if modality == "videos":
|
||||
video_input = modalities["videos"]
|
||||
assert video_input is not None
|
||||
video_embeddings = self._process_vision_input(video_input)
|
||||
multimodal_embeddings += tuple(video_embeddings)
|
||||
|
||||
@@ -1915,23 +1941,27 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
|
||||
|
||||
def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
config = vllm_config.model_config.hf_config
|
||||
version_values: tuple[int, ...]
|
||||
if not hasattr(config, "version"):
|
||||
if config.hidden_size == 2304 and config.query_num == 64:
|
||||
version = (2, 0)
|
||||
version_values = (2, 0)
|
||||
else:
|
||||
version = (2, 5)
|
||||
version_values = (2, 5)
|
||||
else:
|
||||
version = str(config.version).split(".")
|
||||
version = tuple([int(x) for x in version])
|
||||
version_values = tuple(int(x) for x in str(config.version).split("."))
|
||||
# Dispatch class based on version
|
||||
instance_cls = _SUPPORT_VERSION.get(version)
|
||||
if len(version_values) == 2:
|
||||
version_key = (version_values[0], version_values[1])
|
||||
instance_cls = _SUPPORT_VERSION.get(version_key)
|
||||
else:
|
||||
instance_cls = None
|
||||
if instance_cls is None:
|
||||
supported_versions = ", ".join(
|
||||
[f"{v[0]}.{v[1]}" for v in sorted(_SUPPORT_VERSION.keys())]
|
||||
)
|
||||
raise ValueError(
|
||||
f"Currently, MiniCPMV only supports versions "
|
||||
f"{supported_versions}. Got version: {version}"
|
||||
f"{supported_versions}. Got version: {version_values}"
|
||||
)
|
||||
|
||||
# quant_config references base class members,
|
||||
|
||||
Reference in New Issue
Block a user