[Mypy] Fix typing for M models (#54262)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-01 11:26:00 -04:00
committed by GitHub
parent d9eb4e344f
commit 76f3249fbd
26 changed files with 512 additions and 200 deletions
+61 -17
View File
@@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import (
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
@@ -66,8 +66,12 @@ from .minicpmv import (
_MAX_FRAMES_PER_VIDEO,
MiniCPMV2_6,
MiniCPMV4_5,
MiniCPMVBaseModel,
MiniCPMVDummyInputsBuilder,
MiniCPMVImageEmbeddingInputs,
MiniCPMVImagePixelInputs,
MiniCPMVMultiModalDataParser,
MiniCPMVMultiModalInputs,
MiniCPMVMultiModalProcessor,
MiniCPMVProcessingInfo,
_minicpmv_field_config,
@@ -181,6 +185,10 @@ MiniCPMOAudioInputs: TypeAlias = (
)
class MiniCPMOMultiModalInputs(MiniCPMVMultiModalInputs, total=False):
audios: MiniCPMOAudioInputs | None
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
audio_features = hf_inputs.get("audio_features")
audio_feature_lens = hf_inputs.get("audio_feature_lens")
@@ -405,6 +413,7 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
audio_mm_data = {
"audio": self._get_dummy_audios(
@@ -469,7 +478,8 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
if isinstance(lens, torch.Tensor):
flat_feature_lens.extend(lens.flatten().tolist())
else:
flat_feature_lens.append(int(lens))
assert isinstance(lens, int)
flat_feature_lens.append(lens)
unpadded_audio_features = [
feat[:, :length]
for feat, length in zip(
@@ -515,6 +525,9 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
audios = mm_items.get_items(
"audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
)
assert isinstance(
audios, (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
)
if isinstance(audios, MiniCPMOAudioEmbeddingItems):
single_audio_embeds = audios.get(item_idx)["audio_embeds"]
@@ -652,7 +665,7 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
hidden_states, p=self.dropout, training=self.training
)
encoder_states = ()
encoder_states: tuple[torch.Tensor, ...] = ()
for idx, encoder_layer in enumerate(self.layers):
encoder_states = encoder_states + (hidden_states,)
@@ -682,7 +695,13 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
)
class MiniCPMOBaseModel:
if TYPE_CHECKING:
_MiniCPMOBaseModelBase = MiniCPMVBaseModel
else:
_MiniCPMOBaseModelBase = object
class MiniCPMOBaseModel(_MiniCPMOBaseModelBase):
"""Base mixin class for MiniCPM-O models with audio support."""
# Unlike the vision-only MiniCPM-V models, audio weights are loaded here.
@@ -897,8 +916,12 @@ class MiniCPMOBaseModel:
audio_feature_lens=audio_feature_lens,
)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = super()._parse_and_validate_multimodal_inputs(**kwargs)
def _parse_and_validate_multimodal_inputs(
self, **kwargs: object
) -> MiniCPMOMultiModalInputs:
modalities = MiniCPMOMultiModalInputs(
**super()._parse_and_validate_multimodal_inputs(**kwargs)
)
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -918,16 +941,33 @@ class MiniCPMOBaseModel:
if audio_input["type"] == "audio_embeds":
return audio_input["audio_embeds"]
assert isinstance(audio_input, MiniCPMOAudioFeatureInputs)
return self.get_audio_hidden_states(audio_input)
def _process_multimodal_inputs(self, modalities: dict):
multimodal_embeddings = super()._process_multimodal_inputs(modalities)
def _process_multimodal_inputs(self, modalities: Mapping[str, object]):
base_modalities: MiniCPMVMultiModalInputs = {}
image_input = modalities.get("images")
video_input = modalities.get("videos")
assert image_input is None or isinstance(
image_input, (MiniCPMVImagePixelInputs, MiniCPMVImageEmbeddingInputs)
)
assert video_input is None or isinstance(
video_input, (MiniCPMVImagePixelInputs, MiniCPMVImageEmbeddingInputs)
)
if "images" in modalities:
base_modalities["images"] = image_input
if "videos" in modalities:
base_modalities["videos"] = video_input
multimodal_embeddings = super()._process_multimodal_inputs(base_modalities)
for modality in modalities:
if modality == "audios":
audio_input = modalities["audios"]
audio_embeddings = self._process_audio_input(audio_input)
multimodal_embeddings += tuple(audio_embeddings)
audio_input = modalities.get("audios")
if audio_input is not None:
assert isinstance(
audio_input,
(MiniCPMOAudioFeatureInputs, MiniCPMOAudioEmbeddingInputs),
)
audio_embeddings = self._process_audio_input(audio_input)
multimodal_embeddings += tuple(audio_embeddings)
return multimodal_embeddings
@@ -983,7 +1023,7 @@ class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
try:
version_str = str(config.version)
version_parts = version_str.split(".")
version = tuple(int(x) for x in version_parts[:2])
version_values = tuple(int(x) for x in version_parts[:2])
except (ValueError, TypeError) as e:
raise ValueError(
f"Invalid model version format in config: {config.version}. "
@@ -991,17 +1031,21 @@ class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
) from e
else:
# Default to 2.6 for backward compatibility
version = (2, 6)
version_values = (2, 6)
# Dispatch class based on version
instance_cls = _MINICPMO_SUPPORT_VERSION.get(version)
if len(version_values) == 2:
version_key = (version_values[0], version_values[1])
instance_cls = _MINICPMO_SUPPORT_VERSION.get(version_key)
else:
instance_cls = None
if instance_cls is None:
supported_versions = ", ".join(
[f"{v[0]}.{v[1]}" for v in sorted(_MINICPMO_SUPPORT_VERSION.keys())]
)
raise ValueError(
f"Currently, MiniCPMO only supports versions "
f"{supported_versions}. Got version: {version}"
f"{supported_versions}. Got version: {version_values}"
)
return instance_cls(vllm_config=vllm_config, prefix=prefix)