[Mypy] Fix typing for M models (#54262)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import (
|
||||
)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||
from vllm.inputs import ModalityData, MultiModalDataDict
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -66,8 +66,12 @@ from .minicpmv import (
|
||||
_MAX_FRAMES_PER_VIDEO,
|
||||
MiniCPMV2_6,
|
||||
MiniCPMV4_5,
|
||||
MiniCPMVBaseModel,
|
||||
MiniCPMVDummyInputsBuilder,
|
||||
MiniCPMVImageEmbeddingInputs,
|
||||
MiniCPMVImagePixelInputs,
|
||||
MiniCPMVMultiModalDataParser,
|
||||
MiniCPMVMultiModalInputs,
|
||||
MiniCPMVMultiModalProcessor,
|
||||
MiniCPMVProcessingInfo,
|
||||
_minicpmv_field_config,
|
||||
@@ -181,6 +185,10 @@ MiniCPMOAudioInputs: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
class MiniCPMOMultiModalInputs(MiniCPMVMultiModalInputs, total=False):
|
||||
audios: MiniCPMOAudioInputs | None
|
||||
|
||||
|
||||
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
audio_features = hf_inputs.get("audio_features")
|
||||
audio_feature_lens = hf_inputs.get("audio_feature_lens")
|
||||
@@ -405,6 +413,7 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
|
||||
)
|
||||
|
||||
audio_overrides = mm_options.get("audio")
|
||||
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||
|
||||
audio_mm_data = {
|
||||
"audio": self._get_dummy_audios(
|
||||
@@ -469,7 +478,8 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
if isinstance(lens, torch.Tensor):
|
||||
flat_feature_lens.extend(lens.flatten().tolist())
|
||||
else:
|
||||
flat_feature_lens.append(int(lens))
|
||||
assert isinstance(lens, int)
|
||||
flat_feature_lens.append(lens)
|
||||
unpadded_audio_features = [
|
||||
feat[:, :length]
|
||||
for feat, length in zip(
|
||||
@@ -515,6 +525,9 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
audios = mm_items.get_items(
|
||||
"audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
|
||||
)
|
||||
assert isinstance(
|
||||
audios, (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
|
||||
)
|
||||
|
||||
if isinstance(audios, MiniCPMOAudioEmbeddingItems):
|
||||
single_audio_embeds = audios.get(item_idx)["audio_embeds"]
|
||||
@@ -652,7 +665,7 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
|
||||
hidden_states, p=self.dropout, training=self.training
|
||||
)
|
||||
|
||||
encoder_states = ()
|
||||
encoder_states: tuple[torch.Tensor, ...] = ()
|
||||
|
||||
for idx, encoder_layer in enumerate(self.layers):
|
||||
encoder_states = encoder_states + (hidden_states,)
|
||||
@@ -682,7 +695,13 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
|
||||
)
|
||||
|
||||
|
||||
class MiniCPMOBaseModel:
|
||||
if TYPE_CHECKING:
|
||||
_MiniCPMOBaseModelBase = MiniCPMVBaseModel
|
||||
else:
|
||||
_MiniCPMOBaseModelBase = object
|
||||
|
||||
|
||||
class MiniCPMOBaseModel(_MiniCPMOBaseModelBase):
|
||||
"""Base mixin class for MiniCPM-O models with audio support."""
|
||||
|
||||
# Unlike the vision-only MiniCPM-V models, audio weights are loaded here.
|
||||
@@ -897,8 +916,12 @@ class MiniCPMOBaseModel:
|
||||
audio_feature_lens=audio_feature_lens,
|
||||
)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = super()._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
def _parse_and_validate_multimodal_inputs(
|
||||
self, **kwargs: object
|
||||
) -> MiniCPMOMultiModalInputs:
|
||||
modalities = MiniCPMOMultiModalInputs(
|
||||
**super()._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
)
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -918,16 +941,33 @@ class MiniCPMOBaseModel:
|
||||
if audio_input["type"] == "audio_embeds":
|
||||
return audio_input["audio_embeds"]
|
||||
|
||||
assert isinstance(audio_input, MiniCPMOAudioFeatureInputs)
|
||||
return self.get_audio_hidden_states(audio_input)
|
||||
|
||||
def _process_multimodal_inputs(self, modalities: dict):
|
||||
multimodal_embeddings = super()._process_multimodal_inputs(modalities)
|
||||
def _process_multimodal_inputs(self, modalities: Mapping[str, object]):
|
||||
base_modalities: MiniCPMVMultiModalInputs = {}
|
||||
image_input = modalities.get("images")
|
||||
video_input = modalities.get("videos")
|
||||
assert image_input is None or isinstance(
|
||||
image_input, (MiniCPMVImagePixelInputs, MiniCPMVImageEmbeddingInputs)
|
||||
)
|
||||
assert video_input is None or isinstance(
|
||||
video_input, (MiniCPMVImagePixelInputs, MiniCPMVImageEmbeddingInputs)
|
||||
)
|
||||
if "images" in modalities:
|
||||
base_modalities["images"] = image_input
|
||||
if "videos" in modalities:
|
||||
base_modalities["videos"] = video_input
|
||||
multimodal_embeddings = super()._process_multimodal_inputs(base_modalities)
|
||||
|
||||
for modality in modalities:
|
||||
if modality == "audios":
|
||||
audio_input = modalities["audios"]
|
||||
audio_embeddings = self._process_audio_input(audio_input)
|
||||
multimodal_embeddings += tuple(audio_embeddings)
|
||||
audio_input = modalities.get("audios")
|
||||
if audio_input is not None:
|
||||
assert isinstance(
|
||||
audio_input,
|
||||
(MiniCPMOAudioFeatureInputs, MiniCPMOAudioEmbeddingInputs),
|
||||
)
|
||||
audio_embeddings = self._process_audio_input(audio_input)
|
||||
multimodal_embeddings += tuple(audio_embeddings)
|
||||
|
||||
return multimodal_embeddings
|
||||
|
||||
@@ -983,7 +1023,7 @@ class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
|
||||
try:
|
||||
version_str = str(config.version)
|
||||
version_parts = version_str.split(".")
|
||||
version = tuple(int(x) for x in version_parts[:2])
|
||||
version_values = tuple(int(x) for x in version_parts[:2])
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"Invalid model version format in config: {config.version}. "
|
||||
@@ -991,17 +1031,21 @@ class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
|
||||
) from e
|
||||
else:
|
||||
# Default to 2.6 for backward compatibility
|
||||
version = (2, 6)
|
||||
version_values = (2, 6)
|
||||
|
||||
# Dispatch class based on version
|
||||
instance_cls = _MINICPMO_SUPPORT_VERSION.get(version)
|
||||
if len(version_values) == 2:
|
||||
version_key = (version_values[0], version_values[1])
|
||||
instance_cls = _MINICPMO_SUPPORT_VERSION.get(version_key)
|
||||
else:
|
||||
instance_cls = None
|
||||
if instance_cls is None:
|
||||
supported_versions = ", ".join(
|
||||
[f"{v[0]}.{v[1]}" for v in sorted(_MINICPMO_SUPPORT_VERSION.keys())]
|
||||
)
|
||||
raise ValueError(
|
||||
f"Currently, MiniCPMO only supports versions "
|
||||
f"{supported_versions}. Got version: {version}"
|
||||
f"{supported_versions}. Got version: {version_values}"
|
||||
)
|
||||
|
||||
return instance_cls(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
Reference in New Issue
Block a user