[Mypy] Fix mypy typing for Transformers models (#54320)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-17 12:11:31 -04:00
committed by GitHub
co-authored by Harry Mellor
parent b78fbab4be
commit e960ead0aa
16 changed files with 224 additions and 145 deletions
@@ -17,17 +17,24 @@
"""Transformers modeling backend mixin for multi-modal models."""
from collections import defaultdict
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from contextlib import ExitStack, contextmanager
from typing import TYPE_CHECKING, Any
import torch
import transformers
from packaging.version import Version
from transformers.utils.generic import ModelOutput
from vllm.compilation.decorators import should_torch_compile_mm_encoder
from vllm.config.multimodal import AudioDummyOptions
from vllm.config.utils import getattr_iter
from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
from vllm.inputs import (
MultiModalDataBuiltins,
MultiModalDataDict,
MultiModalInput,
mm_input,
)
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
@@ -62,6 +69,8 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
from .base import Base
if TYPE_CHECKING:
from transformers import BatchFeature, PreTrainedModel
@@ -148,7 +157,7 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
def get_max_image_tokens(self) -> int:
width, height = self.get_image_size_with_most_features()
processor = self.get_hf_processor()
multimodal_config = self.ctx.model_config.multimodal_config
multimodal_config = self.ctx.model_config.get_multimodal_config()
mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}
mm_tokens = processor._get_num_multimodal_tokens(
image_sizes=([height, width],), **mm_processor_kwargs
@@ -188,7 +197,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
mm_counts: Mapping[str, int],
mm_options: Mapping[str, "BaseDummyOptions"],
) -> MultiModalDataDict:
data: MultiModalDataDict = {}
data = MultiModalDataBuiltins()
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
sampling_rate = self.info._get_audio_sampling_rate()
sub = self.info._get_audio_processor()
@@ -196,10 +205,14 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
if chunk_length is None:
chunk_length = 30
audio_len = int(chunk_length * sampling_rate)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(
audio_overrides, AudioDummyOptions
)
data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=mm_options.get("audio"),
overrides=audio_overrides,
)
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
target_width, target_height = self.info.get_image_size_with_most_features()
@@ -359,7 +372,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
itself is to process it by itself.
"""
audios = mm_data.get("audio")
if not audios or len({len(audio) for audio in audios}) == 1:
if not isinstance(audios, Iterable) or not audios:
return
if len({len(audio) for audio in audios}) == 1:
return
alone = [
@@ -444,7 +459,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
token_id = getattr(processor, names[0], getattr_iter(config, names))
if token_id is None:
token = getattr(processor, f"{modality}_token", None)
token_id = info.get_tokenizer().get_vocab().get(token)
if token is not None:
token_id = info.get_tokenizer().get_vocab().get(token)
if token_id is None:
raise ValueError(
f"Cannot find {modality}_token_id on processor or model config"
@@ -722,13 +738,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
for modality, items in out_mm_kwargs.items():
# Popped so they are neither cached nor sent to the model; the updates
# they produce are cached alongside the item instead
replacements = [
PromptUpdateDetails.select_token_id(
(ids := item.pop(f"{modality}_replacement_ids").data).tolist(),
_get_embed_token_id(ids),
replacements = []
for item in items:
ids = item.pop(f"{modality}_replacement_ids").data
assert isinstance(ids, torch.Tensor)
replacements.append(
PromptUpdateDetails.select_token_id(
ids.tolist(), _get_embed_token_id(ids)
)
)
for item in items
]
token = getattr(hf_processor, f"{modality}_token")
updates.append(
PromptReplacement(
@@ -777,7 +795,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
) -> list[int] | None:
"""Ask the HF processor how many rows of image data each image produces."""
images = mm_data.get("images")
if not images:
if not isinstance(images, Iterable) or not images:
return None
try:
sizes = [(image.height, image.width) for image in images]
@@ -893,7 +911,7 @@ MultiModalProcessor = (
)
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip SupportsMRoPE.__init__ and call the next class in MRO
super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix)
@@ -925,7 +943,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
if model_config.skip_tokenizer_init:
# Determining the supported modalities needs the HF processor, which in
# turn needs a tokenizer
mm_config = model_config.multimodal_config
mm_config = model_config.get_multimodal_config()
if mm_config.mm_encoder_only or any(
mm_config.get_limit_per_prompt(modality) == 0
for modality in encoder_classes
@@ -1029,7 +1047,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
# Exclude MultiModalMixin itself
bases = [b for b in bases if b is not MultiModalMixin]
class LanguageModel(*bases):
class LanguageModel(*bases): # type: ignore[misc]
def __init__(self, multimodal_model):
# Don't call super().__init__() to avoid re-initialization
self.__dict__.update(multimodal_model.__dict__)
@@ -1203,7 +1221,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
# from `self.get_image_features`
if isinstance(features, tuple):
return features[0]
if isinstance(features, dict):
if isinstance(features, ModelOutput):
return features.pooler_output
return features