[Mypy] Fix mypy typing for Transformers models (#54320)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
b78fbab4be
commit
e960ead0aa
@@ -17,17 +17,24 @@
|
||||
"""Transformers modeling backend mixin for multi-modal models."""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers.utils.generic import ModelOutput
|
||||
|
||||
from vllm.compilation.decorators import should_torch_compile_mm_encoder
|
||||
from vllm.config.multimodal import AudioDummyOptions
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
|
||||
from vllm.inputs import (
|
||||
MultiModalDataBuiltins,
|
||||
MultiModalDataDict,
|
||||
MultiModalInput,
|
||||
mm_input,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -62,6 +69,8 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import BatchFeature, PreTrainedModel
|
||||
|
||||
@@ -148,7 +157,7 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
def get_max_image_tokens(self) -> int:
|
||||
width, height = self.get_image_size_with_most_features()
|
||||
processor = self.get_hf_processor()
|
||||
multimodal_config = self.ctx.model_config.multimodal_config
|
||||
multimodal_config = self.ctx.model_config.get_multimodal_config()
|
||||
mm_processor_kwargs = multimodal_config.mm_processor_kwargs or {}
|
||||
mm_tokens = processor._get_num_multimodal_tokens(
|
||||
image_sizes=([height, width],), **mm_processor_kwargs
|
||||
@@ -188,7 +197,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, "BaseDummyOptions"],
|
||||
) -> MultiModalDataDict:
|
||||
data: MultiModalDataDict = {}
|
||||
data = MultiModalDataBuiltins()
|
||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||
sampling_rate = self.info._get_audio_sampling_rate()
|
||||
sub = self.info._get_audio_processor()
|
||||
@@ -196,10 +205,14 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
if chunk_length is None:
|
||||
chunk_length = 30
|
||||
audio_len = int(chunk_length * sampling_rate)
|
||||
audio_overrides = mm_options.get("audio")
|
||||
assert audio_overrides is None or isinstance(
|
||||
audio_overrides, AudioDummyOptions
|
||||
)
|
||||
data["audio"] = self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
overrides=mm_options.get("audio"),
|
||||
overrides=audio_overrides,
|
||||
)
|
||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||
target_width, target_height = self.info.get_image_size_with_most_features()
|
||||
@@ -359,7 +372,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
itself is to process it by itself.
|
||||
"""
|
||||
audios = mm_data.get("audio")
|
||||
if not audios or len({len(audio) for audio in audios}) == 1:
|
||||
if not isinstance(audios, Iterable) or not audios:
|
||||
return
|
||||
if len({len(audio) for audio in audios}) == 1:
|
||||
return
|
||||
|
||||
alone = [
|
||||
@@ -444,7 +459,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
token_id = getattr(processor, names[0], getattr_iter(config, names))
|
||||
if token_id is None:
|
||||
token = getattr(processor, f"{modality}_token", None)
|
||||
token_id = info.get_tokenizer().get_vocab().get(token)
|
||||
if token is not None:
|
||||
token_id = info.get_tokenizer().get_vocab().get(token)
|
||||
if token_id is None:
|
||||
raise ValueError(
|
||||
f"Cannot find {modality}_token_id on processor or model config"
|
||||
@@ -722,13 +738,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
for modality, items in out_mm_kwargs.items():
|
||||
# Popped so they are neither cached nor sent to the model; the updates
|
||||
# they produce are cached alongside the item instead
|
||||
replacements = [
|
||||
PromptUpdateDetails.select_token_id(
|
||||
(ids := item.pop(f"{modality}_replacement_ids").data).tolist(),
|
||||
_get_embed_token_id(ids),
|
||||
replacements = []
|
||||
for item in items:
|
||||
ids = item.pop(f"{modality}_replacement_ids").data
|
||||
assert isinstance(ids, torch.Tensor)
|
||||
replacements.append(
|
||||
PromptUpdateDetails.select_token_id(
|
||||
ids.tolist(), _get_embed_token_id(ids)
|
||||
)
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
token = getattr(hf_processor, f"{modality}_token")
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
@@ -777,7 +795,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
) -> list[int] | None:
|
||||
"""Ask the HF processor how many rows of image data each image produces."""
|
||||
images = mm_data.get("images")
|
||||
if not images:
|
||||
if not isinstance(images, Iterable) or not images:
|
||||
return None
|
||||
try:
|
||||
sizes = [(image.height, image.width) for image in images]
|
||||
@@ -893,7 +911,7 @@ MultiModalProcessor = (
|
||||
)
|
||||
|
||||
|
||||
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip SupportsMRoPE.__init__ and call the next class in MRO
|
||||
super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
@@ -925,7 +943,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
if model_config.skip_tokenizer_init:
|
||||
# Determining the supported modalities needs the HF processor, which in
|
||||
# turn needs a tokenizer
|
||||
mm_config = model_config.multimodal_config
|
||||
mm_config = model_config.get_multimodal_config()
|
||||
if mm_config.mm_encoder_only or any(
|
||||
mm_config.get_limit_per_prompt(modality) == 0
|
||||
for modality in encoder_classes
|
||||
@@ -1029,7 +1047,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
# Exclude MultiModalMixin itself
|
||||
bases = [b for b in bases if b is not MultiModalMixin]
|
||||
|
||||
class LanguageModel(*bases):
|
||||
class LanguageModel(*bases): # type: ignore[misc]
|
||||
def __init__(self, multimodal_model):
|
||||
# Don't call super().__init__() to avoid re-initialization
|
||||
self.__dict__.update(multimodal_model.__dict__)
|
||||
@@ -1203,7 +1221,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
# from `self.get_image_features`
|
||||
if isinstance(features, tuple):
|
||||
return features[0]
|
||||
if isinstance(features, dict):
|
||||
if isinstance(features, ModelOutput):
|
||||
return features.pooler_output
|
||||
return features
|
||||
|
||||
|
||||
Reference in New Issue
Block a user