[1/N] Reorganize multimodal processing code (#32327)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-14 15:25:31 +00:00
committed by GitHub
parent 552b262936
commit 9ea07b41da
76 changed files with 717 additions and 670 deletions
+9 -5
View File
@@ -33,8 +33,6 @@ if TYPE_CHECKING:
from transformers.feature_extraction_utils import BatchFeature
from .base import MediaWithBytes
from .processing import MultiModalHashes
else:
torch = LazyLoader("torch", globals(), "torch")
@@ -979,9 +977,15 @@ MultiModalKwargsOptionalItems: TypeAlias = (
)
MultiModalHashes = dict[str, list[str]]
"""
A dictionary containing per-item hashes for each modality.
"""
MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]]
"""
A dictionary containing placeholder ranges for each modality.
A dictionary containing per-item placeholder ranges for each modality.
"""
@@ -1001,10 +1005,10 @@ class MultiModalInputs(TypedDict):
mm_kwargs: MultiModalKwargsOptionalItems
"""Keyword arguments to be directly passed to the model after batching."""
mm_hashes: "MultiModalHashes"
mm_hashes: MultiModalHashes
"""The hashes of the multi-modal data."""
mm_placeholders: "MultiModalPlaceholderDict"
mm_placeholders: MultiModalPlaceholderDict
"""
For each modality, information about the placeholder tokens in
`prompt_token_ids`.