[Core] Use key-only cache for BaseMultiModalProcessor (#23018)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-08-27 14:19:13 +08:00
committed by GitHub
parent 8dbf6ed7be
commit 69244e67e6
29 changed files with 954 additions and 394 deletions
+110 -77
View File
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
Sequence)
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from enum import Enum
from functools import lru_cache
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
@@ -20,12 +20,11 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
encode_tokens)
from vllm.utils import flatten_2d_lists, full_groupby
from .cache import MultiModalCache
from .hasher import MultiModalHasher
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
MultiModalFieldConfig, MultiModalInputs,
MultiModalKwargsItem, MultiModalKwargsItems,
PlaceholderRange)
MultiModalKwargsOptionalItems, PlaceholderRange)
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
MultiModalDataParser)
@@ -34,6 +33,7 @@ if TYPE_CHECKING:
from transformers.feature_extraction_utils import BatchFeature
from transformers.processing_utils import ProcessorMixin
from .cache import BaseMultiModalProcessorCache
from .profiling import BaseDummyInputsBuilder
logger = init_logger(__name__)
@@ -557,6 +557,15 @@ class ResolvedPromptUpdate:
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
def with_target(self, target: UpdateTarget):
return replace(self, target=target)
def with_content(self, content: PromptUpdateInfo):
if not isinstance(content, PromptUpdateDetails):
content = PromptUpdateDetails.from_seq(content)
return replace(self, content=content)
class _TokenMatch(NamedTuple):
start_idx: int
@@ -865,21 +874,6 @@ def find_mm_placeholders(
return dict(full_groupby_modality(it))
class ProcessingCache(MultiModalCache):
def __init__(self, capacity_gb: float) -> None:
super().__init__()
self._cache = self.get_lru_cache(capacity_gb, MultiModalKwargsItem)
self.get = self._cache.get
self.put = self._cache.put
self.reset = self._cache.clear
_CacheItemOrHash = Union[MultiModalKwargsItem, str]
class BaseProcessingInfo:
"""Base class to provide the information necessary for data processing."""
@@ -982,7 +976,7 @@ For an item `MultiModalPromptUpdates[k][i]`,
class MultiModalProcessingInfo(NamedTuple):
kwargs: MultiModalKwargsItems
kwargs: MultiModalKwargsOptionalItems
hashes: MultiModalHashes
prompt_updates: MultiModalPromptUpdates
@@ -994,11 +988,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
Not to be confused with `transformers.ProcessorMixin`.
"""
def __init__(self,
info: _I,
dummy_inputs: "BaseDummyInputsBuilder[_I]",
*,
cache: Optional[ProcessingCache] = None) -> None:
def __init__(
self,
info: _I,
dummy_inputs: "BaseDummyInputsBuilder[_I]",
*,
cache: Optional["BaseMultiModalProcessorCache"] = None,
) -> None:
super().__init__()
self.info = info
@@ -1355,32 +1351,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return prompt_ids, mm_processed_data, False
def _get_cache_missing_items(
self,
cache: ProcessingCache,
mm_data_items: MultiModalDataItems,
mm_hashes: MultiModalHashes,
) -> tuple[dict[str, list[_CacheItemOrHash]], MultiModalDataItems]:
mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]] = {
modality: [(h if (v := cache.get(h)) is None else v)
for h in hashes]
for modality, hashes in mm_hashes.items()
}
mm_missing_idxs = {
modality: [
idx for idx, item_or_hash in enumerate(items_or_hashes)
if isinstance(item_or_hash, str)
]
for modality, items_or_hashes in mm_cache_items_or_hashes.items()
}
mm_missing_data = {
modality: [mm_data_items[modality][idx] for idx in idxs]
for modality, idxs in mm_missing_idxs.items()
}
return mm_cache_items_or_hashes, self._to_mm_items(mm_missing_data)
def _hash_mm_items(
self,
mm_items: MultiModalDataItems,
@@ -1401,28 +1371,92 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
for modality, items in mm_items.items()
}
def _get_cache_missing_items(
self,
cache: "BaseMultiModalProcessorCache",
mm_data_items: MultiModalDataItems,
mm_hashes: MultiModalHashes,
) -> MultiModalDataItems:
mm_is_cached = {
modality: cache.is_cached(hashes)
for modality, hashes in mm_hashes.items()
}
mm_missing_idxs = {
modality: [
idx for idx, item_is_cached in enumerate(items_is_cached)
if not item_is_cached
]
for modality, items_is_cached in mm_is_cached.items()
}
mm_missing_data = {
modality: [mm_data_items[modality][idx] for idx in idxs]
for modality, idxs in mm_missing_idxs.items()
}
return self._to_mm_items(mm_missing_data)
def _recompute_cached_prompt_update(
self,
cached_update: ResolvedPromptUpdate,
new_item_idx: int,
) -> ResolvedPromptUpdate:
"""
Override this if other attributes of `ResolvedPromptUpdate`
also need to be recomputed after retrieving from the cache.
"""
return replace(cached_update, item_idx=new_item_idx)
def _merge_mm_kwargs(
self,
cache: ProcessingCache,
mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]],
cache: "BaseMultiModalProcessorCache",
mm_hashes: MultiModalHashes,
mm_missing_kwargs: MultiModalKwargsItems,
) -> MultiModalKwargsItems:
mm_missing_prompt_updates: MultiModalPromptUpdates,
) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
# Need to calculate this at the beginning to avoid skipping cache logic
# for subsequently repeated items in the same modality
mm_is_cached = {
modality: cache.is_cached(hashes)
for modality, hashes in mm_hashes.items()
}
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
merged_items = defaultdict[str, list[MultiModalKwargsItem]](list)
for modality, items_or_hashes in mm_cache_items_or_hashes.items():
for item_or_hash in items_or_hashes:
if isinstance(item_or_hash, str):
kw_item = mm_missing_kwargs[modality][
mm_missing_next_idx[modality]]
cache.put(item_or_hash, kw_item)
merged_kwargs = defaultdict[str,
list[Optional[MultiModalKwargsItem]]](list)
merged_prompt_updates = defaultdict[
str, list[Sequence[ResolvedPromptUpdate]]](list)
for modality, hashes in mm_hashes.items():
missing_kwargs = mm_missing_kwargs.get(modality, [])
missing_prompt_updates = mm_missing_prompt_updates.get(
modality, [])
for item_idx, item_hash in enumerate(hashes):
kwargs: Optional[MultiModalKwargsItem]
if not mm_is_cached[modality][item_idx]:
missing_next_idx = mm_missing_next_idx[modality]
kwargs = missing_kwargs[missing_next_idx]
updates = missing_prompt_updates[missing_next_idx]
mm_missing_next_idx[modality] += 1
item = kwargs, updates
else:
kw_item = item_or_hash
item = None
merged_items[modality].append(kw_item)
kwargs, updates = cache.get_and_update_item(item, item_hash)
return MultiModalKwargsItems(merged_items)
merged_kwargs[modality].append(kwargs)
merged_prompt_updates[modality].append([
self._recompute_cached_prompt_update(update, item_idx)
for update in updates
])
mm_kwargs = MultiModalKwargsItems(merged_kwargs)
mm_prompt_updates = dict(merged_prompt_updates)
return mm_kwargs, mm_prompt_updates
def _apply_hf_processor(
self,
@@ -1490,10 +1524,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
tokenization_kwargs)
(
mm_cache_items_or_hashes,
mm_missing_data_items,
) = self._get_cache_missing_items(
mm_missing_data_items = self._get_cache_missing_items(
cache=cache,
mm_data_items=mm_data_items,
mm_hashes=mm_hashes,
@@ -1520,16 +1552,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
hf_processor_mm_kwargs),
)
mm_kwargs = self._merge_mm_kwargs(
cache,
mm_cache_items_or_hashes=mm_cache_items_or_hashes,
mm_missing_kwargs=mm_missing_kwargs,
mm_missing_prompt_updates = self._get_mm_prompt_updates(
mm_missing_data_items,
hf_processor_mm_kwargs,
mm_missing_kwargs,
)
mm_prompt_updates = self._get_mm_prompt_updates(
mm_data_items,
hf_processor_mm_kwargs,
mm_kwargs,
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
cache,
mm_hashes=mm_hashes,
mm_missing_kwargs=mm_missing_kwargs,
mm_missing_prompt_updates=mm_missing_prompt_updates,
)
mm_info = MultiModalProcessingInfo(
@@ -1614,7 +1647,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
def _validate_mm_kwargs(
self,
mm_kwargs: MultiModalKwargsItems,
mm_kwargs: MultiModalKwargsOptionalItems,
mm_item_counts: Mapping[str, int],
) -> None:
for modality, item_count in mm_item_counts.items():
@@ -1655,7 +1688,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsItems,
mm_kwargs: MultiModalKwargsOptionalItems,
mm_prompt_updates: MultiModalPromptUpdates,
is_update_applied: bool,
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]: