[Core] Use key-only cache for BaseMultiModalProcessor (#23018)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+110
-77
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
|
||||
Sequence)
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
|
||||
@@ -20,12 +20,11 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
|
||||
encode_tokens)
|
||||
from vllm.utils import flatten_2d_lists, full_groupby
|
||||
|
||||
from .cache import MultiModalCache
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig, MultiModalInputs,
|
||||
MultiModalKwargsItem, MultiModalKwargsItems,
|
||||
PlaceholderRange)
|
||||
MultiModalKwargsOptionalItems, PlaceholderRange)
|
||||
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
|
||||
MultiModalDataParser)
|
||||
|
||||
@@ -34,6 +33,7 @@ if TYPE_CHECKING:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from .cache import BaseMultiModalProcessorCache
|
||||
from .profiling import BaseDummyInputsBuilder
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -557,6 +557,15 @@ class ResolvedPromptUpdate:
|
||||
|
||||
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
def with_target(self, target: UpdateTarget):
|
||||
return replace(self, target=target)
|
||||
|
||||
def with_content(self, content: PromptUpdateInfo):
|
||||
if not isinstance(content, PromptUpdateDetails):
|
||||
content = PromptUpdateDetails.from_seq(content)
|
||||
|
||||
return replace(self, content=content)
|
||||
|
||||
|
||||
class _TokenMatch(NamedTuple):
|
||||
start_idx: int
|
||||
@@ -865,21 +874,6 @@ def find_mm_placeholders(
|
||||
return dict(full_groupby_modality(it))
|
||||
|
||||
|
||||
class ProcessingCache(MultiModalCache):
|
||||
|
||||
def __init__(self, capacity_gb: float) -> None:
|
||||
super().__init__()
|
||||
|
||||
self._cache = self.get_lru_cache(capacity_gb, MultiModalKwargsItem)
|
||||
|
||||
self.get = self._cache.get
|
||||
self.put = self._cache.put
|
||||
self.reset = self._cache.clear
|
||||
|
||||
|
||||
_CacheItemOrHash = Union[MultiModalKwargsItem, str]
|
||||
|
||||
|
||||
class BaseProcessingInfo:
|
||||
"""Base class to provide the information necessary for data processing."""
|
||||
|
||||
@@ -982,7 +976,7 @@ For an item `MultiModalPromptUpdates[k][i]`,
|
||||
|
||||
|
||||
class MultiModalProcessingInfo(NamedTuple):
|
||||
kwargs: MultiModalKwargsItems
|
||||
kwargs: MultiModalKwargsOptionalItems
|
||||
hashes: MultiModalHashes
|
||||
prompt_updates: MultiModalPromptUpdates
|
||||
|
||||
@@ -994,11 +988,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
Not to be confused with `transformers.ProcessorMixin`.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
info: _I,
|
||||
dummy_inputs: "BaseDummyInputsBuilder[_I]",
|
||||
*,
|
||||
cache: Optional[ProcessingCache] = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
info: _I,
|
||||
dummy_inputs: "BaseDummyInputsBuilder[_I]",
|
||||
*,
|
||||
cache: Optional["BaseMultiModalProcessorCache"] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.info = info
|
||||
@@ -1355,32 +1351,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
return prompt_ids, mm_processed_data, False
|
||||
|
||||
def _get_cache_missing_items(
|
||||
self,
|
||||
cache: ProcessingCache,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_hashes: MultiModalHashes,
|
||||
) -> tuple[dict[str, list[_CacheItemOrHash]], MultiModalDataItems]:
|
||||
mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]] = {
|
||||
modality: [(h if (v := cache.get(h)) is None else v)
|
||||
for h in hashes]
|
||||
for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
|
||||
mm_missing_idxs = {
|
||||
modality: [
|
||||
idx for idx, item_or_hash in enumerate(items_or_hashes)
|
||||
if isinstance(item_or_hash, str)
|
||||
]
|
||||
for modality, items_or_hashes in mm_cache_items_or_hashes.items()
|
||||
}
|
||||
mm_missing_data = {
|
||||
modality: [mm_data_items[modality][idx] for idx in idxs]
|
||||
for modality, idxs in mm_missing_idxs.items()
|
||||
}
|
||||
|
||||
return mm_cache_items_or_hashes, self._to_mm_items(mm_missing_data)
|
||||
|
||||
def _hash_mm_items(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
@@ -1401,28 +1371,92 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
for modality, items in mm_items.items()
|
||||
}
|
||||
|
||||
def _get_cache_missing_items(
|
||||
self,
|
||||
cache: "BaseMultiModalProcessorCache",
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_hashes: MultiModalHashes,
|
||||
) -> MultiModalDataItems:
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes)
|
||||
for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
|
||||
mm_missing_idxs = {
|
||||
modality: [
|
||||
idx for idx, item_is_cached in enumerate(items_is_cached)
|
||||
if not item_is_cached
|
||||
]
|
||||
for modality, items_is_cached in mm_is_cached.items()
|
||||
}
|
||||
mm_missing_data = {
|
||||
modality: [mm_data_items[modality][idx] for idx in idxs]
|
||||
for modality, idxs in mm_missing_idxs.items()
|
||||
}
|
||||
|
||||
return self._to_mm_items(mm_missing_data)
|
||||
|
||||
def _recompute_cached_prompt_update(
|
||||
self,
|
||||
cached_update: ResolvedPromptUpdate,
|
||||
new_item_idx: int,
|
||||
) -> ResolvedPromptUpdate:
|
||||
"""
|
||||
Override this if other attributes of `ResolvedPromptUpdate`
|
||||
also need to be recomputed after retrieving from the cache.
|
||||
"""
|
||||
return replace(cached_update, item_idx=new_item_idx)
|
||||
|
||||
def _merge_mm_kwargs(
|
||||
self,
|
||||
cache: ProcessingCache,
|
||||
mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]],
|
||||
cache: "BaseMultiModalProcessorCache",
|
||||
mm_hashes: MultiModalHashes,
|
||||
mm_missing_kwargs: MultiModalKwargsItems,
|
||||
) -> MultiModalKwargsItems:
|
||||
mm_missing_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]:
|
||||
# Need to calculate this at the beginning to avoid skipping cache logic
|
||||
# for subsequently repeated items in the same modality
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes)
|
||||
for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
|
||||
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
|
||||
|
||||
merged_items = defaultdict[str, list[MultiModalKwargsItem]](list)
|
||||
for modality, items_or_hashes in mm_cache_items_or_hashes.items():
|
||||
for item_or_hash in items_or_hashes:
|
||||
if isinstance(item_or_hash, str):
|
||||
kw_item = mm_missing_kwargs[modality][
|
||||
mm_missing_next_idx[modality]]
|
||||
cache.put(item_or_hash, kw_item)
|
||||
merged_kwargs = defaultdict[str,
|
||||
list[Optional[MultiModalKwargsItem]]](list)
|
||||
merged_prompt_updates = defaultdict[
|
||||
str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
for modality, hashes in mm_hashes.items():
|
||||
missing_kwargs = mm_missing_kwargs.get(modality, [])
|
||||
missing_prompt_updates = mm_missing_prompt_updates.get(
|
||||
modality, [])
|
||||
|
||||
for item_idx, item_hash in enumerate(hashes):
|
||||
kwargs: Optional[MultiModalKwargsItem]
|
||||
if not mm_is_cached[modality][item_idx]:
|
||||
missing_next_idx = mm_missing_next_idx[modality]
|
||||
kwargs = missing_kwargs[missing_next_idx]
|
||||
updates = missing_prompt_updates[missing_next_idx]
|
||||
|
||||
mm_missing_next_idx[modality] += 1
|
||||
|
||||
item = kwargs, updates
|
||||
else:
|
||||
kw_item = item_or_hash
|
||||
item = None
|
||||
|
||||
merged_items[modality].append(kw_item)
|
||||
kwargs, updates = cache.get_and_update_item(item, item_hash)
|
||||
|
||||
return MultiModalKwargsItems(merged_items)
|
||||
merged_kwargs[modality].append(kwargs)
|
||||
merged_prompt_updates[modality].append([
|
||||
self._recompute_cached_prompt_update(update, item_idx)
|
||||
for update in updates
|
||||
])
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems(merged_kwargs)
|
||||
mm_prompt_updates = dict(merged_prompt_updates)
|
||||
|
||||
return mm_kwargs, mm_prompt_updates
|
||||
|
||||
def _apply_hf_processor(
|
||||
self,
|
||||
@@ -1490,10 +1524,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
|
||||
tokenization_kwargs)
|
||||
(
|
||||
mm_cache_items_or_hashes,
|
||||
mm_missing_data_items,
|
||||
) = self._get_cache_missing_items(
|
||||
|
||||
mm_missing_data_items = self._get_cache_missing_items(
|
||||
cache=cache,
|
||||
mm_data_items=mm_data_items,
|
||||
mm_hashes=mm_hashes,
|
||||
@@ -1520,16 +1552,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
mm_kwargs = self._merge_mm_kwargs(
|
||||
cache,
|
||||
mm_cache_items_or_hashes=mm_cache_items_or_hashes,
|
||||
mm_missing_kwargs=mm_missing_kwargs,
|
||||
mm_missing_prompt_updates = self._get_mm_prompt_updates(
|
||||
mm_missing_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_missing_kwargs,
|
||||
)
|
||||
|
||||
mm_prompt_updates = self._get_mm_prompt_updates(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_kwargs,
|
||||
mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs(
|
||||
cache,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_missing_kwargs=mm_missing_kwargs,
|
||||
mm_missing_prompt_updates=mm_missing_prompt_updates,
|
||||
)
|
||||
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
@@ -1614,7 +1647,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
def _validate_mm_kwargs(
|
||||
self,
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_item_counts: Mapping[str, int],
|
||||
) -> None:
|
||||
for modality, item_count in mm_item_counts.items():
|
||||
@@ -1655,7 +1688,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
is_update_applied: bool,
|
||||
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
|
||||
Reference in New Issue
Block a user