[VLM] Support caching in merged multi-modal processor (#11396)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+348
-170
@@ -1,6 +1,6 @@
|
||||
import pickle
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import UserDict
|
||||
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
@@ -8,19 +8,18 @@ from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from blake3 import blake3
|
||||
from PIL.Image import Image
|
||||
from transformers import BatchFeature, ProcessorMixin
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.inputs import DummyData, InputProcessingContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
|
||||
from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
|
||||
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby, is_list_of
|
||||
|
||||
from .audio import resample_audio
|
||||
from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
|
||||
MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
|
||||
VideoItem)
|
||||
from .inputs import (MultiModalDataDict, MultiModalDataItems,
|
||||
MultiModalFieldConfig, MultiModalFieldItem,
|
||||
MultiModalInputsV2, MultiModalKwargs, PlaceholderRange)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -201,111 +200,6 @@ class _BoundPromptReplacement:
|
||||
return bound_replacement
|
||||
|
||||
|
||||
class ImageSize(NamedTuple):
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
class MultiModalDataItems(UserDict[str, list[Any]]):
|
||||
"""
|
||||
As :class:`MultiModalDataDict`, but normalized such that each entry
|
||||
corresponds to a list.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
|
||||
"""
|
||||
Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
|
||||
"""
|
||||
multi_data = MultiModalDataItems()
|
||||
|
||||
for k, v in data.items():
|
||||
# TODO: Make a separate modality for embedding inputs
|
||||
# to avoid confusion
|
||||
# yapf: disable
|
||||
if k == "video":
|
||||
# Special case since even a single item can be a list
|
||||
multi_data[k] = ( # type: ignore[index]
|
||||
v if (isinstance(v, torch.Tensor)
|
||||
or is_list_of(v, list)) else [v]
|
||||
)
|
||||
elif k in ("image", "audio"):
|
||||
multi_data[k] = ( # type: ignore[index]
|
||||
v if isinstance(v, (torch.Tensor, list)) else [v]
|
||||
)
|
||||
else:
|
||||
multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
|
||||
# yapf: enable
|
||||
|
||||
return multi_data
|
||||
|
||||
# NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
|
||||
# `self.images` doesn't update this dictionary, which may be confusing
|
||||
# We annotate the getter methods as `Sequence` to prevent others from
|
||||
# trying to update the list in this way
|
||||
@property
|
||||
def images(self) -> Sequence[ImageItem]:
|
||||
return self.get("image", [])
|
||||
|
||||
@property
|
||||
def videos(self) -> Sequence[VideoItem]:
|
||||
return self.get("video", [])
|
||||
|
||||
@property
|
||||
def audios(self) -> Sequence[AudioItem]:
|
||||
return self.get("audio", [])
|
||||
|
||||
def get_item_counts(self) -> Mapping[str, int]:
|
||||
return {m: len(items) for m, items in self.items()}
|
||||
|
||||
def get_image_size(self, item_idx: int) -> ImageSize:
|
||||
image = self.images[item_idx]
|
||||
|
||||
if isinstance(image, Image):
|
||||
return ImageSize(*image.size)
|
||||
if isinstance(image, (np.ndarray, torch.Tensor)):
|
||||
_, h, w = image.shape
|
||||
return ImageSize(w, h)
|
||||
|
||||
assert_never(image)
|
||||
|
||||
def get_audio_with_sr(
|
||||
self,
|
||||
item_idx: int,
|
||||
*,
|
||||
default_sr: float,
|
||||
) -> tuple[np.ndarray, float]:
|
||||
audio = self.audios[item_idx]
|
||||
|
||||
if isinstance(audio, tuple):
|
||||
return audio
|
||||
if isinstance(audio, list):
|
||||
return np.array(audio), default_sr
|
||||
if isinstance(audio, np.ndarray):
|
||||
return audio, default_sr
|
||||
|
||||
assert_never(audio)
|
||||
|
||||
def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
|
||||
"""
|
||||
If :code:`drop_sr=True`, the audio items in this dictionary are updated
|
||||
to be NumPy arrays which implicitly means that their sampling rate is
|
||||
the same as the model's expected sampling rate; otherwise, they remain
|
||||
as :code:`(audio, new_sr)` tuples.
|
||||
"""
|
||||
if not self.audios:
|
||||
return
|
||||
|
||||
new_audios = []
|
||||
for item_idx in range(len(self.audios)):
|
||||
audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
|
||||
audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
|
||||
|
||||
new_audios.append(audio if drop_sr else (audio, new_sr))
|
||||
|
||||
self["audio"] = new_audios
|
||||
|
||||
|
||||
class _TokenMatch(NamedTuple):
|
||||
start_idx: int
|
||||
end_idx: int
|
||||
@@ -583,11 +477,124 @@ def iter_placeholders(
|
||||
)
|
||||
|
||||
|
||||
class ProcessorInputs(NamedTuple):
|
||||
"""Keyword arguments to :meth:`BaseMultiModalProcessor`"""
|
||||
@dataclass
|
||||
class ProcessorInputs:
|
||||
"""Keyword arguments to :meth:`BaseMultiModalProcessor`."""
|
||||
prompt_text: str
|
||||
mm_data: MultiModalDataDict
|
||||
mm_processor_kwargs: Mapping[str, object]
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ProcessingCache:
|
||||
|
||||
def __init__(self, capacity: int) -> None:
|
||||
super().__init__()
|
||||
|
||||
# DEBUG: Set to None to disable
|
||||
self.debug_cache_hit_ratio_steps: Optional[int] = None
|
||||
|
||||
self._cache = LRUCache[str, Mapping[str,
|
||||
MultiModalFieldItem]](capacity)
|
||||
|
||||
def _maybe_log_cache_stats(self) -> None:
|
||||
steps = self.debug_cache_hit_ratio_steps
|
||||
if not steps:
|
||||
return
|
||||
|
||||
cache_stats = self._cache.stat()
|
||||
if cache_stats.total % steps == 0:
|
||||
logger.debug("ProcessingCache: hit_ratio = %.2f",
|
||||
cache_stats.hit_ratio)
|
||||
|
||||
def _serialize_item(self, obj: object) -> bytes:
|
||||
# Simple cases
|
||||
if isinstance(obj, str):
|
||||
return obj.encode("utf-8")
|
||||
if isinstance(obj, bytes):
|
||||
return obj
|
||||
if isinstance(obj, Image):
|
||||
return obj.tobytes()
|
||||
|
||||
# Convertible to NumPy arrays
|
||||
if isinstance(obj, torch.Tensor):
|
||||
obj = obj.numpy()
|
||||
if isinstance(obj, (int, float)):
|
||||
obj = np.array(obj)
|
||||
if isinstance(obj, np.ndarray):
|
||||
return obj.tobytes()
|
||||
|
||||
logger.warning(
|
||||
"No serialization method found for %s. "
|
||||
"Falling back to pickle.", type(obj))
|
||||
|
||||
return pickle.dumps(obj)
|
||||
|
||||
def _item_to_bytes(
|
||||
self,
|
||||
key: str,
|
||||
obj: object,
|
||||
) -> Iterable[tuple[bytes, bytes]]:
|
||||
# Recursive cases
|
||||
if isinstance(obj, (list, tuple)):
|
||||
for i, elem in enumerate(obj):
|
||||
yield from self._item_to_bytes(f"{key}.{i}", elem)
|
||||
elif isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
yield from self._item_to_bytes(f"{key}.{k}", v)
|
||||
else:
|
||||
key_bytes = self._serialize_item(key)
|
||||
value_bytes = self._serialize_item(obj)
|
||||
yield key_bytes, value_bytes
|
||||
|
||||
def _hash_kwargs(self, **kwargs: object) -> str:
|
||||
hasher = blake3()
|
||||
|
||||
for k, v in kwargs.items():
|
||||
for k_bytes, v_bytes in self._item_to_bytes(k, v):
|
||||
hasher.update(k_bytes)
|
||||
hasher.update(v_bytes)
|
||||
|
||||
return hasher.hexdigest()
|
||||
|
||||
def get(
|
||||
self,
|
||||
model_id: str,
|
||||
modality: str,
|
||||
input_item: object,
|
||||
input_kwargs: Mapping[str, object],
|
||||
) -> Optional[Mapping[str, MultiModalFieldItem]]:
|
||||
"""
|
||||
Get a processed multi-modal item from the cache
|
||||
according to its dependencies, including:
|
||||
|
||||
- The model ID
|
||||
- The modality of the item
|
||||
- The original data item passed to the HF processor
|
||||
- The configuration options of the HF processor
|
||||
"""
|
||||
self._maybe_log_cache_stats()
|
||||
|
||||
cache_key = self._hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
return self._cache.get(cache_key)
|
||||
|
||||
def put(
|
||||
self,
|
||||
model_id: str,
|
||||
modality: str,
|
||||
input_item: object,
|
||||
input_kwargs: Mapping[str, object],
|
||||
output_kwargs: Mapping[str, MultiModalFieldItem],
|
||||
) -> None:
|
||||
"""
|
||||
Put a processed multi-modal item into the cache
|
||||
according to its dependencies (see :meth:`get`).
|
||||
"""
|
||||
cache_key = self._hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
self._cache.put(cache_key, output_kwargs)
|
||||
|
||||
|
||||
class BaseMultiModalProcessor(ABC):
|
||||
@@ -595,18 +602,24 @@ class BaseMultiModalProcessor(ABC):
|
||||
Abstract base class to process multi-modal inputs to be used in vLLM.
|
||||
"""
|
||||
|
||||
def __init__(self, ctx: InputProcessingContext) -> None:
|
||||
def __init__(self,
|
||||
ctx: InputProcessingContext,
|
||||
*,
|
||||
cache: Optional[ProcessingCache] = None,
|
||||
enable_sanity_checks: bool = True) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.ctx = ctx
|
||||
self.cache = cache
|
||||
self.enable_sanity_checks = enable_sanity_checks
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: MultiModalDataDict,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> MultiModalInputsV2:
|
||||
return self.apply(prompt, mm_data, mm_processor_kwargs)
|
||||
return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
|
||||
|
||||
def _get_hf_processor(self) -> ProcessorMixin:
|
||||
"""
|
||||
@@ -624,12 +637,21 @@ class BaseMultiModalProcessor(ABC):
|
||||
) -> MultiModalDataItems:
|
||||
return MultiModalDataItems.from_dict(mm_data)
|
||||
|
||||
@abstractmethod
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
"""Given the HF-processed data, output the metadata of each field."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _get_prompt_replacements(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_inputs: BatchFeature,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargs,
|
||||
) -> list[PromptReplacement]:
|
||||
"""
|
||||
Given the original multi-modal items for this modality
|
||||
@@ -651,7 +673,7 @@ class BaseMultiModalProcessor(ABC):
|
||||
return list(
|
||||
iter_placeholders(all_prompt_repls, new_token_ids, mm_item_counts))
|
||||
|
||||
def _get_processor_data(
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
@@ -669,7 +691,7 @@ class BaseMultiModalProcessor(ABC):
|
||||
and v[0].ndim == 2):
|
||||
# Pass through embedding inputs (multi)
|
||||
passthrough_data[f"{k}_embeds"] = v
|
||||
else:
|
||||
elif len(v) > 0:
|
||||
# Map keys to plural form, e.g.: image -> images
|
||||
processor_data[f"{k}s"] = v
|
||||
else:
|
||||
@@ -679,39 +701,181 @@ class BaseMultiModalProcessor(ABC):
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_processor: ProcessorMixin,
|
||||
prompt: str,
|
||||
processor_data: Mapping[str, object],
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
# Not to be confused with `mm_data` in `self.apply`.
|
||||
# This refers to the data to be passed to HF processor.
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
return self.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
prompt,
|
||||
processor_data,
|
||||
mm_processor_kwargs,
|
||||
self._get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **mm_data),
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
def _apply_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
prompt_text: str,
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# some mm_processor_kwargs may be used in processor initialization
|
||||
# instead of processor call
|
||||
hf_processor = self._get_hf_processor(**mm_processor_kwargs)
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[list[int], MultiModalKwargs]:
|
||||
"""
|
||||
Apply the HF processor on the full prompt text and multi-modal data.
|
||||
"""
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
|
||||
|
||||
processor_data, passthrough_data = self._get_processor_data(mm_items)
|
||||
|
||||
hf_inputs = self._call_hf_processor(
|
||||
hf_processor,
|
||||
prompt=prompt,
|
||||
processor_data=processor_data,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
processed_data = self._call_hf_processor(
|
||||
prompt=prompt_text,
|
||||
mm_data=processor_data,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
hf_inputs.update(passthrough_data)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return hf_inputs
|
||||
prompt_ids, = processed_data.pop("input_ids").tolist()
|
||||
|
||||
mm_kwargs = MultiModalKwargs.from_hf_inputs(
|
||||
processed_data,
|
||||
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
|
||||
enable_sanity_checks=self.enable_sanity_checks,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_kwargs
|
||||
|
||||
def _apply_hf_processor_missing(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_missing_data_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
):
|
||||
"""
|
||||
Apply the HF processor on the full prompt text, but only on the
|
||||
multi-modal data that are missing from the cache.
|
||||
|
||||
Note: We pass prompt text and multi-modal data into the HF processor
|
||||
in separate calls to avoid HF prompt replacement being done for
|
||||
cached items; instead, we rely on our own prompt replacement logic
|
||||
for the full text.
|
||||
"""
|
||||
mm_missing_counts = mm_missing_data_items.get_item_counts()
|
||||
|
||||
prompt_ids, _ = self._apply_hf_processor(
|
||||
prompt_text=prompt_text,
|
||||
mm_items=MultiModalDataItems({}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# Some HF processors (e.g. Qwen2-VL) expect corresponding
|
||||
# multi-modal tokens to be in the prompt text
|
||||
dummy_inputs = self._get_dummy_mm_inputs(mm_missing_counts)
|
||||
|
||||
_, mm_missing_kwargs = self._apply_hf_processor(
|
||||
prompt_text=dummy_inputs.prompt_text,
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_missing_kwargs
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[list[int], MultiModalKwargs]:
|
||||
"""
|
||||
Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
"""
|
||||
cache = self.cache
|
||||
model_id = self.ctx.model_config.model
|
||||
|
||||
if cache is None or mm_data_items.has_embedding_inputs():
|
||||
return self._apply_hf_processor(
|
||||
prompt_text=prompt_text,
|
||||
mm_items=mm_data_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
mm_maybe_cached_field_items = {
|
||||
modality: [
|
||||
cache.get(model_id, modality, item, hf_processor_mm_kwargs)
|
||||
for item in items
|
||||
]
|
||||
for modality, items in mm_data_items.items()
|
||||
}
|
||||
|
||||
mm_missing_idxs = {
|
||||
modality: [idx for idx, out in enumerate(fields) if out is None]
|
||||
for modality, fields in mm_maybe_cached_field_items.items()
|
||||
}
|
||||
mm_missing_data = {
|
||||
modality: [mm_data_items[modality][idx] for idx in idxs]
|
||||
for modality, idxs in mm_missing_idxs.items()
|
||||
}
|
||||
mm_missing_data_items = self._get_mm_items(mm_missing_data)
|
||||
|
||||
prompt_ids, mm_missing_kwargs = self._apply_hf_processor_missing(
|
||||
prompt_text=prompt_text,
|
||||
mm_missing_data_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
mm_missing_next_idx = {
|
||||
modality: 0
|
||||
for modality in mm_missing_data_items
|
||||
}
|
||||
|
||||
mm_merged_field_items = dict[str, list[Mapping[str,
|
||||
MultiModalFieldItem]]]()
|
||||
for modality, modal_items_lst in mm_maybe_cached_field_items.items():
|
||||
merged_modal_items_lst = list[Mapping[str, MultiModalFieldItem]]()
|
||||
|
||||
for idx, modal_items in enumerate(modal_items_lst):
|
||||
if modal_items is None:
|
||||
modal_items = mm_missing_kwargs.get_items_by_modality(
|
||||
modality,
|
||||
mm_missing_next_idx[modality],
|
||||
)
|
||||
|
||||
cache.put(
|
||||
model_id,
|
||||
modality,
|
||||
mm_data_items[modality][idx],
|
||||
hf_processor_mm_kwargs,
|
||||
modal_items,
|
||||
)
|
||||
|
||||
mm_missing_next_idx[modality] += 1
|
||||
|
||||
merged_modal_items_lst.append(modal_items)
|
||||
|
||||
mm_merged_field_items[modality] = merged_modal_items_lst
|
||||
|
||||
if self.enable_sanity_checks:
|
||||
mm_missing_counts = mm_missing_data_items.get_item_counts()
|
||||
assert all(
|
||||
item_count == mm_missing_counts[modality]
|
||||
for modality, item_count in mm_missing_next_idx.items()), dict(
|
||||
mm_missing_next_idx=mm_missing_next_idx,
|
||||
mm_missing_counts=mm_missing_counts)
|
||||
|
||||
mm_kwargs = MultiModalKwargs.from_items_by_modality(
|
||||
mm_merged_field_items,
|
||||
enable_sanity_checks=self.enable_sanity_checks,
|
||||
)
|
||||
|
||||
if self.enable_sanity_checks:
|
||||
mm_item_counts = mm_data_items.get_item_counts()
|
||||
|
||||
for modality, item_count in mm_item_counts.items():
|
||||
for item_idx in range(item_count):
|
||||
try:
|
||||
mm_kwargs.get_items_by_modality(modality, item_idx)
|
||||
except Exception as e:
|
||||
# Make it easy to set a breakpoint in the debugger
|
||||
raise e
|
||||
|
||||
return prompt_ids, mm_kwargs
|
||||
|
||||
def _bind_prompt_replacements(
|
||||
self,
|
||||
@@ -730,6 +894,10 @@ class BaseMultiModalProcessor(ABC):
|
||||
tokenizer = self._get_tokenizer()
|
||||
|
||||
token_matches = find_token_matches(token_ids, prompt_repls)
|
||||
mm_match_counts = {
|
||||
modality: len(matches)
|
||||
for modality, matches in full_groupby_modality(token_matches)
|
||||
}
|
||||
|
||||
# If the search text does not represent a special token,
|
||||
# it may have different token IDs in the prompt, because
|
||||
@@ -742,8 +910,8 @@ class BaseMultiModalProcessor(ABC):
|
||||
# of the search text in the prompt, we instead perform string
|
||||
# replacement on the decoded token IDs, then encode them back.
|
||||
if all(
|
||||
len(matches) >= mm_item_counts[modality]
|
||||
for modality, matches in full_groupby_modality(token_matches)
|
||||
mm_match_counts.get(modality, 0) >= item_count
|
||||
for modality, item_count in mm_item_counts.items()
|
||||
): # yapf: disable
|
||||
token_ids = replace_token_matches(
|
||||
token_ids,
|
||||
@@ -775,7 +943,7 @@ class BaseMultiModalProcessor(ABC):
|
||||
self,
|
||||
prompt_text: str,
|
||||
mm_data: MultiModalDataDict,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> MultiModalInputsV2:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
@@ -792,20 +960,24 @@ class BaseMultiModalProcessor(ABC):
|
||||
"""
|
||||
mm_items = self._get_mm_items(mm_data)
|
||||
|
||||
hf_inputs = self._apply_hf_processor(prompt_text, mm_items,
|
||||
mm_processor_kwargs)
|
||||
prompt_ids, = hf_inputs.pop("input_ids").tolist()
|
||||
mm_kwargs = MultiModalKwargs(hf_inputs)
|
||||
prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
|
||||
prompt_text,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
prompt_repls = self._get_prompt_replacements(mm_items, hf_inputs,
|
||||
mm_processor_kwargs)
|
||||
all_prompt_repls = self._bind_prompt_replacements(prompt_repls)
|
||||
unbound_prompt_repls = self._get_prompt_replacements(
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_kwargs,
|
||||
)
|
||||
prompt_repls = self._bind_prompt_replacements(unbound_prompt_repls)
|
||||
|
||||
# If HF processor already inserts placeholder tokens,
|
||||
# there is no need for us to insert them
|
||||
mm_item_counts = mm_items.get_item_counts()
|
||||
all_placeholders = self._find_placeholders(all_prompt_repls,
|
||||
prompt_ids, mm_item_counts)
|
||||
all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
|
||||
mm_item_counts)
|
||||
|
||||
if all_placeholders:
|
||||
tokenizer = self._get_tokenizer()
|
||||
@@ -817,7 +989,7 @@ class BaseMultiModalProcessor(ABC):
|
||||
all_placeholders,
|
||||
) = self._apply_prompt_replacements(
|
||||
prompt_ids,
|
||||
all_prompt_repls,
|
||||
prompt_repls,
|
||||
mm_item_counts,
|
||||
)
|
||||
|
||||
@@ -855,23 +1027,29 @@ class BaseMultiModalProcessor(ABC):
|
||||
from vllm.sequence import SequenceData
|
||||
|
||||
processor_inputs = self._get_dummy_mm_inputs(mm_counts)
|
||||
mm_inputs = self.apply(*processor_inputs)
|
||||
mm_inputs = self.apply(
|
||||
prompt_text=processor_inputs.prompt_text,
|
||||
mm_data=processor_inputs.mm_data,
|
||||
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
prompt_token_ids = mm_inputs["prompt_token_ids"]
|
||||
placeholders_by_modality = mm_inputs["mm_placeholders"]
|
||||
|
||||
total_placeholders_by_modality = dict[str, int]()
|
||||
for modality, placeholders in placeholders_by_modality.items():
|
||||
num_placeholders = sum(item["length"] for item in placeholders)
|
||||
max_tokens = mm_max_tokens[modality]
|
||||
|
||||
if num_placeholders != max_tokens:
|
||||
logger.warning(
|
||||
"The processed dummy data has a total of %d placeholder "
|
||||
"tokens for the '%s' modality, which is not the expected "
|
||||
"%d tokens.", num_placeholders, modality, max_tokens)
|
||||
|
||||
total_placeholders_by_modality[modality] = num_placeholders
|
||||
total_placeholders_by_modality = {
|
||||
modality: sum(item["length"] for item in placeholders)
|
||||
for modality, placeholders in placeholders_by_modality.items()
|
||||
}
|
||||
expected_placeholders_by_modality = {
|
||||
modality: mm_max_tokens[modality]
|
||||
for modality in placeholders_by_modality
|
||||
}
|
||||
if total_placeholders_by_modality != expected_placeholders_by_modality:
|
||||
raise AssertionError(
|
||||
f"The processed dummy data has a total of "
|
||||
f"{total_placeholders_by_modality} placeholder tokens, which "
|
||||
f"is not the expected {expected_placeholders_by_modality} "
|
||||
"tokens.")
|
||||
|
||||
total_len = len(prompt_token_ids)
|
||||
if total_len > seq_len:
|
||||
|
||||
Reference in New Issue
Block a user