Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+173
-156
@@ -3,13 +3,21 @@
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
|
||||
Sequence)
|
||||
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional,
|
||||
Protocol, Union, cast, overload)
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Generic,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Protocol,
|
||||
Union,
|
||||
cast,
|
||||
overload,
|
||||
)
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
@@ -17,20 +25,28 @@ from typing_extensions import TypeVar, assert_never
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
|
||||
encode_tokens)
|
||||
from vllm.utils import (flatten_2d_lists, full_groupby,
|
||||
get_allowed_kwarg_only_overrides)
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
|
||||
from vllm.utils import flatten_2d_lists, full_groupby, get_allowed_kwarg_only_overrides
|
||||
from vllm.utils.jsontree import JSONTree, json_map_leaves
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig, MultiModalInputs,
|
||||
MultiModalKwargsItem, MultiModalKwargsItems,
|
||||
MultiModalKwargsOptionalItems, MultiModalUUIDDict,
|
||||
PlaceholderRange)
|
||||
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
|
||||
MultiModalDataParser)
|
||||
from .inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalInputs,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalKwargsOptionalItems,
|
||||
MultiModalUUIDDict,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from .parse import (
|
||||
DictEmbeddingItems,
|
||||
EmbeddingItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
@@ -57,9 +73,7 @@ def _cached_encode(
|
||||
*,
|
||||
add_special_tokens: Optional[bool] = None,
|
||||
) -> list[int]:
|
||||
return encode_tokens(tokenizer,
|
||||
text,
|
||||
add_special_tokens=add_special_tokens)
|
||||
return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
@@ -69,9 +83,9 @@ def _cached_decode(
|
||||
*,
|
||||
skip_special_tokens: Optional[bool] = None,
|
||||
) -> str:
|
||||
return decode_tokens(tokenizer,
|
||||
list(token_ids),
|
||||
skip_special_tokens=skip_special_tokens)
|
||||
return decode_tokens(
|
||||
tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
|
||||
|
||||
def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:
|
||||
@@ -89,24 +103,22 @@ def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:
|
||||
|
||||
|
||||
class _GetMatchIndex(Protocol):
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
prompt: PromptSeq,
|
||||
start_idx: int = 0,
|
||||
) -> Optional[int]:
|
||||
...
|
||||
) -> Optional[int]: ...
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromptIndex:
|
||||
"""Resolves to an index in the prompt."""
|
||||
|
||||
get_match_index: _GetMatchIndex
|
||||
|
||||
|
||||
class PromptIndexTargets:
|
||||
|
||||
@staticmethod
|
||||
def start() -> PromptIndex:
|
||||
"""
|
||||
@@ -139,9 +151,7 @@ class PromptIndexTargets:
|
||||
else:
|
||||
if isinstance(prefix, str):
|
||||
# Make both `list[int]`
|
||||
prefix = encode_tokens(tokenizer,
|
||||
prefix,
|
||||
add_special_tokens=False)
|
||||
prefix = encode_tokens(tokenizer, prefix, add_special_tokens=False)
|
||||
|
||||
match_idx = len(prefix)
|
||||
return match_idx if prompt[:match_idx] == prefix else None
|
||||
@@ -181,8 +191,7 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
full: _S
|
||||
"""The full content."""
|
||||
|
||||
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq],
|
||||
torch.Tensor]] = None
|
||||
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq], torch.Tensor]] = None
|
||||
"""
|
||||
Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
|
||||
return a boolean mask of shape `(len(full),)` indicating which positions
|
||||
@@ -203,7 +212,6 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
seq: _S,
|
||||
embed_text: str,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
|
||||
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
|
||||
embed_token_ids = encode_tokens(tokenizer, embed_text)
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
@@ -220,7 +228,6 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
seq: _S,
|
||||
embed_token_id: int,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
|
||||
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
@@ -238,8 +245,7 @@ use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
|
||||
specify which part.
|
||||
"""
|
||||
|
||||
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
|
||||
PromptUpdateInfo]
|
||||
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo], PromptUpdateInfo]
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
@@ -408,11 +414,13 @@ class PromptReplacement(PromptUpdate):
|
||||
modality="image",
|
||||
target="<image>",
|
||||
replacement=PromptUpdateDetails(
|
||||
full="".join([
|
||||
"<image_bos>",
|
||||
"<image>" * image_feature_size,
|
||||
"<image_eos>",
|
||||
]),
|
||||
full="".join(
|
||||
[
|
||||
"<image_bos>",
|
||||
"<image>" * image_feature_size,
|
||||
"<image_eos>",
|
||||
]
|
||||
),
|
||||
features="<image>" * image_feature_size,
|
||||
),
|
||||
)
|
||||
@@ -426,8 +434,9 @@ class PromptReplacement(PromptUpdate):
|
||||
modality="image",
|
||||
target=[image_token_id],
|
||||
replacement=PromptUpdateDetails(
|
||||
full=([image_bos_id] + [image_token_id] * image_feature_size
|
||||
+ [image_eos_id]),
|
||||
full=(
|
||||
[image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
|
||||
),
|
||||
features=[image_token_id] * image_feature_size,
|
||||
),
|
||||
)
|
||||
@@ -459,10 +468,8 @@ class _HasModalityAttr(Protocol):
|
||||
|
||||
|
||||
class _HasModalityProp(Protocol):
|
||||
|
||||
@property
|
||||
def modality(self) -> str:
|
||||
...
|
||||
def modality(self) -> str: ...
|
||||
|
||||
|
||||
_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
|
||||
@@ -520,9 +527,7 @@ class ResolvedPromptUpdate:
|
||||
|
||||
target_token_ids = _seq2tokens(tokenizer, target)
|
||||
|
||||
for match in iter_token_matches(prompt,
|
||||
target_token_ids,
|
||||
start_idx=start_idx):
|
||||
for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
|
||||
yield PromptTargetMatch(match.start_idx, match.end_idx)
|
||||
|
||||
def iter_text_matches(
|
||||
@@ -544,8 +549,7 @@ class ResolvedPromptUpdate:
|
||||
|
||||
target_text = _seq2text(tokenizer, target)
|
||||
|
||||
for match in re.finditer(re.escape(target_text), prompt,
|
||||
pos=start_idx):
|
||||
for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
|
||||
yield PromptTargetMatch(match.start(), match.end())
|
||||
|
||||
def iter_matches(
|
||||
@@ -557,9 +561,7 @@ class ResolvedPromptUpdate:
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
"""Yield each instance of `self.target` found in `prompt`."""
|
||||
if isinstance(prompt, str):
|
||||
return self.iter_text_matches(prompt,
|
||||
tokenizer,
|
||||
start_idx=start_idx)
|
||||
return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
@@ -680,9 +682,9 @@ def _find_matches(
|
||||
break # Already found a match for this item
|
||||
|
||||
for match in update.iter_matches(
|
||||
prompt,
|
||||
tokenizer,
|
||||
start_idx=prev_end_idx,
|
||||
prompt,
|
||||
tokenizer,
|
||||
start_idx=prev_end_idx,
|
||||
):
|
||||
# All matches should share the same mode
|
||||
if mode is None:
|
||||
@@ -723,8 +725,7 @@ def _apply_matches(
|
||||
|
||||
out_seqs = list[Union[str, list[int]]]()
|
||||
out_result: MultiModalPromptUpdatesApplyResult = {
|
||||
m: [None] * len(items)
|
||||
for m, items in mm_prompt_updates.items()
|
||||
m: [None] * len(items) for m, items in mm_prompt_updates.items()
|
||||
}
|
||||
|
||||
start_idx = prev_end_idx = 0
|
||||
@@ -743,8 +744,7 @@ def _apply_matches(
|
||||
for (modality, item_idx), (match, update_idx) in matches_to_apply:
|
||||
found = True
|
||||
|
||||
matched_update = mm_prompt_updates[modality][item_idx][
|
||||
update_idx]
|
||||
matched_update = mm_prompt_updates[modality][item_idx][update_idx]
|
||||
matched_content = matched_update.content.full
|
||||
|
||||
if mode == UpdateMode.INSERT:
|
||||
@@ -756,9 +756,10 @@ def _apply_matches(
|
||||
|
||||
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
|
||||
out_seqs.append(
|
||||
_seq2text(tokenizer, matched_content
|
||||
) if isinstance(prompt, str) else _seq2tokens(
|
||||
tokenizer, matched_content))
|
||||
_seq2text(tokenizer, matched_content)
|
||||
if isinstance(prompt, str)
|
||||
else _seq2tokens(tokenizer, matched_content)
|
||||
)
|
||||
out_result[modality][item_idx] = update_idx
|
||||
|
||||
# Exclude overlapping matches
|
||||
@@ -784,8 +785,7 @@ def apply_token_matches(
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
"""
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates,
|
||||
tokenizer)
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
|
||||
return flatten_2d_lists(token_id_seqs), result
|
||||
|
||||
@@ -847,8 +847,7 @@ def _iter_placeholders(
|
||||
if prompt[start_idx:end_idx_full] == content_tokens_full:
|
||||
content_is_embed = content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(
|
||||
tokenizer, content.full)
|
||||
content_is_embed = content_is_embed(tokenizer, content.full)
|
||||
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
@@ -899,16 +898,14 @@ class InputProcessingContext:
|
||||
"""The tokenizer used to tokenize the inputs."""
|
||||
|
||||
@overload
|
||||
def get_hf_config(self, /) -> "PretrainedConfig":
|
||||
...
|
||||
def get_hf_config(self, /) -> "PretrainedConfig": ...
|
||||
|
||||
@overload
|
||||
def get_hf_config(
|
||||
self,
|
||||
typ: Union[type[_C], tuple[type[_C], ...]],
|
||||
/,
|
||||
) -> _C:
|
||||
...
|
||||
) -> _C: ...
|
||||
|
||||
def get_hf_config(
|
||||
self,
|
||||
@@ -930,9 +927,11 @@ class InputProcessingContext:
|
||||
|
||||
hf_config = self.model_config.hf_config
|
||||
if not isinstance(hf_config, typ):
|
||||
raise TypeError("Invalid type of HuggingFace config. "
|
||||
f"Expected type: {typ}, but "
|
||||
f"found type: {type(hf_config)}")
|
||||
raise TypeError(
|
||||
"Invalid type of HuggingFace config. "
|
||||
f"Expected type: {typ}, but "
|
||||
f"found type: {type(hf_config)}"
|
||||
)
|
||||
|
||||
return hf_config
|
||||
|
||||
@@ -956,8 +955,7 @@ class InputProcessingContext:
|
||||
return mm_config
|
||||
|
||||
@overload
|
||||
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin":
|
||||
...
|
||||
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin": ...
|
||||
|
||||
@overload
|
||||
def get_hf_processor(
|
||||
@@ -965,8 +963,7 @@ class InputProcessingContext:
|
||||
typ: Union[type[_P], tuple[type[_P], ...]],
|
||||
/,
|
||||
**kwargs: object,
|
||||
) -> _P:
|
||||
...
|
||||
) -> _P: ...
|
||||
|
||||
def get_hf_processor(
|
||||
self,
|
||||
@@ -1017,7 +1014,6 @@ class InputProcessingContext:
|
||||
self,
|
||||
output: JSONTree,
|
||||
) -> JSONTree:
|
||||
|
||||
def _postprocess_one(x: object):
|
||||
if isinstance(x, torch.Tensor): # noqa: SIM102
|
||||
# This mimics the behavior of transformers.BatchFeature
|
||||
@@ -1054,17 +1050,21 @@ class InputProcessingContext:
|
||||
)
|
||||
|
||||
try:
|
||||
output = hf_processor(**data,
|
||||
**allowed_kwargs,
|
||||
return_tensors="pt")
|
||||
output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
|
||||
except Exception as exc:
|
||||
# See https://github.com/huggingface/tokenizers/issues/537
|
||||
if (isinstance(exc, RuntimeError) and exc
|
||||
and exc.args[0] == "Already borrowed"
|
||||
and num_tries < max_tries):
|
||||
if (
|
||||
isinstance(exc, RuntimeError)
|
||||
and exc
|
||||
and exc.args[0] == "Already borrowed"
|
||||
and num_tries < max_tries
|
||||
):
|
||||
logger.warning(
|
||||
"Failed to acquire tokenizer in current thread. "
|
||||
"Retrying (%d/%d)...", num_tries, max_tries)
|
||||
"Retrying (%d/%d)...",
|
||||
num_tries,
|
||||
max_tries,
|
||||
)
|
||||
time.sleep(0.5)
|
||||
return self.call_hf_processor(
|
||||
hf_processor,
|
||||
@@ -1074,8 +1074,10 @@ class InputProcessingContext:
|
||||
max_tries=max_tries,
|
||||
)
|
||||
|
||||
msg = (f"Failed to apply {type(hf_processor).__name__} "
|
||||
f"on data={data} with kwargs={allowed_kwargs}")
|
||||
msg = (
|
||||
f"Failed to apply {type(hf_processor).__name__} "
|
||||
f"on data={data} with kwargs={allowed_kwargs}"
|
||||
)
|
||||
|
||||
raise ValueError(msg) from exc
|
||||
|
||||
@@ -1142,8 +1144,11 @@ class BaseProcessingInfo:
|
||||
for modality, supported_limit in supported_mm_limits.items():
|
||||
user_limit = mm_config.get_limit_per_prompt(modality)
|
||||
|
||||
allowed_limits[modality] = (user_limit if supported_limit is None
|
||||
else min(user_limit, supported_limit))
|
||||
allowed_limits[modality] = (
|
||||
user_limit
|
||||
if supported_limit is None
|
||||
else min(user_limit, supported_limit)
|
||||
)
|
||||
|
||||
return allowed_limits
|
||||
|
||||
@@ -1154,7 +1159,7 @@ class BaseProcessingInfo:
|
||||
) -> Optional[Mapping[str, int]]:
|
||||
"""
|
||||
Return the maximum number of tokens per item of for each modality.
|
||||
|
||||
|
||||
When `None` (the default) is returned, vLLM will generate dummy inputs
|
||||
(images/videos) at maximum possible sizes and process them to determine
|
||||
the maximum token count per modality.
|
||||
@@ -1165,7 +1170,7 @@ class BaseProcessingInfo:
|
||||
counts, avoiding the need for dummy input generation and processing.
|
||||
|
||||
Note:
|
||||
The maximum number of tokens per item of each modality returned
|
||||
The maximum number of tokens per item of each modality returned
|
||||
from this function should respect the model's maximum sequence
|
||||
length and the maximum number of items of each modality allowed,
|
||||
and agree with dummy inputs (images/videos) at maximum possible
|
||||
@@ -1245,10 +1250,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
*,
|
||||
mm_uuids: Optional[MultiModalUUIDDict] = None,
|
||||
) -> MultiModalInputs:
|
||||
return self.apply(prompt,
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_uuids=mm_uuids)
|
||||
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
|
||||
|
||||
def _get_data_parser(self) -> MultiModalDataParser:
|
||||
"""
|
||||
@@ -1276,8 +1278,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
limit = min(supported_limit, allowed_limit)
|
||||
|
||||
if num_items > limit:
|
||||
msg = (f"At most {limit} {modality}(s) may be provided in "
|
||||
"one prompt.")
|
||||
msg = f"At most {limit} {modality}(s) may be provided in one prompt."
|
||||
|
||||
if num_items <= supported_limit:
|
||||
msg += " Set `--limit-mm-per-prompt` to increase this limit."
|
||||
@@ -1339,8 +1340,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_item_counts: Mapping[str, int],
|
||||
) -> MultiModalPromptUpdates:
|
||||
return {
|
||||
modality: [[update.resolve(item_idx) for update in updates]
|
||||
for item_idx in range(mm_item_counts.get(modality, 0))]
|
||||
modality: [
|
||||
[update.resolve(item_idx) for update in updates]
|
||||
for item_idx in range(mm_item_counts.get(modality, 0))
|
||||
]
|
||||
for modality, updates in full_groupby_modality(prompt_updates)
|
||||
}
|
||||
|
||||
@@ -1385,8 +1388,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates,
|
||||
tokenizer)
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
@@ -1436,7 +1438,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
return not any(
|
||||
isinstance(items, (EmbeddingItems, DictEmbeddingItems))
|
||||
for items in mm_items.values())
|
||||
for items in mm_items.values()
|
||||
)
|
||||
|
||||
def _apply_hf_processor_text_mm(
|
||||
self,
|
||||
@@ -1461,7 +1464,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
prompt_ids, = processed_data.pop("input_ids").tolist()
|
||||
(prompt_ids,) = processed_data.pop("input_ids").tolist()
|
||||
|
||||
is_update_applied = self._hf_processor_applies_updates(
|
||||
prompt_text=prompt_text,
|
||||
@@ -1564,8 +1567,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
prompt_ids = self._apply_hf_processor_text_only(
|
||||
prompt, tokenization_kwargs)
|
||||
prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
|
||||
else:
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
||||
|
||||
@@ -1611,10 +1613,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
|
||||
# are provided. This is because the processed multimodal
|
||||
# inputs can be different depending on the processor kwargs.
|
||||
if item_uuid is None or \
|
||||
hf_processor_mm_kwargs or \
|
||||
tokenization_kwargs:
|
||||
|
||||
if (
|
||||
item_uuid is None
|
||||
or hf_processor_mm_kwargs
|
||||
or tokenization_kwargs
|
||||
):
|
||||
# NOTE: use provided hash string to hash with kwargs
|
||||
# if available for better performance.
|
||||
item = item_uuid if item_uuid is not None else item
|
||||
@@ -1623,16 +1626,20 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
**tokenization_kwargs))
|
||||
**tokenization_kwargs,
|
||||
)
|
||||
)
|
||||
else:
|
||||
computed.append(item_uuid)
|
||||
hashes[modality] = computed
|
||||
else:
|
||||
hashes[modality] = [
|
||||
MultiModalHasher.hash_kwargs(model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
**tokenization_kwargs)
|
||||
MultiModalHasher.hash_kwargs(
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
**tokenization_kwargs,
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
|
||||
@@ -1645,13 +1652,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_hashes: MultiModalHashes,
|
||||
) -> MultiModalDataItems:
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes)
|
||||
for modality, hashes in mm_hashes.items()
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
|
||||
mm_missing_idxs = {
|
||||
modality: [
|
||||
idx for idx, item_is_cached in enumerate(items_is_cached)
|
||||
idx
|
||||
for idx, item_is_cached in enumerate(items_is_cached)
|
||||
if not item_is_cached
|
||||
]
|
||||
for modality, items_is_cached in mm_is_cached.items()
|
||||
@@ -1664,7 +1671,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
f"Cache miss for {modality} at index {idx} "
|
||||
f"but data is not provided.")
|
||||
f"but data is not provided."
|
||||
)
|
||||
else:
|
||||
missing_modality_data.append(data)
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
@@ -1692,20 +1700,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# Need to calculate this at the beginning to avoid skipping cache logic
|
||||
# for subsequently repeated items in the same modality
|
||||
mm_is_cached = {
|
||||
modality: cache.is_cached(hashes)
|
||||
for modality, hashes in mm_hashes.items()
|
||||
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
|
||||
}
|
||||
|
||||
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
|
||||
|
||||
merged_kwargs = defaultdict[str,
|
||||
list[Optional[MultiModalKwargsItem]]](list)
|
||||
merged_prompt_updates = defaultdict[
|
||||
str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
merged_kwargs = defaultdict[str, list[Optional[MultiModalKwargsItem]]](list)
|
||||
merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
|
||||
list
|
||||
)
|
||||
for modality, hashes in mm_hashes.items():
|
||||
missing_kwargs = mm_missing_kwargs.get(modality, [])
|
||||
missing_prompt_updates = mm_missing_prompt_updates.get(
|
||||
modality, [])
|
||||
missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
|
||||
|
||||
for item_idx, item_hash in enumerate(hashes):
|
||||
kwargs: Optional[MultiModalKwargsItem]
|
||||
@@ -1723,10 +1729,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
kwargs, updates = cache.get_and_update_item(item, item_hash)
|
||||
|
||||
merged_kwargs[modality].append(kwargs)
|
||||
merged_prompt_updates[modality].append([
|
||||
self._recompute_cached_prompt_update(update, item_idx)
|
||||
for update in updates
|
||||
])
|
||||
merged_prompt_updates[modality].append(
|
||||
[
|
||||
self._recompute_cached_prompt_update(update, item_idx)
|
||||
for update in updates
|
||||
]
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems(merged_kwargs)
|
||||
mm_prompt_updates = dict(merged_prompt_updates)
|
||||
@@ -1756,15 +1764,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
mm_processed_data,
|
||||
self._get_mm_fields_config(mm_processed_data,
|
||||
hf_processor_mm_kwargs),
|
||||
self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
mm_hashes = self._hash_mm_items(mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids)
|
||||
mm_hashes = self._hash_mm_items(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_prompt_updates = self._get_mm_prompt_updates(
|
||||
mm_data_items,
|
||||
@@ -1805,10 +1814,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_hashes = self._hash_mm_items(mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids)
|
||||
mm_hashes = self._hash_mm_items(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
mm_uuids=mm_uuids,
|
||||
)
|
||||
|
||||
mm_missing_data_items = self._get_cache_missing_items(
|
||||
cache=cache,
|
||||
@@ -1833,8 +1844,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
mm_missing_processed_data,
|
||||
self._get_mm_fields_config(mm_missing_processed_data,
|
||||
hf_processor_mm_kwargs),
|
||||
self._get_mm_fields_config(
|
||||
mm_missing_processed_data, hf_processor_mm_kwargs
|
||||
),
|
||||
)
|
||||
|
||||
mm_missing_prompt_updates = self._get_mm_prompt_updates(
|
||||
@@ -1897,8 +1909,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
if not all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()):
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
new_text, match_result = self._apply_text_matches(
|
||||
decode_tokens(tokenizer, token_ids),
|
||||
mm_prompt_updates,
|
||||
@@ -1910,16 +1923,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
add_special_tokens=False,
|
||||
)
|
||||
|
||||
matched_updates = defaultdict[
|
||||
str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
for modality, update_idxs in match_result.items():
|
||||
for item_idx, update_idx in enumerate(update_idxs):
|
||||
assert update_idx is not None, (
|
||||
"Failed to apply prompt replacement for "
|
||||
f"mm_items[{modality!r}][{item_idx}]")
|
||||
f"mm_items[{modality!r}][{item_idx}]"
|
||||
)
|
||||
|
||||
matched_updates[modality].append(
|
||||
[mm_prompt_updates[modality][item_idx][update_idx]])
|
||||
[mm_prompt_updates[modality][item_idx][update_idx]]
|
||||
)
|
||||
|
||||
placeholders = self._find_mm_placeholders(
|
||||
new_token_ids,
|
||||
@@ -1944,7 +1958,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"There is likely a problem with your "
|
||||
"implementation of merged multi-modal processor for this "
|
||||
"model (usually arising from an inconsistency between "
|
||||
"`_call_hf_processor` and `_get_mm_fields_config`).")
|
||||
"`_call_hf_processor` and `_get_mm_fields_config`)."
|
||||
)
|
||||
|
||||
def _validate_mm_updates(
|
||||
self,
|
||||
@@ -1962,7 +1977,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"This is likely because you forgot to include input "
|
||||
"placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
|
||||
"in the prompt. If the model has a chat template, make "
|
||||
"sure you have applied it before calling `LLM.generate`.")
|
||||
"sure you have applied it before calling `LLM.generate`."
|
||||
)
|
||||
|
||||
def _validate_mm_placeholders(
|
||||
self,
|
||||
@@ -1978,7 +1994,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
f"corresponding to {item_count} {modality} items, but "
|
||||
f"instead found {len(placeholders)} prompt placeholders! "
|
||||
"Make sure the implementation of `_call_hf_processor` and "
|
||||
"`_get_mm_fields_config` are consistent with each other.")
|
||||
"`_get_mm_fields_config` are consistent with each other."
|
||||
)
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
@@ -2070,7 +2087,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
|
||||
class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
@abstractmethod
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
@@ -2104,15 +2120,16 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
|
||||
if isinstance(decoder_prompt_raw, str):
|
||||
decoder_prompt_ids = encode_tokens(tokenizer,
|
||||
decoder_prompt_raw,
|
||||
add_special_tokens=False)
|
||||
decoder_prompt_ids = encode_tokens(
|
||||
tokenizer, decoder_prompt_raw, add_special_tokens=False
|
||||
)
|
||||
else:
|
||||
decoder_prompt_ids = decoder_prompt_raw
|
||||
|
||||
mm_inputs = MultiModalEncDecInputs(
|
||||
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
|
||||
**encoder_inputs)
|
||||
**encoder_inputs,
|
||||
)
|
||||
mm_inputs["prompt_token_ids"] = decoder_prompt_ids
|
||||
return mm_inputs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user