Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+173 -156
View File
@@ -3,13 +3,21 @@
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
Sequence)
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
from dataclasses import dataclass, field, replace
from enum import Enum
from functools import lru_cache
from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional,
Protocol, Union, cast, overload)
from typing import (
TYPE_CHECKING,
Any,
Generic,
NamedTuple,
Optional,
Protocol,
Union,
cast,
overload,
)
import regex as re
import torch
@@ -17,20 +25,28 @@ from typing_extensions import TypeVar, assert_never
from vllm.logger import init_logger
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
encode_tokens)
from vllm.utils import (flatten_2d_lists, full_groupby,
get_allowed_kwarg_only_overrides)
from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens
from vllm.utils import flatten_2d_lists, full_groupby, get_allowed_kwarg_only_overrides
from vllm.utils.jsontree import JSONTree, json_map_leaves
from .hasher import MultiModalHasher
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
MultiModalFieldConfig, MultiModalInputs,
MultiModalKwargsItem, MultiModalKwargsItems,
MultiModalKwargsOptionalItems, MultiModalUUIDDict,
PlaceholderRange)
from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
MultiModalDataParser)
from .inputs import (
MultiModalDataDict,
MultiModalEncDecInputs,
MultiModalFieldConfig,
MultiModalInputs,
MultiModalKwargsItem,
MultiModalKwargsItems,
MultiModalKwargsOptionalItems,
MultiModalUUIDDict,
PlaceholderRange,
)
from .parse import (
DictEmbeddingItems,
EmbeddingItems,
MultiModalDataItems,
MultiModalDataParser,
)
if TYPE_CHECKING:
from transformers.configuration_utils import PretrainedConfig
@@ -57,9 +73,7 @@ def _cached_encode(
*,
add_special_tokens: Optional[bool] = None,
) -> list[int]:
return encode_tokens(tokenizer,
text,
add_special_tokens=add_special_tokens)
return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
@lru_cache(maxsize=2048)
@@ -69,9 +83,9 @@ def _cached_decode(
*,
skip_special_tokens: Optional[bool] = None,
) -> str:
return decode_tokens(tokenizer,
list(token_ids),
skip_special_tokens=skip_special_tokens)
return decode_tokens(
tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
)
def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:
@@ -89,24 +103,22 @@ def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:
class _GetMatchIndex(Protocol):
def __call__(
self,
tokenizer: AnyTokenizer,
prompt: PromptSeq,
start_idx: int = 0,
) -> Optional[int]:
...
) -> Optional[int]: ...
@dataclass
class PromptIndex:
"""Resolves to an index in the prompt."""
get_match_index: _GetMatchIndex
class PromptIndexTargets:
@staticmethod
def start() -> PromptIndex:
"""
@@ -139,9 +151,7 @@ class PromptIndexTargets:
else:
if isinstance(prefix, str):
# Make both `list[int]`
prefix = encode_tokens(tokenizer,
prefix,
add_special_tokens=False)
prefix = encode_tokens(tokenizer, prefix, add_special_tokens=False)
match_idx = len(prefix)
return match_idx if prompt[:match_idx] == prefix else None
@@ -181,8 +191,7 @@ class PromptUpdateDetails(Generic[_S]):
full: _S
"""The full content."""
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq],
torch.Tensor]] = None
is_embed: Optional[Callable[[AnyTokenizer, PromptSeq], torch.Tensor]] = None
"""
Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
return a boolean mask of shape `(len(full),)` indicating which positions
@@ -203,7 +212,6 @@ class PromptUpdateDetails(Generic[_S]):
seq: _S,
embed_text: str,
) -> "PromptUpdateDetails[_S]":
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
embed_token_ids = encode_tokens(tokenizer, embed_text)
token_ids = _seq2tokens(tokenizer, full)
@@ -220,7 +228,6 @@ class PromptUpdateDetails(Generic[_S]):
seq: _S,
embed_token_id: int,
) -> "PromptUpdateDetails[_S]":
def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:
token_ids = _seq2tokens(tokenizer, full)
@@ -238,8 +245,7 @@ use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
specify which part.
"""
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
PromptUpdateInfo]
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo], PromptUpdateInfo]
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
@@ -408,11 +414,13 @@ class PromptReplacement(PromptUpdate):
modality="image",
target="<image>",
replacement=PromptUpdateDetails(
full="".join([
"<image_bos>",
"<image>" * image_feature_size,
"<image_eos>",
]),
full="".join(
[
"<image_bos>",
"<image>" * image_feature_size,
"<image_eos>",
]
),
features="<image>" * image_feature_size,
),
)
@@ -426,8 +434,9 @@ class PromptReplacement(PromptUpdate):
modality="image",
target=[image_token_id],
replacement=PromptUpdateDetails(
full=([image_bos_id] + [image_token_id] * image_feature_size
+ [image_eos_id]),
full=(
[image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
),
features=[image_token_id] * image_feature_size,
),
)
@@ -459,10 +468,8 @@ class _HasModalityAttr(Protocol):
class _HasModalityProp(Protocol):
@property
def modality(self) -> str:
...
def modality(self) -> str: ...
_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
@@ -520,9 +527,7 @@ class ResolvedPromptUpdate:
target_token_ids = _seq2tokens(tokenizer, target)
for match in iter_token_matches(prompt,
target_token_ids,
start_idx=start_idx):
for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
yield PromptTargetMatch(match.start_idx, match.end_idx)
def iter_text_matches(
@@ -544,8 +549,7 @@ class ResolvedPromptUpdate:
target_text = _seq2text(tokenizer, target)
for match in re.finditer(re.escape(target_text), prompt,
pos=start_idx):
for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
yield PromptTargetMatch(match.start(), match.end())
def iter_matches(
@@ -557,9 +561,7 @@ class ResolvedPromptUpdate:
) -> Generator[PromptTargetMatch]:
"""Yield each instance of `self.target` found in `prompt`."""
if isinstance(prompt, str):
return self.iter_text_matches(prompt,
tokenizer,
start_idx=start_idx)
return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
@@ -680,9 +682,9 @@ def _find_matches(
break # Already found a match for this item
for match in update.iter_matches(
prompt,
tokenizer,
start_idx=prev_end_idx,
prompt,
tokenizer,
start_idx=prev_end_idx,
):
# All matches should share the same mode
if mode is None:
@@ -723,8 +725,7 @@ def _apply_matches(
out_seqs = list[Union[str, list[int]]]()
out_result: MultiModalPromptUpdatesApplyResult = {
m: [None] * len(items)
for m, items in mm_prompt_updates.items()
m: [None] * len(items) for m, items in mm_prompt_updates.items()
}
start_idx = prev_end_idx = 0
@@ -743,8 +744,7 @@ def _apply_matches(
for (modality, item_idx), (match, update_idx) in matches_to_apply:
found = True
matched_update = mm_prompt_updates[modality][item_idx][
update_idx]
matched_update = mm_prompt_updates[modality][item_idx][update_idx]
matched_content = matched_update.content.full
if mode == UpdateMode.INSERT:
@@ -756,9 +756,10 @@ def _apply_matches(
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
out_seqs.append(
_seq2text(tokenizer, matched_content
) if isinstance(prompt, str) else _seq2tokens(
tokenizer, matched_content))
_seq2text(tokenizer, matched_content)
if isinstance(prompt, str)
else _seq2tokens(tokenizer, matched_content)
)
out_result[modality][item_idx] = update_idx
# Exclude overlapping matches
@@ -784,8 +785,7 @@ def apply_token_matches(
the same placeholder tokens. In that case, the modality that
appears earlier in `mm_prompt_updates` takes priority.
"""
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates,
tokenizer)
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
return flatten_2d_lists(token_id_seqs), result
@@ -847,8 +847,7 @@ def _iter_placeholders(
if prompt[start_idx:end_idx_full] == content_tokens_full:
content_is_embed = content.is_embed
if content_is_embed is not None:
content_is_embed = content_is_embed(
tokenizer, content.full)
content_is_embed = content_is_embed(tokenizer, content.full)
yield PlaceholderFeaturesInfo(
modality=modality,
@@ -899,16 +898,14 @@ class InputProcessingContext:
"""The tokenizer used to tokenize the inputs."""
@overload
def get_hf_config(self, /) -> "PretrainedConfig":
...
def get_hf_config(self, /) -> "PretrainedConfig": ...
@overload
def get_hf_config(
self,
typ: Union[type[_C], tuple[type[_C], ...]],
/,
) -> _C:
...
) -> _C: ...
def get_hf_config(
self,
@@ -930,9 +927,11 @@ class InputProcessingContext:
hf_config = self.model_config.hf_config
if not isinstance(hf_config, typ):
raise TypeError("Invalid type of HuggingFace config. "
f"Expected type: {typ}, but "
f"found type: {type(hf_config)}")
raise TypeError(
"Invalid type of HuggingFace config. "
f"Expected type: {typ}, but "
f"found type: {type(hf_config)}"
)
return hf_config
@@ -956,8 +955,7 @@ class InputProcessingContext:
return mm_config
@overload
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin":
...
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin": ...
@overload
def get_hf_processor(
@@ -965,8 +963,7 @@ class InputProcessingContext:
typ: Union[type[_P], tuple[type[_P], ...]],
/,
**kwargs: object,
) -> _P:
...
) -> _P: ...
def get_hf_processor(
self,
@@ -1017,7 +1014,6 @@ class InputProcessingContext:
self,
output: JSONTree,
) -> JSONTree:
def _postprocess_one(x: object):
if isinstance(x, torch.Tensor): # noqa: SIM102
# This mimics the behavior of transformers.BatchFeature
@@ -1054,17 +1050,21 @@ class InputProcessingContext:
)
try:
output = hf_processor(**data,
**allowed_kwargs,
return_tensors="pt")
output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
except Exception as exc:
# See https://github.com/huggingface/tokenizers/issues/537
if (isinstance(exc, RuntimeError) and exc
and exc.args[0] == "Already borrowed"
and num_tries < max_tries):
if (
isinstance(exc, RuntimeError)
and exc
and exc.args[0] == "Already borrowed"
and num_tries < max_tries
):
logger.warning(
"Failed to acquire tokenizer in current thread. "
"Retrying (%d/%d)...", num_tries, max_tries)
"Retrying (%d/%d)...",
num_tries,
max_tries,
)
time.sleep(0.5)
return self.call_hf_processor(
hf_processor,
@@ -1074,8 +1074,10 @@ class InputProcessingContext:
max_tries=max_tries,
)
msg = (f"Failed to apply {type(hf_processor).__name__} "
f"on data={data} with kwargs={allowed_kwargs}")
msg = (
f"Failed to apply {type(hf_processor).__name__} "
f"on data={data} with kwargs={allowed_kwargs}"
)
raise ValueError(msg) from exc
@@ -1142,8 +1144,11 @@ class BaseProcessingInfo:
for modality, supported_limit in supported_mm_limits.items():
user_limit = mm_config.get_limit_per_prompt(modality)
allowed_limits[modality] = (user_limit if supported_limit is None
else min(user_limit, supported_limit))
allowed_limits[modality] = (
user_limit
if supported_limit is None
else min(user_limit, supported_limit)
)
return allowed_limits
@@ -1154,7 +1159,7 @@ class BaseProcessingInfo:
) -> Optional[Mapping[str, int]]:
"""
Return the maximum number of tokens per item of for each modality.
When `None` (the default) is returned, vLLM will generate dummy inputs
(images/videos) at maximum possible sizes and process them to determine
the maximum token count per modality.
@@ -1165,7 +1170,7 @@ class BaseProcessingInfo:
counts, avoiding the need for dummy input generation and processing.
Note:
The maximum number of tokens per item of each modality returned
The maximum number of tokens per item of each modality returned
from this function should respect the model's maximum sequence
length and the maximum number of items of each modality allowed,
and agree with dummy inputs (images/videos) at maximum possible
@@ -1245,10 +1250,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
*,
mm_uuids: Optional[MultiModalUUIDDict] = None,
) -> MultiModalInputs:
return self.apply(prompt,
mm_data,
hf_processor_mm_kwargs,
mm_uuids=mm_uuids)
return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
def _get_data_parser(self) -> MultiModalDataParser:
"""
@@ -1276,8 +1278,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
limit = min(supported_limit, allowed_limit)
if num_items > limit:
msg = (f"At most {limit} {modality}(s) may be provided in "
"one prompt.")
msg = f"At most {limit} {modality}(s) may be provided in one prompt."
if num_items <= supported_limit:
msg += " Set `--limit-mm-per-prompt` to increase this limit."
@@ -1339,8 +1340,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_item_counts: Mapping[str, int],
) -> MultiModalPromptUpdates:
return {
modality: [[update.resolve(item_idx) for update in updates]
for item_idx in range(mm_item_counts.get(modality, 0))]
modality: [
[update.resolve(item_idx) for update in updates]
for item_idx in range(mm_item_counts.get(modality, 0))
]
for modality, updates in full_groupby_modality(prompt_updates)
}
@@ -1385,8 +1388,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
tokenizer = self.info.get_tokenizer()
return find_mm_placeholders(new_token_ids, mm_prompt_updates,
tokenizer)
return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
def _get_hf_mm_data(
self,
@@ -1436,7 +1438,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
return not any(
isinstance(items, (EmbeddingItems, DictEmbeddingItems))
for items in mm_items.values())
for items in mm_items.values()
)
def _apply_hf_processor_text_mm(
self,
@@ -1461,7 +1464,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
)
processed_data.update(passthrough_data)
prompt_ids, = processed_data.pop("input_ids").tolist()
(prompt_ids,) = processed_data.pop("input_ids").tolist()
is_update_applied = self._hf_processor_applies_updates(
prompt_text=prompt_text,
@@ -1564,8 +1567,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
tokenization_kwargs=tokenization_kwargs,
)
prompt_ids = self._apply_hf_processor_text_only(
prompt, tokenization_kwargs)
prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)
else:
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
@@ -1611,10 +1613,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# hash if `hf_processor_mm_kwargs` or `tokenization_kwargs`
# are provided. This is because the processed multimodal
# inputs can be different depending on the processor kwargs.
if item_uuid is None or \
hf_processor_mm_kwargs or \
tokenization_kwargs:
if (
item_uuid is None
or hf_processor_mm_kwargs
or tokenization_kwargs
):
# NOTE: use provided hash string to hash with kwargs
# if available for better performance.
item = item_uuid if item_uuid is not None else item
@@ -1623,16 +1626,20 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
**tokenization_kwargs))
**tokenization_kwargs,
)
)
else:
computed.append(item_uuid)
hashes[modality] = computed
else:
hashes[modality] = [
MultiModalHasher.hash_kwargs(model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
**tokenization_kwargs)
MultiModalHasher.hash_kwargs(
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
**tokenization_kwargs,
)
for item in items
]
@@ -1645,13 +1652,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_hashes: MultiModalHashes,
) -> MultiModalDataItems:
mm_is_cached = {
modality: cache.is_cached(hashes)
for modality, hashes in mm_hashes.items()
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
}
mm_missing_idxs = {
modality: [
idx for idx, item_is_cached in enumerate(items_is_cached)
idx
for idx, item_is_cached in enumerate(items_is_cached)
if not item_is_cached
]
for modality, items_is_cached in mm_is_cached.items()
@@ -1664,7 +1671,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
if data is None:
raise ValueError(
f"Cache miss for {modality} at index {idx} "
f"but data is not provided.")
f"but data is not provided."
)
else:
missing_modality_data.append(data)
mm_missing_data[modality] = missing_modality_data
@@ -1692,20 +1700,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# Need to calculate this at the beginning to avoid skipping cache logic
# for subsequently repeated items in the same modality
mm_is_cached = {
modality: cache.is_cached(hashes)
for modality, hashes in mm_hashes.items()
modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items()
}
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
merged_kwargs = defaultdict[str,
list[Optional[MultiModalKwargsItem]]](list)
merged_prompt_updates = defaultdict[
str, list[Sequence[ResolvedPromptUpdate]]](list)
merged_kwargs = defaultdict[str, list[Optional[MultiModalKwargsItem]]](list)
merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](
list
)
for modality, hashes in mm_hashes.items():
missing_kwargs = mm_missing_kwargs.get(modality, [])
missing_prompt_updates = mm_missing_prompt_updates.get(
modality, [])
missing_prompt_updates = mm_missing_prompt_updates.get(modality, [])
for item_idx, item_hash in enumerate(hashes):
kwargs: Optional[MultiModalKwargsItem]
@@ -1723,10 +1729,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
kwargs, updates = cache.get_and_update_item(item, item_hash)
merged_kwargs[modality].append(kwargs)
merged_prompt_updates[modality].append([
self._recompute_cached_prompt_update(update, item_idx)
for update in updates
])
merged_prompt_updates[modality].append(
[
self._recompute_cached_prompt_update(update, item_idx)
for update in updates
]
)
mm_kwargs = MultiModalKwargsItems(merged_kwargs)
mm_prompt_updates = dict(merged_prompt_updates)
@@ -1756,15 +1764,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
mm_processed_data,
self._get_mm_fields_config(mm_processed_data,
hf_processor_mm_kwargs),
self._get_mm_fields_config(mm_processed_data, hf_processor_mm_kwargs),
)
# Use overrides if provided; fallback to data-dependent hashing.
mm_hashes = self._hash_mm_items(mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids)
mm_hashes = self._hash_mm_items(
mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids,
)
mm_prompt_updates = self._get_mm_prompt_updates(
mm_data_items,
@@ -1805,10 +1814,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_uuids=mm_uuids,
)
mm_hashes = self._hash_mm_items(mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids)
mm_hashes = self._hash_mm_items(
mm_data_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
mm_uuids=mm_uuids,
)
mm_missing_data_items = self._get_cache_missing_items(
cache=cache,
@@ -1833,8 +1844,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
mm_missing_processed_data,
self._get_mm_fields_config(mm_missing_processed_data,
hf_processor_mm_kwargs),
self._get_mm_fields_config(
mm_missing_processed_data, hf_processor_mm_kwargs
),
)
mm_missing_prompt_updates = self._get_mm_prompt_updates(
@@ -1897,8 +1909,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
if not all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()):
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
new_text, match_result = self._apply_text_matches(
decode_tokens(tokenizer, token_ids),
mm_prompt_updates,
@@ -1910,16 +1923,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
add_special_tokens=False,
)
matched_updates = defaultdict[
str, list[Sequence[ResolvedPromptUpdate]]](list)
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
for modality, update_idxs in match_result.items():
for item_idx, update_idx in enumerate(update_idxs):
assert update_idx is not None, (
"Failed to apply prompt replacement for "
f"mm_items[{modality!r}][{item_idx}]")
f"mm_items[{modality!r}][{item_idx}]"
)
matched_updates[modality].append(
[mm_prompt_updates[modality][item_idx][update_idx]])
[mm_prompt_updates[modality][item_idx][update_idx]]
)
placeholders = self._find_mm_placeholders(
new_token_ids,
@@ -1944,7 +1958,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"There is likely a problem with your "
"implementation of merged multi-modal processor for this "
"model (usually arising from an inconsistency between "
"`_call_hf_processor` and `_get_mm_fields_config`).")
"`_call_hf_processor` and `_get_mm_fields_config`)."
)
def _validate_mm_updates(
self,
@@ -1962,7 +1977,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"This is likely because you forgot to include input "
"placeholder tokens (e.g., `<image>`, `<|image_pad|>`) "
"in the prompt. If the model has a chat template, make "
"sure you have applied it before calling `LLM.generate`.")
"sure you have applied it before calling `LLM.generate`."
)
def _validate_mm_placeholders(
self,
@@ -1978,7 +1994,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
f"corresponding to {item_count} {modality} items, but "
f"instead found {len(placeholders)} prompt placeholders! "
"Make sure the implementation of `_call_hf_processor` and "
"`_get_mm_fields_config` are consistent with each other.")
"`_get_mm_fields_config` are consistent with each other."
)
def _maybe_apply_prompt_updates(
self,
@@ -2070,7 +2087,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
@abstractmethod
def create_encoder_prompt(
self,
@@ -2104,15 +2120,16 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
tokenizer = self.info.get_tokenizer()
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
if isinstance(decoder_prompt_raw, str):
decoder_prompt_ids = encode_tokens(tokenizer,
decoder_prompt_raw,
add_special_tokens=False)
decoder_prompt_ids = encode_tokens(
tokenizer, decoder_prompt_raw, add_special_tokens=False
)
else:
decoder_prompt_ids = decoder_prompt_raw
mm_inputs = MultiModalEncDecInputs(
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
**encoder_inputs)
**encoder_inputs,
)
mm_inputs["prompt_token_ids"] = decoder_prompt_ids
return mm_inputs