[Refactor] Get prompt updates earlier (#23097)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -989,6 +989,18 @@ A collection of hashes with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
MultiModalPromptUpdates = dict[str, Sequence[BoundPromptUpdate]]
|
||||
"""
|
||||
A collection of prompt updates with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
|
||||
class MultiModalProcessingInfo(NamedTuple):
|
||||
kwargs: MultiModalKwargsItems
|
||||
hashes: Optional[MultiModalHashes]
|
||||
prompt_updates: MultiModalPromptUpdates
|
||||
|
||||
|
||||
class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
@@ -1363,7 +1375,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
cache: ProcessingCache,
|
||||
mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]],
|
||||
mm_missing_kwargs: MultiModalKwargsItems,
|
||||
) -> dict[str, list[MultiModalKwargsItem]]:
|
||||
) -> MultiModalKwargsItems:
|
||||
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
|
||||
|
||||
merged_items = defaultdict[str, list[MultiModalKwargsItem]](list)
|
||||
@@ -1379,7 +1391,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
merged_items[modality].append(kw_item)
|
||||
|
||||
return dict(merged_items)
|
||||
return MultiModalKwargsItems(merged_items)
|
||||
|
||||
def _apply_hf_processor(
|
||||
self,
|
||||
@@ -1389,8 +1401,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
*,
|
||||
return_mm_hashes: bool,
|
||||
) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
|
||||
bool]:
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
(
|
||||
prompt_ids,
|
||||
mm_processed_data,
|
||||
@@ -1413,7 +1424,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
tokenization_kwargs)
|
||||
if return_mm_hashes else None)
|
||||
|
||||
return prompt_ids, mm_kwargs, mm_hashes, is_update_applied
|
||||
unbound_prompt_updates = self._get_prompt_updates(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_kwargs,
|
||||
)
|
||||
mm_prompt_updates = self._bind_and_group_updates(
|
||||
unbound_prompt_updates)
|
||||
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_info, is_update_applied
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
@@ -1423,8 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
*,
|
||||
return_mm_hashes: bool,
|
||||
) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
|
||||
bool]:
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
"""
|
||||
Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
@@ -1475,18 +1499,27 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
mm_cache_items_merged = self._merge_mm_kwargs(
|
||||
mm_kwargs = self._merge_mm_kwargs(
|
||||
cache,
|
||||
mm_cache_items_or_hashes=mm_cache_items_or_hashes,
|
||||
mm_missing_kwargs=mm_missing_kwargs,
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_seq([
|
||||
item for cache_items in mm_cache_items_merged.values()
|
||||
for item in cache_items
|
||||
])
|
||||
unbound_prompt_updates = self._get_prompt_updates(
|
||||
mm_data_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_kwargs,
|
||||
)
|
||||
mm_prompt_updates = self._bind_and_group_updates(
|
||||
unbound_prompt_updates)
|
||||
|
||||
return prompt_ids, mm_kwargs, mm_hashes_to_return, is_update_applied
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes_to_return,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return prompt_ids, mm_info, is_update_applied
|
||||
|
||||
def _bind_and_group_updates(
|
||||
self,
|
||||
@@ -1626,19 +1659,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
is_update_applied: bool,
|
||||
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
unbound_prompt_updates = self._get_prompt_updates(
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_kwargs,
|
||||
)
|
||||
mm_prompt_updates = self._bind_and_group_updates(
|
||||
unbound_prompt_updates)
|
||||
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
|
||||
@@ -1694,8 +1719,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
(
|
||||
prompt_ids,
|
||||
mm_kwargs,
|
||||
mm_hashes,
|
||||
mm_info,
|
||||
is_update_applied,
|
||||
) = self._cached_apply_hf_processor(
|
||||
prompt,
|
||||
@@ -1708,9 +1732,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
# NOTE: tokenization_kwargs are not required to init processor
|
||||
prompt_ids, prompt, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
prompt_ids=prompt_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_prompt_updates=mm_info.prompt_updates,
|
||||
is_update_applied=is_update_applied,
|
||||
)
|
||||
|
||||
@@ -1723,8 +1747,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
type="multimodal",
|
||||
prompt=prompt,
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_hashes=mm_info.hashes,
|
||||
mm_placeholders=mm_placeholder_ranges,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user