[Model] Use helper function to run MM processors with token inputs (where applicable) (#38018)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -5,8 +5,15 @@ from collections import defaultdict
|
||||
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Generic, NamedTuple, Protocol, TypeAlias, cast
|
||||
from functools import lru_cache, partial
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Generic,
|
||||
NamedTuple,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
cast,
|
||||
)
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
@@ -21,6 +28,7 @@ from vllm.inputs import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.transformers_utils.processor import call_hf_processor_mm_only
|
||||
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
||||
|
||||
from ..inputs import (
|
||||
@@ -1150,7 +1158,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
(prompt_ids,) = processed_data.pop("input_ids").tolist()
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
|
||||
(prompt_ids,) = input_ids
|
||||
|
||||
is_update_applied = self._hf_processor_applies_updates(
|
||||
prompt_text=prompt_text,
|
||||
@@ -1213,16 +1225,35 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
|
||||
to go along with the multi-modal data.
|
||||
"""
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
# Custom logic based on text inputs
|
||||
if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return mm_processed_data
|
||||
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
return mm_processed_data
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
partial(
|
||||
call_hf_processor_mm_only,
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
),
|
||||
processor_data,
|
||||
dict(**hf_processor_mm_kwargs, **tokenization_kwargs),
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
return processed_data
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user