[VLM] Implement merged multimodal processor for Mllama (#11427)
This commit is contained in:
@@ -739,3 +739,19 @@ class MultiModalInputs(TypedDict):
|
||||
For each modality, information about the placeholder tokens in
|
||||
:code:`prompt_token_ids`.
|
||||
"""
|
||||
|
||||
|
||||
class MultiModalEncDecInputs(MultiModalInputs):
|
||||
"""
|
||||
Represents the outputs of :class:`vllm.multimodal.EncDecMultiModalProcessor`
|
||||
ready to be passed to vLLM internals.
|
||||
"""
|
||||
|
||||
encoder_prompt: str
|
||||
"""The processed encoder prompt text."""
|
||||
|
||||
encoder_prompt_token_ids: list[int]
|
||||
"""The processed token IDs of the encoder prompt."""
|
||||
|
||||
encoder_token_type_ids: NotRequired[list[int]]
|
||||
"""The token type IDs of the encoder prompt."""
|
||||
|
||||
@@ -20,9 +20,9 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
|
||||
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
||||
MultiModalInputs, MultiModalKwargs, MultiModalKwargsItem,
|
||||
PlaceholderRange)
|
||||
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
|
||||
MultiModalFieldConfig, MultiModalInputs, MultiModalKwargs,
|
||||
MultiModalKwargsItem, PlaceholderRange)
|
||||
from .parse import MultiModalDataItems, MultiModalDataParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -1293,3 +1293,57 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholder_ranges,
|
||||
)
|
||||
|
||||
|
||||
class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
@abstractmethod
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: Union[str, list[int]],
|
||||
mm_data: MultiModalDataDict,
|
||||
) -> Union[str, list[int]]:
|
||||
"""Create input prompt for the encoder."""
|
||||
raise NotImplementedError
|
||||
|
||||
def apply(
|
||||
self,
|
||||
prompt: Union[str, list[int]],
|
||||
mm_data: MultiModalDataDict,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> MultiModalEncDecInputs:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
The main processing steps are modified to fit encoder-decoder model:
|
||||
1. Create encoder prompt from input prompt text.
|
||||
2. Apply the HF processor on encoder prompt.
|
||||
3. Copy the input prompt text as decoder prompt inputs.
|
||||
"""
|
||||
encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
|
||||
encoder_inputs = super().apply(
|
||||
encoder_prompt,
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# We assumed the decoder prompt text is copied from
|
||||
# the original encoder prompt without extra process
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
if isinstance(prompt, str):
|
||||
decoder_prompt = prompt
|
||||
decoder_prompt_ids = encode_tokens(tokenizer,
|
||||
prompt,
|
||||
add_special_tokens=False)
|
||||
else:
|
||||
decoder_prompt = decode_tokens(tokenizer, prompt)
|
||||
decoder_prompt_ids = prompt
|
||||
|
||||
mm_inputs = MultiModalEncDecInputs(
|
||||
encoder_prompt=encoder_inputs["prompt"],
|
||||
encoder_prompt_token_ids=encoder_inputs["prompt_token_ids"],
|
||||
**encoder_inputs)
|
||||
mm_inputs.update({
|
||||
"prompt": decoder_prompt,
|
||||
"prompt_token_ids": decoder_prompt_ids
|
||||
})
|
||||
return mm_inputs
|
||||
|
||||
@@ -144,7 +144,11 @@ class MultiModalProfiler(Generic[_I]):
|
||||
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
def get_dummy_data(self, seq_len: int) -> DummyData:
|
||||
def get_dummy_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
is_encoder_data: bool = False,
|
||||
) -> DummyData:
|
||||
# Avoid circular import
|
||||
from vllm.sequence import SequenceData
|
||||
|
||||
@@ -183,16 +187,18 @@ class MultiModalProfiler(Generic[_I]):
|
||||
total_len = len(prompt_token_ids)
|
||||
|
||||
# V0 does not support chunked prefill.
|
||||
if total_len > seq_len and not envs.VLLM_USE_V1:
|
||||
logger.warning(
|
||||
"The context length (%d) of the model is too short "
|
||||
"to hold the multi-modal embeddings in the worst case "
|
||||
"(%d tokens in total, out of which %s are reserved for "
|
||||
"multi-modal embeddings). This may cause certain multi-modal "
|
||||
"inputs to fail during inference, even when the input text is "
|
||||
"short. To avoid this, you should increase `max_model_len`, "
|
||||
"reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
|
||||
total_len, total_placeholders_by_modality)
|
||||
if (total_len > seq_len and not envs.VLLM_USE_V1) or is_encoder_data:
|
||||
if total_len > seq_len:
|
||||
logger.warning(
|
||||
"The context length (%d) of the model is too short "
|
||||
"to hold the multi-modal embeddings in the worst case "
|
||||
"(%d tokens in total, out of which %s are reserved for "
|
||||
"multi-modal embeddings). This may cause certain "
|
||||
"multi-modal inputs to fail during inference, even when "
|
||||
"the input text is short. To avoid this, you should "
|
||||
"increase `max_model_len`, reduce `max_num_seqs`, "
|
||||
"and/or reduce `mm_counts`.", seq_len, total_len,
|
||||
total_placeholders_by_modality)
|
||||
|
||||
return DummyData(
|
||||
seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),
|
||||
|
||||
Reference in New Issue
Block a user