[VLM] Support multimodal inputs for Florence-2 models (#13320)

This commit is contained in:
Isotr0py
2025-02-27 02:06:41 -08:00
committed by GitHub
parent 788f284b53
commit edf309ebbe
13 changed files with 1075 additions and 114 deletions
+13 -7
View File
@@ -1303,6 +1303,14 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
"""
raise NotImplementedError
def create_decoder_prompt(
self,
prompt: Union[str, list[int]],
mm_data: MultiModalDataDict,
) -> Union[str, list[int]]:
"""Create input prompt for the decoder."""
return prompt
def apply(
self,
prompt: Union[str, list[int]],
@@ -1323,17 +1331,15 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
hf_processor_mm_kwargs,
)
# We assumed the decoder prompt text is copied from
# the original encoder prompt without extra process
tokenizer = self.info.get_tokenizer()
if isinstance(prompt, str):
decoder_prompt = prompt
decoder_prompt = self.create_decoder_prompt(prompt, mm_data)
if isinstance(decoder_prompt, str):
decoder_prompt_ids = encode_tokens(tokenizer,
prompt,
decoder_prompt,
add_special_tokens=False)
else:
decoder_prompt = decode_tokens(tokenizer, prompt)
decoder_prompt_ids = prompt
decoder_prompt_ids = decoder_prompt
decoder_prompt = decode_tokens(tokenizer, decoder_prompt)
mm_inputs = MultiModalEncDecInputs(
encoder_prompt=encoder_inputs["prompt"],