[VLM] Support multimodal inputs for Florence-2 models (#13320)
This commit is contained in:
@@ -1303,6 +1303,14 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def create_decoder_prompt(
|
||||
self,
|
||||
prompt: Union[str, list[int]],
|
||||
mm_data: MultiModalDataDict,
|
||||
) -> Union[str, list[int]]:
|
||||
"""Create input prompt for the decoder."""
|
||||
return prompt
|
||||
|
||||
def apply(
|
||||
self,
|
||||
prompt: Union[str, list[int]],
|
||||
@@ -1323,17 +1331,15 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# We assumed the decoder prompt text is copied from
|
||||
# the original encoder prompt without extra process
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
if isinstance(prompt, str):
|
||||
decoder_prompt = prompt
|
||||
decoder_prompt = self.create_decoder_prompt(prompt, mm_data)
|
||||
if isinstance(decoder_prompt, str):
|
||||
decoder_prompt_ids = encode_tokens(tokenizer,
|
||||
prompt,
|
||||
decoder_prompt,
|
||||
add_special_tokens=False)
|
||||
else:
|
||||
decoder_prompt = decode_tokens(tokenizer, prompt)
|
||||
decoder_prompt_ids = prompt
|
||||
decoder_prompt_ids = decoder_prompt
|
||||
decoder_prompt = decode_tokens(tokenizer, decoder_prompt)
|
||||
|
||||
mm_inputs = MultiModalEncDecInputs(
|
||||
encoder_prompt=encoder_inputs["prompt"],
|
||||
|
||||
Reference in New Issue
Block a user