[VLM] Avoid unnecessary tokenization (#12310)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-01-22 11:08:31 +00:00
committed by GitHub
parent 68ad4e3a8d
commit cd7b6f0857
9 changed files with 71 additions and 40 deletions
+7 -3
View File
@@ -205,16 +205,20 @@ class UltravoxMultiModalProcessor(
out_mm_kwargs: MultiModalKwargs,
) -> list[PromptReplacement]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
placeholder = hf_processor.audio_token_replacement # type: ignore
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
replacement_id = vocab[
hf_processor.audio_token_replacement] # type: ignore
def get_replacement_ultravox(item_idx: int):
audio_token_len = out_mm_kwargs["audio_token_len"][item_idx]
return placeholder * audio_token_len
return [replacement_id] * int(audio_token_len) # type: ignore
return [
PromptReplacement(
modality="audio",
target="<|audio|>",
target='<|audio|>',
replacement=get_replacement_ultravox,
)
]