[VLM] Avoid unnecessary tokenization (#12310)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -205,16 +205,20 @@ class UltravoxMultiModalProcessor(
|
||||
out_mm_kwargs: MultiModalKwargs,
|
||||
) -> list[PromptReplacement]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
placeholder = hf_processor.audio_token_replacement # type: ignore
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
replacement_id = vocab[
|
||||
hf_processor.audio_token_replacement] # type: ignore
|
||||
|
||||
def get_replacement_ultravox(item_idx: int):
|
||||
audio_token_len = out_mm_kwargs["audio_token_len"][item_idx]
|
||||
return placeholder * audio_token_len
|
||||
return [replacement_id] * int(audio_token_len) # type: ignore
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target="<|audio|>",
|
||||
target='<|audio|>',
|
||||
replacement=get_replacement_ultravox,
|
||||
)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user