[Misc] Fix input processing for Ultravox (#13871)

This commit is contained in:
Roger Wang
2025-02-25 23:56:34 -08:00
committed by GitHub
parent 5157338ed9
commit 7ca1da020f
3 changed files with 6 additions and 15 deletions
+2 -11
View File
@@ -146,7 +146,8 @@ class UltravoxMultiModalProcessor(
) -> BatchFeature:
# Text-only input not supported in composite processor
if not mm_data or not mm_data.get("audios", []):
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self.info.get_tokenizer().encode(
prompt, add_special_tokens=False)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
@@ -185,16 +186,6 @@ class UltravoxMultiModalProcessor(
)
return BatchFeature(combined_outputs)
def _apply_hf_processor_tokens_only(
self,
prompt_tokens: list[int],
) -> list[int]:
# HF processor omits bos_token_id by setting add_special_tokens=False
tokenizer = self.info.get_tokenizer()
assert prompt_tokens[0] == tokenizer.bos_token_id
return prompt_tokens[1:]
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,