[BUGFIX] fix ovis2_5 multimodal tokens (#52623)

Signed-off-by: Milosz Grunwald <milosz.grunwald@intel.com>
This commit is contained in:
Miłosz Grunwald
2026-09-25 07:18:54 +08:00
committed by GitHub
parent 6a86bf626e
commit bd4cb3fe4e
2 changed files with 19 additions and 3 deletions
@@ -96,9 +96,14 @@ class Ovis2_5Processor(ProcessorMixin):
# missing from the vocab. Re-add them to restore the expected ids.
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
# in TokenizerPool tokenizer.get_added_vocab() gets values from an instance
# that was updated with tokenizer.add_tokens(), so it is favourable over
# self.tokenizer.convert_tokens_to_ids(), which refers to a copy before
# mutation by tokenizer.add_tokens()
added_vocab = self.tokenizer.get_added_vocab()
return {
key: self.tokenizer.convert_tokens_to_ids(token_name)
for key, token_name in required_tokens.items()
key: added_vocab[token_name] for key, token_name in required_tokens.items()
}
def __call__(