[Bugfix] Fix Ovis2_5 special tokens for transformers v5 (#47298)

Signed-off-by: mgrunwal <milosz.grunwald@intel.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Miłosz Grunwald
2026-07-21 08:09:20 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Harry Mellor
parent adfbbc1005
commit 3e0c887511
2 changed files with 12 additions and 19 deletions
+3 -4
View File
@@ -402,12 +402,11 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> list[PromptReplacement]:
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
hf_processor = self.info.get_hf_processor()
placeholder = {
"image": vocab[IMAGE_TOKEN],
"video": vocab[VIDEO_TOKEN],
"image": hf_processor.get_token_value("image_token"),
"video": hf_processor.get_token_value("video_token"),
}
def get_replacement_ovis(item_idx, modality: str):
+9 -15
View File
@@ -78,7 +78,6 @@ class Ovis2_5Processor(ProcessorMixin):
@cached_property
def extra_special_tokens(self):
vocab = self.tokenizer.get_vocab()
required_tokens = {
"image_token": "<image>",
"video_token": "<video>",
@@ -90,21 +89,16 @@ class Ovis2_5Processor(ProcessorMixin):
"image_pad": "<|image_pad|>",
}
extra_special_tokens = {}
suggestion = (
"please add '<image>', '<video>', '<ovis_visual_atom>', "
"'<ovis_image_start>', '<ovis_image_end>', '<ovis_video_start>', "
"'<ovis_video_end>' in 'additional_special_tokens' of "
"tokenizer_config.json, You can refer to "
"https://huggingface.co/AIDC-AI/Ovis2.6-30B-A3B/blob/main/tokenizer_config.json"
)
# The checkpoint defines both `additional_special_tokens` and
# `extra_special_tokens`, with the latter empty. Transformers ignores
# the former because the latter is explicitly empty, so the tokens are
# missing from the vocab. Re-add them to restore the expected ids.
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
for key, token_name in required_tokens.items():
if token_name not in vocab:
raise ValueError(f"Can not find {token_name}, {suggestion}")
extra_special_tokens[key] = vocab[token_name]
return extra_special_tokens
return {
key: self.tokenizer.convert_tokens_to_ids(token_name)
for key, token_name in required_tokens.items()
}
def __call__(
self,