[Bugfix] Fix Ovis2_5 special tokens for transformers v5 (#47298)
Signed-off-by: mgrunwal <milosz.grunwald@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Harry Mellor
parent
adfbbc1005
commit
3e0c887511
@@ -402,12 +402,11 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> list[PromptReplacement]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
|
||||
placeholder = {
|
||||
"image": vocab[IMAGE_TOKEN],
|
||||
"video": vocab[VIDEO_TOKEN],
|
||||
"image": hf_processor.get_token_value("image_token"),
|
||||
"video": hf_processor.get_token_value("video_token"),
|
||||
}
|
||||
|
||||
def get_replacement_ovis(item_idx, modality: str):
|
||||
|
||||
@@ -78,7 +78,6 @@ class Ovis2_5Processor(ProcessorMixin):
|
||||
|
||||
@cached_property
|
||||
def extra_special_tokens(self):
|
||||
vocab = self.tokenizer.get_vocab()
|
||||
required_tokens = {
|
||||
"image_token": "<image>",
|
||||
"video_token": "<video>",
|
||||
@@ -90,21 +89,16 @@ class Ovis2_5Processor(ProcessorMixin):
|
||||
"image_pad": "<|image_pad|>",
|
||||
}
|
||||
|
||||
extra_special_tokens = {}
|
||||
suggestion = (
|
||||
"please add '<image>', '<video>', '<ovis_visual_atom>', "
|
||||
"'<ovis_image_start>', '<ovis_image_end>', '<ovis_video_start>', "
|
||||
"'<ovis_video_end>' in 'additional_special_tokens' of "
|
||||
"tokenizer_config.json, You can refer to "
|
||||
"https://huggingface.co/AIDC-AI/Ovis2.6-30B-A3B/blob/main/tokenizer_config.json"
|
||||
)
|
||||
# The checkpoint defines both `additional_special_tokens` and
|
||||
# `extra_special_tokens`, with the latter empty. Transformers ignores
|
||||
# the former because the latter is explicitly empty, so the tokens are
|
||||
# missing from the vocab. Re-add them to restore the expected ids.
|
||||
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
|
||||
|
||||
for key, token_name in required_tokens.items():
|
||||
if token_name not in vocab:
|
||||
raise ValueError(f"Can not find {token_name}, {suggestion}")
|
||||
extra_special_tokens[key] = vocab[token_name]
|
||||
|
||||
return extra_special_tokens
|
||||
return {
|
||||
key: self.tokenizer.convert_tokens_to_ids(token_name)
|
||||
for key, token_name in required_tokens.items()
|
||||
}
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user