[BUGFIX] fix ovis2_5 multimodal tokens (#52623)
Signed-off-by: Milosz Grunwald <milosz.grunwald@intel.com>
This commit is contained in:
@@ -175,6 +175,18 @@ class VisualTokenizer(torch.nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||||
|
def __init__(self, ctx) -> None:
|
||||||
|
super().__init__(ctx)
|
||||||
|
# extra_special_tokens adds tokens to the tokenizer lazily, on first
|
||||||
|
# access, but the renderer deep-copies this tokenizer into a thread
|
||||||
|
# pool right after this processor is constructed; the copies never
|
||||||
|
# see mutations made afterwards. Force it now, before that happens,
|
||||||
|
# on the tokenizer we actually use (get_hf_processor()'s cache can
|
||||||
|
# otherwise hand back one bound to a self-loaded tokenizer instead).
|
||||||
|
hf_processor = self.get_hf_processor()
|
||||||
|
hf_processor.tokenizer = self.get_tokenizer()
|
||||||
|
hf_processor.extra_special_tokens # noqa: B018
|
||||||
|
|
||||||
def get_hf_config(self):
|
def get_hf_config(self):
|
||||||
return self.ctx.get_hf_config()
|
return self.ctx.get_hf_config()
|
||||||
|
|
||||||
@@ -390,7 +402,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
|
|||||||
out_mm_kwargs: MultiModalKwargsItems,
|
out_mm_kwargs: MultiModalKwargsItems,
|
||||||
) -> list[PromptReplacement]:
|
) -> list[PromptReplacement]:
|
||||||
hf_processor = self.info.get_hf_processor()
|
hf_processor = self.info.get_hf_processor()
|
||||||
|
|
||||||
placeholder = {
|
placeholder = {
|
||||||
"image": hf_processor.get_token_value("image_token"),
|
"image": hf_processor.get_token_value("image_token"),
|
||||||
"video": hf_processor.get_token_value("video_token"),
|
"video": hf_processor.get_token_value("video_token"),
|
||||||
|
|||||||
@@ -96,9 +96,14 @@ class Ovis2_5Processor(ProcessorMixin):
|
|||||||
# missing from the vocab. Re-add them to restore the expected ids.
|
# missing from the vocab. Re-add them to restore the expected ids.
|
||||||
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
|
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
|
||||||
|
|
||||||
|
# in TokenizerPool tokenizer.get_added_vocab() gets values from an instance
|
||||||
|
# that was updated with tokenizer.add_tokens(), so it is favourable over
|
||||||
|
# self.tokenizer.convert_tokens_to_ids(), which refers to a copy before
|
||||||
|
# mutation by tokenizer.add_tokens()
|
||||||
|
added_vocab = self.tokenizer.get_added_vocab()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
key: self.tokenizer.convert_tokens_to_ids(token_name)
|
key: added_vocab[token_name] for key, token_name in required_tokens.items()
|
||||||
for key, token_name in required_tokens.items()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def __call__(
|
def __call__(
|
||||||
|
|||||||
Reference in New Issue
Block a user