From bd4cb3fe4efd1e19ceec1f31fcd4bfe599b0c3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20Grunwald?= Date: Fri, 25 Sep 2026 01:18:54 +0200 Subject: [PATCH] [BUGFIX] fix ovis2_5 multimodal tokens (#52623) Signed-off-by: Milosz Grunwald --- vllm/model_executor/models/ovis2_5.py | 13 ++++++++++++- vllm/transformers_utils/processors/ovis2_5.py | 9 +++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 456ae0b7fe..86a32e270a 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -175,6 +175,18 @@ class VisualTokenizer(torch.nn.Module): class Ovis2_5ProcessingInfo(BaseProcessingInfo): + def __init__(self, ctx) -> None: + super().__init__(ctx) + # extra_special_tokens adds tokens to the tokenizer lazily, on first + # access, but the renderer deep-copies this tokenizer into a thread + # pool right after this processor is constructed; the copies never + # see mutations made afterwards. Force it now, before that happens, + # on the tokenizer we actually use (get_hf_processor()'s cache can + # otherwise hand back one bound to a self-loaded tokenizer instead). + hf_processor = self.get_hf_processor() + hf_processor.tokenizer = self.get_tokenizer() + hf_processor.extra_special_tokens # noqa: B018 + def get_hf_config(self): return self.ctx.get_hf_config() @@ -390,7 +402,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]) out_mm_kwargs: MultiModalKwargsItems, ) -> list[PromptReplacement]: hf_processor = self.info.get_hf_processor() - placeholder = { "image": hf_processor.get_token_value("image_token"), "video": hf_processor.get_token_value("video_token"), diff --git a/vllm/transformers_utils/processors/ovis2_5.py b/vllm/transformers_utils/processors/ovis2_5.py index 2d5764bfd0..9e4e47738a 100644 --- a/vllm/transformers_utils/processors/ovis2_5.py +++ b/vllm/transformers_utils/processors/ovis2_5.py @@ -96,9 +96,14 @@ class Ovis2_5Processor(ProcessorMixin): # missing from the vocab. Re-add them to restore the expected ids. self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True) + # in TokenizerPool tokenizer.get_added_vocab() gets values from an instance + # that was updated with tokenizer.add_tokens(), so it is favourable over + # self.tokenizer.convert_tokens_to_ids(), which refers to a copy before + # mutation by tokenizer.add_tokens() + added_vocab = self.tokenizer.get_added_vocab() + return { - key: self.tokenizer.convert_tokens_to_ids(token_name) - for key, token_name in required_tokens.items() + key: added_vocab[token_name] for key, token_name in required_tokens.items() } def __call__(