[MM] Cache common token sequences (#53560)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Signed-off-by: Cyrus Leung <tlleungac@connect.ust.hk>

tests/multimodal/test_processing.py

@@ -1082,7 +1082,7 @@ class _FakeTokenizer:
     _MERGES = {"food": (1000,), "foo": (101, 111, 111)}
     _INVERSE = {ids: text for text, ids in _MERGES.items()}
 
-    def encode(self, text: str, add_special_tokens: bool = True) -> list[int]:
+    def encode(self, text: str, **kwargs) -> list[int]:
         token_ids = list[int]()
         pos = 0
         while pos < len(text):
@@ -1097,11 +1097,7 @@ class _FakeTokenizer:
                 pos += 1
         return token_ids
 
-    def decode(
-        self,
-        token_ids: list[int],
-        skip_special_tokens: bool = False,
-    ) -> str:
+    def decode(self, token_ids: list[int], **kwargs) -> str:
         chars = list[str]()
         pos = 0
         while pos < len(token_ids):

vllm/model_executor/models/llava_onevision2.py

@@ -98,6 +98,7 @@ from vllm.multimodal.processing import (
     PromptReplacement,
     PromptUpdate,
     PromptUpdateDetails,
+    cached_encode,
 )
 from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder
 from vllm.multimodal.video import (
@@ -1894,7 +1895,7 @@ class LlavaOnevision2MultiModalProcessor(
         video_pad_id = vocab["<|video_pad|>"]
         vision_start_id = vocab["<|vision_start|>"]
         vision_end_id = vocab["<|vision_end|>"]
-        newline_ids = tokenizer.encode("\n", add_special_tokens=False)
+        newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
         merge_length = image_processor.merge_size**2
         decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1))
 

vllm/model_executor/models/minicpmv.py

@@ -1050,7 +1050,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
         tokenizer = self.info.get_tokenizer()
         for modality, pattern in placeholders:
             sub_pattern = tokenizer.decode(
-                tokenizer.encode(pattern, add_special_tokens=False)
+                cached_encode(tokenizer, pattern, add_special_tokens=False)
             )
             if sub_pattern != pattern:
                 additional_placeholders.append((modality, sub_pattern))

vllm/model_executor/models/minicpmv4_6.py

@@ -328,7 +328,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
         additional_placeholders = []
         for modality, pattern in placeholders:
             sub_pattern = tokenizer.decode(
-                tokenizer.encode(pattern, add_special_tokens=False)
+                cached_encode(tokenizer, pattern, add_special_tokens=False)
             )
             if sub_pattern != pattern:
                 additional_placeholders.append((modality, sub_pattern))

vllm/model_executor/models/moondream3.py

@@ -57,6 +57,7 @@ from vllm.multimodal.processing import (
     PromptReplacement,
     PromptUpdate,
     PromptUpdateDetails,
+    cached_encode,
 )
 from vllm.sequence import IntermediateTensors
 from vllm.transformers_utils.configs.moondream3 import (
@@ -945,7 +946,8 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
     @cached_property
     def bos_image_placeholder_tokens(self) -> list[int]:
         tokenizer = self.info.get_tokenizer()
-        token_ids = tokenizer.encode(
+        token_ids = cached_encode(
+            tokenizer,
             self.bos_image_placeholder,
             add_special_tokens=False,
         )

vllm/model_executor/models/moss_audio.py

@@ -53,6 +53,7 @@ from vllm.multimodal.processing import (
     PromptReplacement,
     PromptUpdate,
     PromptUpdateDetails,
+    cached_encode,
 )
 from vllm.sequence import IntermediateTensors
 from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
@@ -1376,6 +1377,8 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
         out_mm_kwargs: MultiModalKwargsItems,
     ) -> Sequence[PromptUpdate]:
         processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+        tokenizer = processor.tokenizer
+
         out_mm_data = out_mm_kwargs.get_data()
         audio_data_seqlens = out_mm_data.get("audio_data_seqlens")
         if audio_data_seqlens is None:
@@ -1431,11 +1434,13 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
             )
         ]
         for suffix in ("", "\n"):
-            tokenizer_target = processor.tokenizer.encode(
+            tokenizer_target = cached_encode(
+                tokenizer,
                 MOSS_AUDIO_PLACEHOLDER + suffix,
                 add_special_tokens=False,
             )
-            suffix_token_ids = processor.tokenizer.encode(
+            suffix_token_ids = cached_encode(
+                tokenizer,
                 suffix,
                 add_special_tokens=False,
             )

vllm/model_executor/models/moss_transcribe_diarize.py

@@ -449,10 +449,12 @@ class MossTranscribeDiarizeDummyInputsBuilder(
         dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
         num_audios = mm_counts.get("audio", 0)
         tokenizer = self.info.get_tokenizer()
-        prompt = tokenizer.encode(
+        prompt = cached_encode(
+            tokenizer,
             AUDIO_PLACEHOLDER * num_audios,
             add_special_tokens=False,
-        ) or tokenizer.encode(
+        ) or cached_encode(
+            tokenizer,
             "\n",
             add_special_tokens=False,
         )

vllm/model_executor/models/paligemma.py

@@ -216,8 +216,9 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
         prompt_token_ids = mm_inputs["prompt_token_ids"]
 
         tokenizer = self.info.get_tokenizer()
-        newline_prompt = "\n"
-        newline_token_id = tokenizer.encode(newline_prompt)[-1]  # 108
+        vocab = tokenizer.get_vocab()
+        newline_token_id = vocab["\n"]
+
         # Force to add newline at the end of prompt for paligemma's format
         # This step can NOT be replacemented by current PromptUpdate methods
         if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id:

vllm/model_executor/models/phi3v.py

@@ -535,7 +535,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
 
         # Keep the behavior in line with HF processor
         if len(mm_prompt_updates) and (
-            token_ids[:2] == tokenizer.encode("<s> <|image|>", add_special_tokens=False)
+            token_ids[:2]
+            == cached_encode(tokenizer, "<s> <|image|>", add_special_tokens=False)
         ):
             token_ids = [token_ids[0], *token_ids[2:]]
             placeholders = {

vllm/model_executor/models/qwen3_asr.py

@@ -81,6 +81,7 @@ from vllm.multimodal.processing import (
     BaseProcessingInfo,
     PromptReplacement,
     PromptUpdate,
+    cached_encode,
 )
 from vllm.sequence import IntermediateTensors
 from vllm.tokenizers import cached_tokenizer_from_config
@@ -691,7 +692,7 @@ class Qwen3ASRForConditionalGeneration(
             full_lang_name = cls.supported_languages.get(lang_code, lang_code)
             prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}"
 
-        prompt_token_ids = tokenizer.encode(prompt)
+        prompt_token_ids = cached_encode(tokenizer, prompt)
 
         return TokensPrompt(
             prompt_token_ids=prompt_token_ids,

vllm/model_executor/models/qwen3_asr_realtime.py

@@ -39,10 +39,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
 from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
 from vllm.multimodal.parse import MultiModalDataItems
-from vllm.multimodal.processing import BaseDummyInputsBuilder
 from vllm.multimodal.processing.processor import (
+    BaseDummyInputsBuilder,
     MultiModalPromptUpdates,
     PlaceholderFeaturesInfo,
+    cached_encode,
 )
 from vllm.tokenizers import cached_tokenizer_from_config
 from vllm.transformers_utils.processor import cached_processor_from_config
@@ -205,7 +206,7 @@ class Qwen3ASRRealtimeGeneration(Qwen3ASRForConditionalGeneration, SupportsRealt
             f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n"
         )
 
-        prompt_token_ids = tokenizer.encode(prompt_template)
+        prompt_token_ids = cached_encode(tokenizer, prompt_template)
 
         async for audio_chunk in audio_stream:
             buffer.write_audio(audio_chunk)

vllm/model_executor/models/transformers/multimodal.py

@@ -841,7 +841,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
         replacements = defaultdict[str, list[list[int]]](list)
         for entry in offsets[0]:
             replacements[entry["type"]].append(
-                tokenizer.encode(entry["replacement"], add_special_tokens=False)
+                cached_encode(tokenizer, entry["replacement"], add_special_tokens=False)
             )
 
         for modality, seqs in replacements.items():

vllm/multimodal/processing/dummy_inputs.py

@@ -91,7 +91,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
             # prompt tokens.
             dummy_prompt = []
         else:
-            dummy_prompt = tokenizer.encode(dummy_text, truncation=False)
+            from .processor import cached_encode
+
+            dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)
 
         return ProcessorInputs(
             prompt=dummy_prompt,

vllm/multimodal/processing/processor.py

@@ -57,10 +57,15 @@ def cached_encode(
     tokenizer: TokenizerLike,
     text: str,
     *,
+    truncation: bool | None = None,
     add_special_tokens: bool = True,
 ) -> list[int]:
     """Encode text while caching repeated tokenizer calls."""
-    return tokenizer.encode(text, add_special_tokens=add_special_tokens)
+    return tokenizer.encode(
+        text,
+        truncation=truncation,
+        add_special_tokens=add_special_tokens,
+    )
 
 
 class _GetMatchIndex(Protocol):