From ecfa7bb37316a3c1dab345fea4178d81f63b1ce4 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Mon, 24 Aug 2026 22:48:10 +0800 Subject: [PATCH] [MM] Cache common token sequences (#53560) Signed-off-by: DarkLight1337 Signed-off-by: Cyrus Leung --- tests/multimodal/test_processing.py | 8 ++------ vllm/model_executor/models/llava_onevision2.py | 3 ++- vllm/model_executor/models/minicpmv.py | 2 +- vllm/model_executor/models/minicpmv4_6.py | 2 +- vllm/model_executor/models/moondream3.py | 4 +++- vllm/model_executor/models/moss_audio.py | 9 +++++++-- vllm/model_executor/models/moss_transcribe_diarize.py | 6 ++++-- vllm/model_executor/models/paligemma.py | 5 +++-- vllm/model_executor/models/phi3v.py | 3 ++- vllm/model_executor/models/qwen3_asr.py | 3 ++- vllm/model_executor/models/qwen3_asr_realtime.py | 5 +++-- vllm/model_executor/models/transformers/multimodal.py | 2 +- vllm/multimodal/processing/dummy_inputs.py | 4 +++- vllm/multimodal/processing/processor.py | 7 ++++++- 14 files changed, 40 insertions(+), 23 deletions(-) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index ec6a8f93df..a3f4cf7e47 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1082,7 +1082,7 @@ class _FakeTokenizer: _MERGES = {"food": (1000,), "foo": (101, 111, 111)} _INVERSE = {ids: text for text, ids in _MERGES.items()} - def encode(self, text: str, add_special_tokens: bool = True) -> list[int]: + def encode(self, text: str, **kwargs) -> list[int]: token_ids = list[int]() pos = 0 while pos < len(text): @@ -1097,11 +1097,7 @@ class _FakeTokenizer: pos += 1 return token_ids - def decode( - self, - token_ids: list[int], - skip_special_tokens: bool = False, - ) -> str: + def decode(self, token_ids: list[int], **kwargs) -> str: chars = list[str]() pos = 0 while pos < len(token_ids): diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index 5dfb834128..ba3678fbc8 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -98,6 +98,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, ) from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder from vllm.multimodal.video import ( @@ -1894,7 +1895,7 @@ class LlavaOnevision2MultiModalProcessor( video_pad_id = vocab["<|video_pad|>"] vision_start_id = vocab["<|vision_start|>"] vision_end_id = vocab["<|vision_end|>"] - newline_ids = tokenizer.encode("\n", add_special_tokens=False) + newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False) merge_length = image_processor.merge_size**2 decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1)) diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 6f1e170120..55853c86e2 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -1050,7 +1050,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): tokenizer = self.info.get_tokenizer() for modality, pattern in placeholders: sub_pattern = tokenizer.decode( - tokenizer.encode(pattern, add_special_tokens=False) + cached_encode(tokenizer, pattern, add_special_tokens=False) ) if sub_pattern != pattern: additional_placeholders.append((modality, sub_pattern)) diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py index dab93a66e5..618e31242f 100644 --- a/vllm/model_executor/models/minicpmv4_6.py +++ b/vllm/model_executor/models/minicpmv4_6.py @@ -328,7 +328,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): additional_placeholders = [] for modality, pattern in placeholders: sub_pattern = tokenizer.decode( - tokenizer.encode(pattern, add_special_tokens=False) + cached_encode(tokenizer, pattern, add_special_tokens=False) ) if sub_pattern != pattern: additional_placeholders.append((modality, sub_pattern)) diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py index 45833f6241..1331408dc5 100644 --- a/vllm/model_executor/models/moondream3.py +++ b/vllm/model_executor/models/moondream3.py @@ -57,6 +57,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.moondream3 import ( @@ -945,7 +946,8 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing @cached_property def bos_image_placeholder_tokens(self) -> list[int]: tokenizer = self.info.get_tokenizer() - token_ids = tokenizer.encode( + token_ids = cached_encode( + tokenizer, self.bos_image_placeholder, add_special_tokens=False, ) diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index 248606ceb1..70ce6ac4e2 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -53,6 +53,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.repo_utils import get_hf_file_to_dict @@ -1376,6 +1377,8 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn out_mm_kwargs: MultiModalKwargsItems, ) -> Sequence[PromptUpdate]: processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + tokenizer = processor.tokenizer + out_mm_data = out_mm_kwargs.get_data() audio_data_seqlens = out_mm_data.get("audio_data_seqlens") if audio_data_seqlens is None: @@ -1431,11 +1434,13 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn ) ] for suffix in ("", "\n"): - tokenizer_target = processor.tokenizer.encode( + tokenizer_target = cached_encode( + tokenizer, MOSS_AUDIO_PLACEHOLDER + suffix, add_special_tokens=False, ) - suffix_token_ids = processor.tokenizer.encode( + suffix_token_ids = cached_encode( + tokenizer, suffix, add_special_tokens=False, ) diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index b558e6ba3b..c26ebf5206 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -449,10 +449,12 @@ class MossTranscribeDiarizeDummyInputsBuilder( dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) num_audios = mm_counts.get("audio", 0) tokenizer = self.info.get_tokenizer() - prompt = tokenizer.encode( + prompt = cached_encode( + tokenizer, AUDIO_PLACEHOLDER * num_audios, add_special_tokens=False, - ) or tokenizer.encode( + ) or cached_encode( + tokenizer, "\n", add_special_tokens=False, ) diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index 781e9a620d..3160341aff 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -216,8 +216,9 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn prompt_token_ids = mm_inputs["prompt_token_ids"] tokenizer = self.info.get_tokenizer() - newline_prompt = "\n" - newline_token_id = tokenizer.encode(newline_prompt)[-1] # 108 + vocab = tokenizer.get_vocab() + newline_token_id = vocab["\n"] + # Force to add newline at the end of prompt for paligemma's format # This step can NOT be replacemented by current PromptUpdate methods if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id: diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py index cb870728a4..478e610489 100644 --- a/vllm/model_executor/models/phi3v.py +++ b/vllm/model_executor/models/phi3v.py @@ -535,7 +535,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]): # Keep the behavior in line with HF processor if len(mm_prompt_updates) and ( - token_ids[:2] == tokenizer.encode(" <|image|>", add_special_tokens=False) + token_ids[:2] + == cached_encode(tokenizer, " <|image|>", add_special_tokens=False) ): token_ids = [token_ids[0], *token_ids[2:]] placeholders = { diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index a65bff986b..7cea97ef5b 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -81,6 +81,7 @@ from vllm.multimodal.processing import ( BaseProcessingInfo, PromptReplacement, PromptUpdate, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config @@ -691,7 +692,7 @@ class Qwen3ASRForConditionalGeneration( full_lang_name = cls.supported_languages.get(lang_code, lang_code) prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}" - prompt_token_ids = tokenizer.encode(prompt) + prompt_token_ids = cached_encode(tokenizer, prompt) return TokensPrompt( prompt_token_ids=prompt_token_ids, diff --git a/vllm/model_executor/models/qwen3_asr_realtime.py b/vllm/model_executor/models/qwen3_asr_realtime.py index 917bfce134..15ecd861f1 100644 --- a/vllm/model_executor/models/qwen3_asr_realtime.py +++ b/vllm/model_executor/models/qwen3_asr_realtime.py @@ -39,10 +39,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache from vllm.multimodal.inputs import MultiModalKwargsOptionalItems from vllm.multimodal.parse import MultiModalDataItems -from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( + BaseDummyInputsBuilder, MultiModalPromptUpdates, PlaceholderFeaturesInfo, + cached_encode, ) from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.processor import cached_processor_from_config @@ -205,7 +206,7 @@ class Qwen3ASRRealtimeGeneration(Qwen3ASRForConditionalGeneration, SupportsRealt f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n" ) - prompt_token_ids = tokenizer.encode(prompt_template) + prompt_token_ids = cached_encode(tokenizer, prompt_template) async for audio_chunk in audio_stream: buffer.write_audio(audio_chunk) diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 0caecb8d9d..e0911d8e2c 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -841,7 +841,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase): replacements = defaultdict[str, list[list[int]]](list) for entry in offsets[0]: replacements[entry["type"]].append( - tokenizer.encode(entry["replacement"], add_special_tokens=False) + cached_encode(tokenizer, entry["replacement"], add_special_tokens=False) ) for modality, seqs in replacements.items(): diff --git a/vllm/multimodal/processing/dummy_inputs.py b/vllm/multimodal/processing/dummy_inputs.py index ac3fe54721..29f373ccc8 100644 --- a/vllm/multimodal/processing/dummy_inputs.py +++ b/vllm/multimodal/processing/dummy_inputs.py @@ -91,7 +91,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]): # prompt tokens. dummy_prompt = [] else: - dummy_prompt = tokenizer.encode(dummy_text, truncation=False) + from .processor import cached_encode + + dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False) return ProcessorInputs( prompt=dummy_prompt, diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 1056611da5..2c3ac6d591 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -57,10 +57,15 @@ def cached_encode( tokenizer: TokenizerLike, text: str, *, + truncation: bool | None = None, add_special_tokens: bool = True, ) -> list[int]: """Encode text while caching repeated tokenizer calls.""" - return tokenizer.encode(text, add_special_tokens=add_special_tokens) + return tokenizer.encode( + text, + truncation=truncation, + add_special_tokens=add_special_tokens, + ) class _GetMatchIndex(Protocol):