[MM] Cache common token sequences (#53560)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Signed-off-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-24 07:48:10 -07:00
committed by GitHub
parent 4ca856b0b5
commit ecfa7bb373
14 changed files with 40 additions and 23 deletions
+2 -6
View File
@@ -1082,7 +1082,7 @@ class _FakeTokenizer:
_MERGES = {"food": (1000,), "foo": (101, 111, 111)} _MERGES = {"food": (1000,), "foo": (101, 111, 111)}
_INVERSE = {ids: text for text, ids in _MERGES.items()} _INVERSE = {ids: text for text, ids in _MERGES.items()}
def encode(self, text: str, add_special_tokens: bool = True) -> list[int]: def encode(self, text: str, **kwargs) -> list[int]:
token_ids = list[int]() token_ids = list[int]()
pos = 0 pos = 0
while pos < len(text): while pos < len(text):
@@ -1097,11 +1097,7 @@ class _FakeTokenizer:
pos += 1 pos += 1
return token_ids return token_ids
def decode( def decode(self, token_ids: list[int], **kwargs) -> str:
self,
token_ids: list[int],
skip_special_tokens: bool = False,
) -> str:
chars = list[str]() chars = list[str]()
pos = 0 pos = 0
while pos < len(token_ids): while pos < len(token_ids):
@@ -98,6 +98,7 @@ from vllm.multimodal.processing import (
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
PromptUpdateDetails, PromptUpdateDetails,
cached_encode,
) )
from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder
from vllm.multimodal.video import ( from vllm.multimodal.video import (
@@ -1894,7 +1895,7 @@ class LlavaOnevision2MultiModalProcessor(
video_pad_id = vocab["<|video_pad|>"] video_pad_id = vocab["<|video_pad|>"]
vision_start_id = vocab["<|vision_start|>"] vision_start_id = vocab["<|vision_start|>"]
vision_end_id = vocab["<|vision_end|>"] vision_end_id = vocab["<|vision_end|>"]
newline_ids = tokenizer.encode("\n", add_special_tokens=False) newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
merge_length = image_processor.merge_size**2 merge_length = image_processor.merge_size**2
decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1)) decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1))
+1 -1
View File
@@ -1050,7 +1050,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
for modality, pattern in placeholders: for modality, pattern in placeholders:
sub_pattern = tokenizer.decode( sub_pattern = tokenizer.decode(
tokenizer.encode(pattern, add_special_tokens=False) cached_encode(tokenizer, pattern, add_special_tokens=False)
) )
if sub_pattern != pattern: if sub_pattern != pattern:
additional_placeholders.append((modality, sub_pattern)) additional_placeholders.append((modality, sub_pattern))
+1 -1
View File
@@ -328,7 +328,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
additional_placeholders = [] additional_placeholders = []
for modality, pattern in placeholders: for modality, pattern in placeholders:
sub_pattern = tokenizer.decode( sub_pattern = tokenizer.decode(
tokenizer.encode(pattern, add_special_tokens=False) cached_encode(tokenizer, pattern, add_special_tokens=False)
) )
if sub_pattern != pattern: if sub_pattern != pattern:
additional_placeholders.append((modality, sub_pattern)) additional_placeholders.append((modality, sub_pattern))
+3 -1
View File
@@ -57,6 +57,7 @@ from vllm.multimodal.processing import (
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
PromptUpdateDetails, PromptUpdateDetails,
cached_encode,
) )
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.moondream3 import ( from vllm.transformers_utils.configs.moondream3 import (
@@ -945,7 +946,8 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
@cached_property @cached_property
def bos_image_placeholder_tokens(self) -> list[int]: def bos_image_placeholder_tokens(self) -> list[int]:
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
token_ids = tokenizer.encode( token_ids = cached_encode(
tokenizer,
self.bos_image_placeholder, self.bos_image_placeholder,
add_special_tokens=False, add_special_tokens=False,
) )
+7 -2
View File
@@ -53,6 +53,7 @@ from vllm.multimodal.processing import (
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
PromptUpdateDetails, PromptUpdateDetails,
cached_encode,
) )
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
@@ -1376,6 +1377,8 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
out_mm_kwargs: MultiModalKwargsItems, out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]: ) -> Sequence[PromptUpdate]:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = processor.tokenizer
out_mm_data = out_mm_kwargs.get_data() out_mm_data = out_mm_kwargs.get_data()
audio_data_seqlens = out_mm_data.get("audio_data_seqlens") audio_data_seqlens = out_mm_data.get("audio_data_seqlens")
if audio_data_seqlens is None: if audio_data_seqlens is None:
@@ -1431,11 +1434,13 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
) )
] ]
for suffix in ("", "\n"): for suffix in ("", "\n"):
tokenizer_target = processor.tokenizer.encode( tokenizer_target = cached_encode(
tokenizer,
MOSS_AUDIO_PLACEHOLDER + suffix, MOSS_AUDIO_PLACEHOLDER + suffix,
add_special_tokens=False, add_special_tokens=False,
) )
suffix_token_ids = processor.tokenizer.encode( suffix_token_ids = cached_encode(
tokenizer,
suffix, suffix,
add_special_tokens=False, add_special_tokens=False,
) )
@@ -449,10 +449,12 @@ class MossTranscribeDiarizeDummyInputsBuilder(
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
num_audios = mm_counts.get("audio", 0) num_audios = mm_counts.get("audio", 0)
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
prompt = tokenizer.encode( prompt = cached_encode(
tokenizer,
AUDIO_PLACEHOLDER * num_audios, AUDIO_PLACEHOLDER * num_audios,
add_special_tokens=False, add_special_tokens=False,
) or tokenizer.encode( ) or cached_encode(
tokenizer,
"\n", "\n",
add_special_tokens=False, add_special_tokens=False,
) )
+3 -2
View File
@@ -216,8 +216,9 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
prompt_token_ids = mm_inputs["prompt_token_ids"] prompt_token_ids = mm_inputs["prompt_token_ids"]
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
newline_prompt = "\n" vocab = tokenizer.get_vocab()
newline_token_id = tokenizer.encode(newline_prompt)[-1] # 108 newline_token_id = vocab["\n"]
# Force to add newline at the end of prompt for paligemma's format # Force to add newline at the end of prompt for paligemma's format
# This step can NOT be replacemented by current PromptUpdate methods # This step can NOT be replacemented by current PromptUpdate methods
if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id: if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id:
+2 -1
View File
@@ -535,7 +535,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
# Keep the behavior in line with HF processor # Keep the behavior in line with HF processor
if len(mm_prompt_updates) and ( if len(mm_prompt_updates) and (
token_ids[:2] == tokenizer.encode("<s> <|image|>", add_special_tokens=False) token_ids[:2]
== cached_encode(tokenizer, "<s> <|image|>", add_special_tokens=False)
): ):
token_ids = [token_ids[0], *token_ids[2:]] token_ids = [token_ids[0], *token_ids[2:]]
placeholders = { placeholders = {
+2 -1
View File
@@ -81,6 +81,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo, BaseProcessingInfo,
PromptReplacement, PromptReplacement,
PromptUpdate, PromptUpdate,
cached_encode,
) )
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers import cached_tokenizer_from_config
@@ -691,7 +692,7 @@ class Qwen3ASRForConditionalGeneration(
full_lang_name = cls.supported_languages.get(lang_code, lang_code) full_lang_name = cls.supported_languages.get(lang_code, lang_code)
prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}" prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}"
prompt_token_ids = tokenizer.encode(prompt) prompt_token_ids = cached_encode(tokenizer, prompt)
return TokensPrompt( return TokensPrompt(
prompt_token_ids=prompt_token_ids, prompt_token_ids=prompt_token_ids,
@@ -39,10 +39,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import BaseDummyInputsBuilder
from vllm.multimodal.processing.processor import ( from vllm.multimodal.processing.processor import (
BaseDummyInputsBuilder,
MultiModalPromptUpdates, MultiModalPromptUpdates,
PlaceholderFeaturesInfo, PlaceholderFeaturesInfo,
cached_encode,
) )
from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processor import cached_processor_from_config from vllm.transformers_utils.processor import cached_processor_from_config
@@ -205,7 +206,7 @@ class Qwen3ASRRealtimeGeneration(Qwen3ASRForConditionalGeneration, SupportsRealt
f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n" f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n"
) )
prompt_token_ids = tokenizer.encode(prompt_template) prompt_token_ids = cached_encode(tokenizer, prompt_template)
async for audio_chunk in audio_stream: async for audio_chunk in audio_stream:
buffer.write_audio(audio_chunk) buffer.write_audio(audio_chunk)
@@ -841,7 +841,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
replacements = defaultdict[str, list[list[int]]](list) replacements = defaultdict[str, list[list[int]]](list)
for entry in offsets[0]: for entry in offsets[0]:
replacements[entry["type"]].append( replacements[entry["type"]].append(
tokenizer.encode(entry["replacement"], add_special_tokens=False) cached_encode(tokenizer, entry["replacement"], add_special_tokens=False)
) )
for modality, seqs in replacements.items(): for modality, seqs in replacements.items():
+3 -1
View File
@@ -91,7 +91,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
# prompt tokens. # prompt tokens.
dummy_prompt = [] dummy_prompt = []
else: else:
dummy_prompt = tokenizer.encode(dummy_text, truncation=False) from .processor import cached_encode
dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)
return ProcessorInputs( return ProcessorInputs(
prompt=dummy_prompt, prompt=dummy_prompt,
+6 -1
View File
@@ -57,10 +57,15 @@ def cached_encode(
tokenizer: TokenizerLike, tokenizer: TokenizerLike,
text: str, text: str,
*, *,
truncation: bool | None = None,
add_special_tokens: bool = True, add_special_tokens: bool = True,
) -> list[int]: ) -> list[int]:
"""Encode text while caching repeated tokenizer calls.""" """Encode text while caching repeated tokenizer calls."""
return tokenizer.encode(text, add_special_tokens=add_special_tokens) return tokenizer.encode(
text,
truncation=truncation,
add_special_tokens=add_special_tokens,
)
class _GetMatchIndex(Protocol): class _GetMatchIndex(Protocol):