[MM] Cache common token sequences (#53560)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1082,7 +1082,7 @@ class _FakeTokenizer:
|
|||||||
_MERGES = {"food": (1000,), "foo": (101, 111, 111)}
|
_MERGES = {"food": (1000,), "foo": (101, 111, 111)}
|
||||||
_INVERSE = {ids: text for text, ids in _MERGES.items()}
|
_INVERSE = {ids: text for text, ids in _MERGES.items()}
|
||||||
|
|
||||||
def encode(self, text: str, add_special_tokens: bool = True) -> list[int]:
|
def encode(self, text: str, **kwargs) -> list[int]:
|
||||||
token_ids = list[int]()
|
token_ids = list[int]()
|
||||||
pos = 0
|
pos = 0
|
||||||
while pos < len(text):
|
while pos < len(text):
|
||||||
@@ -1097,11 +1097,7 @@ class _FakeTokenizer:
|
|||||||
pos += 1
|
pos += 1
|
||||||
return token_ids
|
return token_ids
|
||||||
|
|
||||||
def decode(
|
def decode(self, token_ids: list[int], **kwargs) -> str:
|
||||||
self,
|
|
||||||
token_ids: list[int],
|
|
||||||
skip_special_tokens: bool = False,
|
|
||||||
) -> str:
|
|
||||||
chars = list[str]()
|
chars = list[str]()
|
||||||
pos = 0
|
pos = 0
|
||||||
while pos < len(token_ids):
|
while pos < len(token_ids):
|
||||||
|
|||||||
@@ -98,6 +98,7 @@ from vllm.multimodal.processing import (
|
|||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
PromptUpdateDetails,
|
PromptUpdateDetails,
|
||||||
|
cached_encode,
|
||||||
)
|
)
|
||||||
from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder
|
from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder
|
||||||
from vllm.multimodal.video import (
|
from vllm.multimodal.video import (
|
||||||
@@ -1894,7 +1895,7 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
video_pad_id = vocab["<|video_pad|>"]
|
video_pad_id = vocab["<|video_pad|>"]
|
||||||
vision_start_id = vocab["<|vision_start|>"]
|
vision_start_id = vocab["<|vision_start|>"]
|
||||||
vision_end_id = vocab["<|vision_end|>"]
|
vision_end_id = vocab["<|vision_end|>"]
|
||||||
newline_ids = tokenizer.encode("\n", add_special_tokens=False)
|
newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
|
||||||
merge_length = image_processor.merge_size**2
|
merge_length = image_processor.merge_size**2
|
||||||
decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1))
|
decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1))
|
||||||
|
|
||||||
|
|||||||
@@ -1050,7 +1050,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
for modality, pattern in placeholders:
|
for modality, pattern in placeholders:
|
||||||
sub_pattern = tokenizer.decode(
|
sub_pattern = tokenizer.decode(
|
||||||
tokenizer.encode(pattern, add_special_tokens=False)
|
cached_encode(tokenizer, pattern, add_special_tokens=False)
|
||||||
)
|
)
|
||||||
if sub_pattern != pattern:
|
if sub_pattern != pattern:
|
||||||
additional_placeholders.append((modality, sub_pattern))
|
additional_placeholders.append((modality, sub_pattern))
|
||||||
|
|||||||
@@ -328,7 +328,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
|||||||
additional_placeholders = []
|
additional_placeholders = []
|
||||||
for modality, pattern in placeholders:
|
for modality, pattern in placeholders:
|
||||||
sub_pattern = tokenizer.decode(
|
sub_pattern = tokenizer.decode(
|
||||||
tokenizer.encode(pattern, add_special_tokens=False)
|
cached_encode(tokenizer, pattern, add_special_tokens=False)
|
||||||
)
|
)
|
||||||
if sub_pattern != pattern:
|
if sub_pattern != pattern:
|
||||||
additional_placeholders.append((modality, sub_pattern))
|
additional_placeholders.append((modality, sub_pattern))
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ from vllm.multimodal.processing import (
|
|||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
PromptUpdateDetails,
|
PromptUpdateDetails,
|
||||||
|
cached_encode,
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.transformers_utils.configs.moondream3 import (
|
from vllm.transformers_utils.configs.moondream3 import (
|
||||||
@@ -945,7 +946,8 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing
|
|||||||
@cached_property
|
@cached_property
|
||||||
def bos_image_placeholder_tokens(self) -> list[int]:
|
def bos_image_placeholder_tokens(self) -> list[int]:
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
token_ids = tokenizer.encode(
|
token_ids = cached_encode(
|
||||||
|
tokenizer,
|
||||||
self.bos_image_placeholder,
|
self.bos_image_placeholder,
|
||||||
add_special_tokens=False,
|
add_special_tokens=False,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ from vllm.multimodal.processing import (
|
|||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
PromptUpdateDetails,
|
PromptUpdateDetails,
|
||||||
|
cached_encode,
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||||
@@ -1376,6 +1377,8 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
|||||||
out_mm_kwargs: MultiModalKwargsItems,
|
out_mm_kwargs: MultiModalKwargsItems,
|
||||||
) -> Sequence[PromptUpdate]:
|
) -> Sequence[PromptUpdate]:
|
||||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
tokenizer = processor.tokenizer
|
||||||
|
|
||||||
out_mm_data = out_mm_kwargs.get_data()
|
out_mm_data = out_mm_kwargs.get_data()
|
||||||
audio_data_seqlens = out_mm_data.get("audio_data_seqlens")
|
audio_data_seqlens = out_mm_data.get("audio_data_seqlens")
|
||||||
if audio_data_seqlens is None:
|
if audio_data_seqlens is None:
|
||||||
@@ -1431,11 +1434,13 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
for suffix in ("", "\n"):
|
for suffix in ("", "\n"):
|
||||||
tokenizer_target = processor.tokenizer.encode(
|
tokenizer_target = cached_encode(
|
||||||
|
tokenizer,
|
||||||
MOSS_AUDIO_PLACEHOLDER + suffix,
|
MOSS_AUDIO_PLACEHOLDER + suffix,
|
||||||
add_special_tokens=False,
|
add_special_tokens=False,
|
||||||
)
|
)
|
||||||
suffix_token_ids = processor.tokenizer.encode(
|
suffix_token_ids = cached_encode(
|
||||||
|
tokenizer,
|
||||||
suffix,
|
suffix,
|
||||||
add_special_tokens=False,
|
add_special_tokens=False,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -449,10 +449,12 @@ class MossTranscribeDiarizeDummyInputsBuilder(
|
|||||||
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
|
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
|
||||||
num_audios = mm_counts.get("audio", 0)
|
num_audios = mm_counts.get("audio", 0)
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
prompt = tokenizer.encode(
|
prompt = cached_encode(
|
||||||
|
tokenizer,
|
||||||
AUDIO_PLACEHOLDER * num_audios,
|
AUDIO_PLACEHOLDER * num_audios,
|
||||||
add_special_tokens=False,
|
add_special_tokens=False,
|
||||||
) or tokenizer.encode(
|
) or cached_encode(
|
||||||
|
tokenizer,
|
||||||
"\n",
|
"\n",
|
||||||
add_special_tokens=False,
|
add_special_tokens=False,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -216,8 +216,9 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
|
|||||||
prompt_token_ids = mm_inputs["prompt_token_ids"]
|
prompt_token_ids = mm_inputs["prompt_token_ids"]
|
||||||
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
newline_prompt = "\n"
|
vocab = tokenizer.get_vocab()
|
||||||
newline_token_id = tokenizer.encode(newline_prompt)[-1] # 108
|
newline_token_id = vocab["\n"]
|
||||||
|
|
||||||
# Force to add newline at the end of prompt for paligemma's format
|
# Force to add newline at the end of prompt for paligemma's format
|
||||||
# This step can NOT be replacemented by current PromptUpdate methods
|
# This step can NOT be replacemented by current PromptUpdate methods
|
||||||
if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id:
|
if len(prompt_token_ids) and prompt_token_ids[-1] != newline_token_id:
|
||||||
|
|||||||
@@ -535,7 +535,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
|||||||
|
|
||||||
# Keep the behavior in line with HF processor
|
# Keep the behavior in line with HF processor
|
||||||
if len(mm_prompt_updates) and (
|
if len(mm_prompt_updates) and (
|
||||||
token_ids[:2] == tokenizer.encode("<s> <|image|>", add_special_tokens=False)
|
token_ids[:2]
|
||||||
|
== cached_encode(tokenizer, "<s> <|image|>", add_special_tokens=False)
|
||||||
):
|
):
|
||||||
token_ids = [token_ids[0], *token_ids[2:]]
|
token_ids = [token_ids[0], *token_ids[2:]]
|
||||||
placeholders = {
|
placeholders = {
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ from vllm.multimodal.processing import (
|
|||||||
BaseProcessingInfo,
|
BaseProcessingInfo,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
|
cached_encode,
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.tokenizers import cached_tokenizer_from_config
|
from vllm.tokenizers import cached_tokenizer_from_config
|
||||||
@@ -691,7 +692,7 @@ class Qwen3ASRForConditionalGeneration(
|
|||||||
full_lang_name = cls.supported_languages.get(lang_code, lang_code)
|
full_lang_name = cls.supported_languages.get(lang_code, lang_code)
|
||||||
prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}"
|
prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}"
|
||||||
|
|
||||||
prompt_token_ids = tokenizer.encode(prompt)
|
prompt_token_ids = cached_encode(tokenizer, prompt)
|
||||||
|
|
||||||
return TokensPrompt(
|
return TokensPrompt(
|
||||||
prompt_token_ids=prompt_token_ids,
|
prompt_token_ids=prompt_token_ids,
|
||||||
|
|||||||
@@ -39,10 +39,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
|||||||
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache
|
||||||
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
|
from vllm.multimodal.inputs import MultiModalKwargsOptionalItems
|
||||||
from vllm.multimodal.parse import MultiModalDataItems
|
from vllm.multimodal.parse import MultiModalDataItems
|
||||||
from vllm.multimodal.processing import BaseDummyInputsBuilder
|
|
||||||
from vllm.multimodal.processing.processor import (
|
from vllm.multimodal.processing.processor import (
|
||||||
|
BaseDummyInputsBuilder,
|
||||||
MultiModalPromptUpdates,
|
MultiModalPromptUpdates,
|
||||||
PlaceholderFeaturesInfo,
|
PlaceholderFeaturesInfo,
|
||||||
|
cached_encode,
|
||||||
)
|
)
|
||||||
from vllm.tokenizers import cached_tokenizer_from_config
|
from vllm.tokenizers import cached_tokenizer_from_config
|
||||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||||
@@ -205,7 +206,7 @@ class Qwen3ASRRealtimeGeneration(Qwen3ASRForConditionalGeneration, SupportsRealt
|
|||||||
f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n"
|
f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
prompt_token_ids = tokenizer.encode(prompt_template)
|
prompt_token_ids = cached_encode(tokenizer, prompt_template)
|
||||||
|
|
||||||
async for audio_chunk in audio_stream:
|
async for audio_chunk in audio_stream:
|
||||||
buffer.write_audio(audio_chunk)
|
buffer.write_audio(audio_chunk)
|
||||||
|
|||||||
@@ -841,7 +841,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
|||||||
replacements = defaultdict[str, list[list[int]]](list)
|
replacements = defaultdict[str, list[list[int]]](list)
|
||||||
for entry in offsets[0]:
|
for entry in offsets[0]:
|
||||||
replacements[entry["type"]].append(
|
replacements[entry["type"]].append(
|
||||||
tokenizer.encode(entry["replacement"], add_special_tokens=False)
|
cached_encode(tokenizer, entry["replacement"], add_special_tokens=False)
|
||||||
)
|
)
|
||||||
|
|
||||||
for modality, seqs in replacements.items():
|
for modality, seqs in replacements.items():
|
||||||
|
|||||||
@@ -91,7 +91,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
|||||||
# prompt tokens.
|
# prompt tokens.
|
||||||
dummy_prompt = []
|
dummy_prompt = []
|
||||||
else:
|
else:
|
||||||
dummy_prompt = tokenizer.encode(dummy_text, truncation=False)
|
from .processor import cached_encode
|
||||||
|
|
||||||
|
dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)
|
||||||
|
|
||||||
return ProcessorInputs(
|
return ProcessorInputs(
|
||||||
prompt=dummy_prompt,
|
prompt=dummy_prompt,
|
||||||
|
|||||||
@@ -57,10 +57,15 @@ def cached_encode(
|
|||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
text: str,
|
text: str,
|
||||||
*,
|
*,
|
||||||
|
truncation: bool | None = None,
|
||||||
add_special_tokens: bool = True,
|
add_special_tokens: bool = True,
|
||||||
) -> list[int]:
|
) -> list[int]:
|
||||||
"""Encode text while caching repeated tokenizer calls."""
|
"""Encode text while caching repeated tokenizer calls."""
|
||||||
return tokenizer.encode(text, add_special_tokens=add_special_tokens)
|
return tokenizer.encode(
|
||||||
|
text,
|
||||||
|
truncation=truncation,
|
||||||
|
add_special_tokens=add_special_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class _GetMatchIndex(Protocol):
|
class _GetMatchIndex(Protocol):
|
||||||
|
|||||||
Reference in New Issue
Block a user