From ebcd60646741be239026db75358715ba707eb50f Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sun, 23 Aug 2026 14:50:32 +0800 Subject: [PATCH] [MM] Simplify prompt updates: replace `PromptSeq` with `list[int]` (#53372) Signed-off-by: DarkLight1337 Co-authored-by: Qwen Code --- docs/contributing/model/multimodal.md | 6 + .../multimodal/processing/test_openvla.py | 4 +- tests/multimodal/test_cache.py | 4 +- tests/multimodal/test_processing.py | 495 +++++----------- vllm/model_executor/models/audioflamingo3.py | 2 +- vllm/model_executor/models/cohere2_vision.py | 8 +- vllm/model_executor/models/ernie45_vl.py | 11 +- vllm/model_executor/models/funaudiochat.py | 2 +- vllm/model_executor/models/gemma3_mm.py | 11 +- vllm/model_executor/models/gemma3n_mm.py | 27 +- vllm/model_executor/models/gemma4_mm.py | 18 +- vllm/model_executor/models/glm4_1v.py | 13 +- vllm/model_executor/models/glmasr.py | 2 +- vllm/model_executor/models/h2ovl.py | 5 +- vllm/model_executor/models/idefics3.py | 15 +- vllm/model_executor/models/interns1.py | 31 +- vllm/model_executor/models/internvl.py | 9 +- vllm/model_executor/models/isaac.py | 11 +- vllm/model_executor/models/kanana_v.py | 5 +- vllm/model_executor/models/lfm2_vl.py | 15 +- vllm/model_executor/models/midashenglm.py | 2 +- vllm/model_executor/models/mimo_v2_omni.py | 2 +- vllm/model_executor/models/minicpmo.py | 19 +- vllm/model_executor/models/minicpmv.py | 69 ++- vllm/model_executor/models/minicpmv4_6.py | 77 ++- vllm/model_executor/models/mllama4.py | 12 +- vllm/model_executor/models/molmo.py | 5 +- vllm/model_executor/models/moss_audio.py | 4 +- .../models/moss_transcribe_diarize.py | 7 +- vllm/model_executor/models/muse_glimmer.py | 4 +- .../model_executor/models/nano_nemotron_vl.py | 14 +- vllm/model_executor/models/nvlm_d.py | 13 +- vllm/model_executor/models/openvla.py | 2 +- vllm/model_executor/models/ovis.py | 5 +- vllm/model_executor/models/phi3v.py | 16 +- vllm/model_executor/models/phi4mm.py | 31 +- vllm/model_executor/models/phi4siglip.py | 7 +- vllm/model_executor/models/pixtral.py | 2 +- .../models/qwen2_5_omni_thinker.py | 6 +- vllm/model_executor/models/qwen3_asr.py | 2 +- .../models/qwen3_omni_moe_thinker.py | 6 +- vllm/model_executor/models/qwen3_vl.py | 15 +- .../models/transformers/multimodal.py | 13 +- vllm/model_executor/models/ultravox.py | 12 +- vllm/model_executor/models/voxtral.py | 2 +- vllm/models/dots3_note/common/processor.py | 12 +- vllm/models/kimi_k3/common/mm_preprocess.py | 13 +- vllm/multimodal/processing/__init__.py | 2 + vllm/multimodal/processing/processor.py | 534 ++++++++---------- vllm/renderers/hf.py | 8 +- .../transformers_utils/processors/internvl.py | 31 +- .../processors/nano_nemotron_vl.py | 37 +- vllm/transformers_utils/processors/nvlm_d.py | 8 +- 53 files changed, 831 insertions(+), 855 deletions(-) diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index 36830dd8ea..c59c1a80bf 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -532,6 +532,12 @@ return a list of [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instanc Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies an update operation (e.g.: insertion, replacement) performed by the HF processor. +!!! note + The target and content of each update are token sequences. When converting + text to token sequences, remember to encode it with + `cached_encode(..., add_special_tokens=False)`. Otherwise, the updated prompt + may contain duplicated special tokens or fail to match the target. + === "Basic example: LLaVA" Looking at HF's `LlavaProcessor`: diff --git a/tests/models/multimodal/processing/test_openvla.py b/tests/models/multimodal/processing/test_openvla.py index 992608f893..36775d53e4 100644 --- a/tests/models/multimodal/processing/test_openvla.py +++ b/tests/models/multimodal/processing/test_openvla.py @@ -192,11 +192,11 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None: assert resolved.modality == "image" assert [ (match.start_idx, match.end_idx) - for match in resolved.iter_matches([1, 10, 11], _FakeTokenizer()) + for match in resolved.iter_token_matches([1, 10, 11]) ] == [(1, 1)] assert content.full == [32000] * 256 - is_embed = content.is_embed(None, content.full) + is_embed = content.is_embed(content.full) assert is_embed.dtype == torch.bool assert is_embed.tolist() == [True] * 256 diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index c20e554e57..7c93808f28 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -93,7 +93,7 @@ def test_cache_item_size(item, expected_size): cache[""] = item assert cache.currsize == expected_size - prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0) + prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0) cache[""] = MultiModalProcessorCacheItem(item, [prompt_update]) assert cache.currsize == expected_size @@ -151,7 +151,7 @@ def _compare_caches( for item in all_items ] - prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0) + prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0) for it in range(n_iter): num_items_to_select = rng.randint(0, max_items_per_iter) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 601fe9b8a8..ec6a8f93df 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -12,18 +12,19 @@ from vllm.exceptions import VLLMValidationError from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.processing.context import InputProcessingContext from vllm.multimodal.processing.processor import ( + BaseMultiModalProcessor, PlaceholderFeaturesInfo, PromptIndexTargets, PromptInsertion, PromptReplacement, _apply_matches, _apply_token_matches_with_placeholders, - apply_text_matches, apply_token_matches, find_mm_placeholders, iter_token_matches, replace_token_matches, ) +from vllm.utils.collection_utils import flatten_2d_lists from .utils import random_image @@ -242,7 +243,7 @@ def test_find_token_matches( for key, target in target_by_key.items() } result = { - key: list(update.iter_token_matches(prompt, tokenizer=None)) + key: list(update.iter_token_matches(prompt)) for key, update in prompt_updates.items() } @@ -259,315 +260,6 @@ def test_find_token_matches( } == expected_by_key -@pytest.mark.parametrize( - ("prompt", "target_by_key", "expected_by_key"), - [ - # Detokenized test cases of `test_find_token_matches` - # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf - ( - "", - { - "pattern_1": "", - "pattern_2": "", - "pattern_3": PromptIndexTargets.start(), - "pattern_4": PromptIndexTargets.prefix(""), - "pattern_5": PromptIndexTargets.end(), - }, - { - "pattern_1": [{"start_idx": 0, "end_idx": 0}], - "pattern_2": [], - "pattern_3": [ - {"start_idx": 0, "end_idx": 0}, - ], - "pattern_4": [], - "pattern_5": [ - {"start_idx": 0, "end_idx": 0}, - ], - }, - ), - ( - "", - { - "pattern_1": "", - "pattern_2": "", - "pattern_3": "", - "pattern_4": PromptIndexTargets.start(), - "pattern_5": PromptIndexTargets.prefix(""), - "pattern_6": PromptIndexTargets.end(), - }, - { - "pattern_1": [ - {"start_idx": 0, "end_idx": 7}, - {"start_idx": 7, "end_idx": 14}, - {"start_idx": 14, "end_idx": 21}, - {"start_idx": 21, "end_idx": 28}, - ], - "pattern_2": [ - {"start_idx": 0, "end_idx": 14}, - {"start_idx": 14, "end_idx": 28}, - ], - "pattern_3": [ - {"start_idx": 0, "end_idx": 21}, - ], - "pattern_4": [ - {"start_idx": 0, "end_idx": 0}, - ], - "pattern_5": [ - {"start_idx": 7, "end_idx": 7}, - ], - "pattern_6": [ - {"start_idx": 28, "end_idx": 28}, - ], - }, - ), - ( - "Image:Image:!", - { - "pattern_1": "Image:", - "pattern_2": "Image:", - "pattern_3": "Image:", - "pattern_4": PromptIndexTargets.start(), - "pattern_5": PromptIndexTargets.prefix("Image:"), - "pattern_6": PromptIndexTargets.end(), - }, - { - "pattern_1": [ - {"start_idx": 0, "end_idx": 13}, - {"start_idx": 27, "end_idx": 40}, - ], - "pattern_2": [ - {"start_idx": 0, "end_idx": 27}, - ], - "pattern_3": [], - "pattern_4": [ - {"start_idx": 0, "end_idx": 0}, - ], - "pattern_5": [ - {"start_idx": 13, "end_idx": 13}, - ], - "pattern_6": [ - {"start_idx": 48, "end_idx": 48}, - ], - }, - ), - # Test regex escape - ( - "<|image|><|image|>", - { - "pattern_1": "<|image|>", - "pattern_2": "<|image|>", - "pattern_3": "<|image|><|image|>", - }, - { - "pattern_1": [ - {"start_idx": 0, "end_idx": 9}, - {"start_idx": 16, "end_idx": 25}, - ], - "pattern_2": [ - {"start_idx": 0, "end_idx": 16}, - {"start_idx": 16, "end_idx": 32}, - ], - "pattern_3": [ - {"start_idx": 0, "end_idx": 25}, - ], - }, - ), - ], -) -@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement]) -def test_find_text_matches( - prompt, - target_by_key, - expected_by_key, - update_type, -): - prompt_updates = { - key: update_type(key, target, []).resolve(0) - for key, target in target_by_key.items() - } - result = { - key: list(update.iter_text_matches(prompt, tokenizer=None)) - for key, update in prompt_updates.items() - } - - # Only displayed on error - print("result:", result) - - # Manually constructed results - assert { - key: [ - dict(start_idx=item.start_idx, end_idx=item.end_idx) - for item in result.get(key, []) - ] - for key in expected_by_key - } == expected_by_key - - -@pytest.mark.parametrize( - ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501 - [ - ( - "Image:Image:!", - { - # We use `` before `Image:` to test matches that - # occur out of order - "pattern_1": "", - "pattern_2": "Image:", - "pattern_3": "!", - }, - { - # Test whether target is confused with replacement - "pattern_1": "", - # Test empty replacement - "pattern_2": "", - # Test dynamic replacement (beyond the form of `unit * count`) - "pattern_3": "?!?", - }, - { - PromptInsertion: { - 0: "Image:Image:!", - 1: "Image:Image:!?!?", - 2: "Image:Image:!?!??!?", # noqa: E501 - }, - PromptReplacement: { - 0: "Image:Image:!", - 1: "Image:?!?", - 2: "?!?", - }, - }, - ), - # Test index targets - ( - "", - { - "pattern_1": PromptIndexTargets.start(), - "pattern_2": PromptIndexTargets.prefix(""), - "pattern_3": PromptIndexTargets.end(), - }, - { - "pattern_1": "1", - "pattern_2": "2", - "pattern_3": "3", - }, - { - PromptInsertion: { - 0: "", - 1: "13", - 2: "1133", - }, - PromptReplacement: { - 0: "", - 1: "13", - 2: "1133", - }, - }, - ), - ( - "", - { - "pattern_1": PromptIndexTargets.start(), - "pattern_2": PromptIndexTargets.prefix(""), - "pattern_3": PromptIndexTargets.end(), - }, - { - "pattern_1": "1", - "pattern_2": "2", - "pattern_3": "3", - }, - { - PromptInsertion: { - 0: "", - 1: "123", - 2: "112233", - }, - PromptReplacement: { - 0: "", - 1: "123", - 2: "112233", - }, - }, - ), - # Test different replacement per item - ( - "", - { - "pattern_1": "", - }, - { - "pattern_1": lambda idx: str(idx + 1), - }, - { - PromptInsertion: { - 0: "", - 1: "1", - 2: "12", - }, - PromptReplacement: { - 0: "", - 1: "1", - 2: "12", - }, - }, - ), - ( - "", - { - "pattern_1": PromptIndexTargets.prefix(""), - }, - { - "pattern_1": lambda idx: str(idx + 1), - }, - { - PromptInsertion: { - 0: "", - 1: "1", - 2: "12", - }, - PromptReplacement: { - 0: "", - 1: "1", - 2: "12", - }, - }, - ), - ], -) -def test_find_update_text( - prompt, - target_by_key, - repl_by_key, - expected_by_update_type_mm_count, -): - for ( - update_type, - expected_by_mm_count, - ) in expected_by_update_type_mm_count.items(): - for mm_count, expected in expected_by_mm_count.items(): - mm_prompt_updates = { - key: [ - [update_type(key, target, repl_by_key[key]).resolve(i)] - for i in range(mm_count) - ] - for key, target in target_by_key.items() - } - - new_prompt, result = apply_text_matches( - prompt, - mm_prompt_updates, - tokenizer=None, - ) - - # Only displayed on error - print("update_type:", update_type) - print("mm_count:", mm_count) - print("mm_prompt_updates:", mm_prompt_updates) - print("new_prompt:", new_prompt) - print("result:", result) - - # Manually constructed results - assert new_prompt == expected - - FIND_UPDATE_TOKENS_TEST_CASES = [ # Tokenized test cases of `test_find_update_text` # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf @@ -891,11 +583,7 @@ def test_find_update_tokens( for key, target in target_by_key.items() } - new_prompt, result = apply_token_matches( - prompt, - mm_prompt_updates, - tokenizer=None, - ) + new_prompt, result = apply_token_matches(prompt, mm_prompt_updates) # Only displayed on error print("update_type:", update_type) @@ -945,7 +633,6 @@ def test_apply_token_matches_with_placeholders( new_prompt, result, placeholders = _apply_token_matches_with_placeholders( prompt, mm_prompt_updates, - tokenizer=None, ) if any( @@ -1097,7 +784,7 @@ def test_find_mm_placeholders( for key, repl in repl_by_key.items() } - result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None) + result = find_mm_placeholders(prompt, mm_prompt_updates) # Only displayed on error print("result:", result) @@ -1285,24 +972,18 @@ def test_apply_matches_no_match_exits_quickly(): With the fix, it should exit immediately when no match is found. """ # Create a long prompt with no placeholder - long_prompt = "x" * 10000 + long_prompt = [1] * 10000 # Create update looking for a placeholder that doesn't exist - mm_prompt_updates = { - "image": [[PromptReplacement("image", "", "REPLACED").resolve(0)]] - } + mm_prompt_updates = {"image": [[PromptReplacement("image", [0], [-1]).resolve(0)]]} start = time.perf_counter() - result, _ = _apply_matches( - long_prompt, - mm_prompt_updates, - tokenizer=None, - ) + result, _ = _apply_matches(long_prompt, mm_prompt_updates) elapsed = time.perf_counter() - start # Should complete in < 100ms (was taking seconds before the fix) assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s" - assert "".join(result) == long_prompt + assert flatten_2d_lists(result) == long_prompt def test_apply_matches_many_shared_targets_scales_linearly(): @@ -1317,11 +998,7 @@ def test_apply_matches_many_shared_targets_scales_linearly(): prompt = [0] * item_count start = time.perf_counter() - result, match_result = apply_token_matches( - prompt, - mm_prompt_updates, - tokenizer=None, - ) + result, match_result = apply_token_matches(prompt, mm_prompt_updates) elapsed = time.perf_counter() - start assert len(result) == item_count * len(replacement) @@ -1360,7 +1037,7 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes(): } start = time.perf_counter() - result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None) + result = find_mm_placeholders(prompt, mm_prompt_updates) elapsed = time.perf_counter() - start assert result == {} @@ -1377,22 +1054,158 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes(): [1, 2, 3, 4, 5], ], ) -def test_find_mm_placeholders_resolves_content_lazily(prompt): +def test_find_mm_placeholders_stops_at_missing_item(prompt): """ - Test that content of items the scan never reaches is not resolved. - - With `tokenizer=None`, resolving string content raises; the scan must - return no placeholders instead of raising on the second item. + Test that the scan returns no placeholders once it fails to find + an item's placeholder, leaving later items unresolved. """ result = find_mm_placeholders( prompt, { "image": [ [PromptReplacement("image", [0], [999]).resolve(0)], - [PromptReplacement("image", [0], "never reached").resolve(1)], + [PromptReplacement("image", [0], [998]).resolve(1)], ] }, - tokenizer=None, ) assert result == {} + + +class _FakeTokenizer: + """ + Character-level tokenizer where "foo" merges into one token differently + depending on whether it is followed by "d", like BPE merging "foo" in + "food" across the search-text boundary. + """ + + _MERGES = {"food": (1000,), "foo": (101, 111, 111)} + _INVERSE = {ids: text for text, ids in _MERGES.items()} + + def encode(self, text: str, add_special_tokens: bool = True) -> list[int]: + token_ids = list[int]() + pos = 0 + while pos < len(text): + for length in (4, 3): + word = text[pos : pos + length] + if word in self._MERGES: + token_ids.extend(self._MERGES[word]) + pos += length + break + else: + token_ids.append(ord(text[pos])) + pos += 1 + return token_ids + + def decode( + self, + token_ids: list[int], + skip_special_tokens: bool = False, + ) -> str: + chars = list[str]() + pos = 0 + while pos < len(token_ids): + for length in (3, 1): + key = tuple(token_ids[pos : pos + length]) + if key in self._INVERSE: + chars.append(self._INVERSE[key]) + pos += length + break + else: + chars.append(chr(token_ids[pos])) + pos += 1 + return "".join(chars) + + +class _FakeProcessingInfo: + def __init__(self, tokenizer) -> None: + self._tokenizer = tokenizer + + def get_tokenizer(self): + return self._tokenizer + + +class _TextFallbackProcessor(BaseMultiModalProcessor): + """Only `self.info.get_tokenizer()` is needed by the text fallback.""" + + def __init__(self, tokenizer: _FakeTokenizer) -> None: + self.info = _FakeProcessingInfo(tokenizer) + + def _get_mm_fields_config(self, hf_inputs, hf_processor_mm_kwargs): + raise NotImplementedError + + def _get_prompt_updates(self, mm_items, hf_processor_mm_kwargs, out_mm_kwargs): + raise NotImplementedError + + +def _text_fallback_processor() -> BaseMultiModalProcessor: + return _TextFallbackProcessor(_FakeTokenizer()) + + +def test_apply_prompt_updates_falls_back_to_text_matching(): + """ + Test that the fallback in `_apply_prompt_updates` finds targets that + tokenize differently inside the prompt ("foo" in "food"). + """ + processor = _text_fallback_processor() + + new_token_ids, placeholders = processor._apply_prompt_updates( + [1000], # "food" + { + "image": [ + [PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)] + ] + }, + ) + + assert new_token_ids == [200, 201, ord("d")] + assert [p.to_range().offset for p in placeholders["image"]] == [0] + assert [p.tokens for p in placeholders["image"]] == [[200, 201]] + + +def test_apply_prompt_updates_falls_back_with_prefix_target(): + """ + Test that `PromptIndexTargets.prefix` targets are resolved against the + decoded text in the fallback path of `_apply_prompt_updates`. + """ + processor = _text_fallback_processor() + + new_token_ids, placeholders = processor._apply_prompt_updates( + [1000], # "food" + { + "image": [ + [ + PromptInsertion( + "image", + PromptIndexTargets.prefix([101, 111, 111]), + [9], + ).resolve(0) + ] + ] + }, + ) + + assert new_token_ids == [101, 111, 111, 9, ord("d")] + assert [p.tokens for p in placeholders["image"]] == [[9]] + + +def test_apply_prompt_updates_falls_back_with_index_targets(): + """ + Test that the text resolvers of `PromptIndexTargets.start`/`end` + match against the decoded text when another item forces the + fallback in `_apply_prompt_updates`. + """ + processor = _text_fallback_processor() + + new_token_ids, placeholders = processor._apply_prompt_updates( + [1000], # "food" + { + "image": [ + [PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)], + [PromptInsertion("image", PromptIndexTargets.end(), [9]).resolve(1)], + ] + }, + ) + + assert new_token_ids == [200, 201, ord("d"), 9] + assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]] diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index 43e6212c7a..7d6e51f4d3 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -483,7 +483,7 @@ class AudioFlamingo3MultiModalProcessor( return [ PromptReplacement( modality="audio", - target=audio_token, + target=[audio_token_id], replacement=get_replacement_audioflamingo3, ) ] diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index 6c13203b27..42252afd67 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -40,6 +40,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -277,10 +278,12 @@ class Cohere2VisionMultiModalProcessor( ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) image_token = hf_processor.image_token + image_token_id = hf_processor.image_token_id img_tokens_per_tile = int(hf_processor.patch_size**2) img_line_break_token = hf_processor.img_line_break_token boi_token = hf_processor.boi_token eoi_token = hf_processor.eoi_token + tokenizer = self.info.get_tokenizer() def get_replacement(item_idx: int): images = mm_items.get_items("image", ImageProcessorItems) @@ -295,12 +298,13 @@ class Cohere2VisionMultiModalProcessor( patch_tokens = image_token * img_tokens_per_tile + img_line_break_token repl = f"{boi_token}{patch_tokens * num_patches}{eoi_token}" - return PromptUpdateDetails.select_text(repl, image_token) + repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False) + return PromptUpdateDetails.select_token_id(repl_ids, image_token_id) return [ PromptReplacement( modality="image", - target=image_token, + target=[image_token_id], replacement=get_replacement, ) ] diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index 96466bad09..060392df33 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -69,6 +69,7 @@ from vllm.multimodal.processing import ( BaseProcessingInfo, PromptReplacement, PromptUpdate, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -1197,6 +1198,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin out_mm_kwargs: MultiModalKwargsItems, ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + tokenizer = self.info.get_tokenizer() before_placeholder = { "image": "<|image@placeholder|>", @@ -1223,12 +1225,17 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin ) else: num_tokens = int(grid_thw.prod()) // merge_length - return after_placeholder[modality] * num_tokens + placeholder_ids = cached_encode( + tokenizer, after_placeholder[modality], add_special_tokens=False + ) + return placeholder_ids * num_tokens return [ PromptReplacement( modality=modality, - target=before_placeholder[modality], + target=cached_encode( + tokenizer, before_placeholder[modality], add_special_tokens=False + ), replacement=partial(get_replacement_ernie45vl, modality=modality), ) for modality in ("image", "video") diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 4b8ba82d70..607eb4ea5d 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -753,7 +753,7 @@ class FunAudioChatMultiModalProcessor( return [ PromptReplacement( modality="audio", - target=audio_token, + target=[audio_token_id], replacement=get_replacement_funaudiochat, ) ] diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index d698483cb8..0808422801 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, replace_token_matches, ) from vllm.sequence import IntermediateTensors @@ -165,7 +166,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo): image_height: int, processor: Gemma3Processor, mm_kwargs: Mapping[str, object], - ) -> PromptUpdateDetails[str]: + ) -> PromptUpdateDetails: boi_token = processor.boi_token num_crops = self.get_num_crops( @@ -189,8 +190,9 @@ class Gemma3ProcessingInfo(BaseProcessingInfo): tokenizer = processor.tokenizer vocab = tokenizer.get_vocab() image_token_id = vocab[tokenizer.image_token] + repl_full_ids = cached_encode(tokenizer, repl_full, add_special_tokens=False) - return PromptUpdateDetails.select_token_id(repl_full, image_token_id) + return PromptUpdateDetails.select_token_id(repl_full_ids, image_token_id) def get_num_image_tokens( self, @@ -313,7 +315,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]): out_mm_kwargs: MultiModalKwargsItems, ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token = hf_processor.boi_token + tokenizer = self.info.get_tokenizer() + boi_token_id = tokenizer.get_vocab()[hf_processor.boi_token] def get_replacement_gemma3(item_idx: int): images = mm_items.get_items("image", ImageProcessorItems) @@ -329,7 +332,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]): return [ PromptReplacement( modality="image", - target=image_token, + target=[boi_token_id], replacement=get_replacement_gemma3, ) ] diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index a473d9f437..4cbaed8d9b 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -51,6 +51,7 @@ from vllm.multimodal.processing.processor import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, replace_token_matches, ) from vllm.sequence import IntermediateTensors @@ -183,31 +184,37 @@ class Gemma3nProcessingInfo(BaseProcessingInfo): image_width: int, image_height: int, processor: Gemma3nProcessor, - ) -> str: + ) -> PromptUpdateDetails: """ - Get the replacement text for image tokens. + Get the replacement metadata for image tokens. For Gemma3n, this should return the full_image_sequence which includes BOI token, repeated image tokens, and EOI token. """ + full_token_ids = cached_encode( + processor.tokenizer, processor.full_image_sequence, add_special_tokens=False + ) return PromptUpdateDetails.select_token_id( - processor.full_image_sequence, processor.image_token_id + full_token_ids, processor.image_token_id ) def get_audio_repl( self, *, processor: Gemma3nProcessor, - ) -> str: + ) -> PromptUpdateDetails: """ - Get the replacement text for audio tokens. + Get the replacement metadata for audio tokens. For Gemma3n, this should return the full_audio_sequence which includes BOA token, repeated audio tokens, and EOA token. """ # Return the full audio sequence as defined by the processor + full_token_ids = cached_encode( + processor.tokenizer, processor.full_audio_sequence, add_special_tokens=False + ) return PromptUpdateDetails.select_token_id( - processor.full_audio_sequence, processor.audio_token_id + full_token_ids, processor.audio_token_id ) @@ -345,7 +352,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]) # Handle image tokens if "image" in mm_items: - image_token = hf_processor.image_token + image_token_id = hf_processor.image_token_id def get_replacement_image(item_idx: int): images = mm_items.get_items("image", ImageProcessorItems) @@ -359,14 +366,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]) prompt_updates.append( PromptReplacement( modality="image", - target=image_token, + target=[image_token_id], replacement=get_replacement_image, ) ) # Handle audio tokens if "audio" in mm_items: - audio_token = hf_processor.audio_token + audio_token_id = hf_processor.audio_token_id def get_replacement_audio(item_idx: int): return self.info.get_audio_repl( @@ -376,7 +383,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]) prompt_updates.append( PromptReplacement( modality="audio", - target=audio_token, + target=[audio_token_id], replacement=get_replacement_audio, ) ) diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 29b9a08aef..c636897aae 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -332,7 +332,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): image_height: int, processor: Gemma4Processor | None, max_soft_tokens: int | None = None, - ) -> PromptUpdateDetails[list[int]]: + ) -> PromptUpdateDetails: """Return the dynamic image token sequence for this image. Computes the exact number of soft tokens the vision tower will @@ -386,7 +386,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): *, audio_len: int, processor: Gemma4Processor | None, - ) -> PromptUpdateDetails[list[int]]: + ) -> PromptUpdateDetails: """Return the dynamic audio token sequence for this audio. Computes the number of soft tokens from the audio waveform @@ -414,7 +414,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): timestamps: list[float], num_soft_tokens_per_frame: list[int], processor: Gemma4Processor, - ) -> PromptUpdateDetails[list[int]]: + ) -> PromptUpdateDetails: """Build the full token replacement for one video. Produces the same interleaved sequence as the HF Gemma4Processor: @@ -818,7 +818,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): # one image_token exists per image in the token stream. # The replacement expands it to the full image sequence # (boi + N×image_token + eoi, where N = max_soft_tokens). - image_token = hf_processor.image_token + image_token_id = hf_processor.image_token_id def get_replacement_image(item_idx: int): images = mm_items.get_items("image", ImageProcessorItems) @@ -848,13 +848,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): prompt_updates.append( PromptReplacement( modality="image", - target=image_token, + target=[image_token_id], replacement=get_replacement_image, ) ) if "video" in mm_items: - video_token = hf_processor.video_token + video_token_id = hf_processor.video_token_id def get_replacement_video(item_idx: int): out_item = out_mm_kwargs["video"][item_idx] @@ -869,13 +869,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): prompt_updates.append( PromptReplacement( modality="video", - target=video_token, + target=[video_token_id], replacement=get_replacement_video, ) ) if "audio" in mm_items: - audio_token = hf_processor.audio_token + audio_token_id = hf_processor.audio_token_id def get_replacement_audio(item_idx: int): audios = mm_items.get_items("audio", AudioProcessorItems) @@ -888,7 +888,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]): prompt_updates.append( PromptReplacement( modality="audio", - target=audio_token, + target=[audio_token_id], replacement=get_replacement_audio, ) ) diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index a4690e9a47..8dd0beb66c 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -94,6 +94,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import get_processor_cls_name_from_config @@ -1694,6 +1695,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) + tokenizer = self.info.get_tokenizer() + vocab = tokenizer.get_vocab() merge_length = image_processor.merge_size**2 @@ -1722,12 +1725,18 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): return [ PromptReplacement( modality="image", - target=hf_processor.image_token, + target=cached_encode( + tokenizer, hf_processor.image_token, add_special_tokens=False + ), replacement=get_image_replacement, ), PromptReplacement( modality="video", - target="<|begin_of_video|><|video|><|end_of_video|>", + target=[ + vocab["<|begin_of_video|>"], + vocab["<|video|>"], + vocab["<|end_of_video|>"], + ], replacement=get_video_replacement, ), ] diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index fcf17c550c..c9d0638e0c 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -896,7 +896,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]) return [ PromptReplacement( modality="audio", - target=audio_token, + target=[audio_token_id], replacement=get_replacement_glmasr, ) ] diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index dc63276d82..02e31e719c 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -25,6 +25,7 @@ from vllm.multimodal.processing.processor import ( ProcessorInputs, PromptReplacement, TimingContext, + cached_encode, ) from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor @@ -90,6 +91,8 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn hf_processor: H2OVLProcessor, out_mm_data: BatchedTensorInputs, ): + tokenizer = self.info.get_tokenizer() + if "image_num_patches" in out_mm_data: image_num_patches = out_mm_data["image_num_patches"] assert isinstance(image_num_patches, torch.Tensor) @@ -127,7 +130,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn return PromptReplacement( modality="image", - target="", + target=cached_encode(tokenizer, "", add_special_tokens=False), replacement=get_replacement_internvl, ) diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index fbf0e7a7c3..223b881196 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -49,6 +49,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -315,7 +316,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo out_mm_kwargs: MultiModalKwargsItems, ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token, _, _ = self.info._get_image_token(hf_processor) + image_token_id = hf_processor.image_token_id + tokenizer = self.info.get_tokenizer() def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails: images = mm_items.get_items("image", ImageProcessorItems) @@ -329,15 +331,18 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo mm_kwargs=hf_processor_mm_kwargs, ) - return PromptUpdateDetails.select_text( - image_repl, - embed_text=image_token, + image_repl_ids = cached_encode( + tokenizer, image_repl, add_special_tokens=False + ) + return PromptUpdateDetails.select_token_id( + image_repl_ids, + image_token_id, ) return [ PromptReplacement( modality="image", - target=image_token, + target=[image_token_id], replacement=get_replacement_idefics3, ) ] diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 2248911b20..f8062cb751 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -45,6 +45,7 @@ from vllm.multimodal.processing import ( PromptReplacement, PromptUpdate, PromptUpdateDetails, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_video_processor_from_config @@ -355,11 +356,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = hf_processor.tokenizer - video_token_id = tokenizer.encode( - hf_processor.video_token, add_special_tokens=False - ) - assert len(video_token_id) == 1 - video_token_id = video_token_id[0] + vocab = tokenizer.get_vocab() + + video_token_id = vocab[hf_processor.video_token] prompt_text = re.sub( hf_processor.image_token, "", prompt_text @@ -469,10 +468,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) img_context_token = hf_processor.image_token + img_context_token_id = hf_processor.image_token_id start_image_token = hf_processor.start_image_token end_image_token = hf_processor.end_image_token video_token = hf_processor.video_token + tokenizer = self.info.get_tokenizer() + video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False) + assert len(video_token_id) == 1 + video_token_id = video_token_id[0] + out_mm_data = out_mm_kwargs.get_data() if "video_num_patches" in out_mm_data: video_num_patches = out_mm_data["video_num_patches"] @@ -501,7 +506,12 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo repl_features = img_context_token * feature_size repl_full = start_image_token + repl_features + end_image_token - return PromptUpdateDetails.select_text(repl_full, img_context_token) + repl_full_ids = cached_encode( + tokenizer, repl_full, add_special_tokens=False + ) + return PromptUpdateDetails.select_token_id( + repl_full_ids, img_context_token_id + ) def get_replacement_interns1_video(item_idx: int): num_patches = video_num_patches[item_idx] @@ -512,17 +522,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)] ) - return PromptUpdateDetails.select_text(repl_full, video_token) + repl_full_ids = cached_encode( + tokenizer, repl_full, add_special_tokens=False + ) + return PromptUpdateDetails.select_token_id(repl_full_ids, video_token_id) return [ PromptReplacement( modality="image", - target=img_context_token, + target=[img_context_token_id], replacement=get_replacement_interns1_image, ), PromptReplacement( modality="video", - target=video_token, + target=[video_token_id], replacement=get_replacement_interns1_video, ), ] diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index ae7b05a553..54fa28cf5a 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -43,6 +43,7 @@ from vllm.multimodal.processing import ( BaseProcessingInfo, PromptReplacement, PromptUpdate, + cached_encode, ) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.internvl import ( @@ -264,6 +265,8 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]): hf_processor: InternVLProcessor, out_mm_data: BatchedTensorInputs, ): + tokenizer = self.info.get_tokenizer() + if "image_num_patches" in out_mm_data: image_num_patches = out_mm_data["image_num_patches"] assert isinstance(image_num_patches, torch.Tensor) @@ -298,7 +301,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]): return PromptReplacement( modality="image", - target="", + target=cached_encode(tokenizer, "", add_special_tokens=False), replacement=get_replacement_internvl, ) @@ -498,6 +501,8 @@ class InternVLMultiModalProcessor( hf_processor: InternVLProcessor, out_mm_data: BatchedTensorInputs, ): + tokenizer = self.info.get_tokenizer() + if "video_num_patches" in out_mm_data: video_num_patches = out_mm_data["video_num_patches"] assert isinstance(video_num_patches, torch.Tensor) @@ -514,7 +519,7 @@ class InternVLMultiModalProcessor( return PromptReplacement( modality="video", - target="