[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
Cyrus Leung
2026-08-23 14:50:32 +08:00
committed by GitHub
co-authored by Qwen Code
parent a3561ef8e4
commit ebcd606467
53 changed files with 831 additions and 855 deletions
+6
View File
@@ -532,6 +532,12 @@ return a list of [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instanc
Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies an update operation
(e.g.: insertion, replacement) performed by the HF processor.
!!! note
The target and content of each update are token sequences. When converting
text to token sequences, remember to encode it with
`cached_encode(..., add_special_tokens=False)`. Otherwise, the updated prompt
may contain duplicated special tokens or fail to match the target.
=== "Basic example: LLaVA"
Looking at HF's `LlavaProcessor`:
@@ -192,11 +192,11 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
assert resolved.modality == "image"
assert [
(match.start_idx, match.end_idx)
for match in resolved.iter_matches([1, 10, 11], _FakeTokenizer())
for match in resolved.iter_token_matches([1, 10, 11])
] == [(1, 1)]
assert content.full == [32000] * 256
is_embed = content.is_embed(None, content.full)
is_embed = content.is_embed(content.full)
assert is_embed.dtype == torch.bool
assert is_embed.tolist() == [True] * 256
+2 -2
View File
@@ -93,7 +93,7 @@ def test_cache_item_size(item, expected_size):
cache[""] = item
assert cache.currsize == expected_size
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0)
cache[""] = MultiModalProcessorCacheItem(item, [prompt_update])
assert cache.currsize == expected_size
@@ -151,7 +151,7 @@ def _compare_caches(
for item in all_items
]
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0)
for it in range(n_iter):
num_items_to_select = rng.randint(0, max_items_per_iter)
+154 -341
View File
@@ -12,18 +12,19 @@ from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
PlaceholderFeaturesInfo,
PromptIndexTargets,
PromptInsertion,
PromptReplacement,
_apply_matches,
_apply_token_matches_with_placeholders,
apply_text_matches,
apply_token_matches,
find_mm_placeholders,
iter_token_matches,
replace_token_matches,
)
from vllm.utils.collection_utils import flatten_2d_lists
from .utils import random_image
@@ -242,7 +243,7 @@ def test_find_token_matches(
for key, target in target_by_key.items()
}
result = {
key: list(update.iter_token_matches(prompt, tokenizer=None))
key: list(update.iter_token_matches(prompt))
for key, update in prompt_updates.items()
}
@@ -259,315 +260,6 @@ def test_find_token_matches(
} == expected_by_key
@pytest.mark.parametrize(
("prompt", "target_by_key", "expected_by_key"),
[
# Detokenized test cases of `test_find_token_matches`
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
(
"",
{
"pattern_1": "",
"pattern_2": "<image>",
"pattern_3": PromptIndexTargets.start(),
"pattern_4": PromptIndexTargets.prefix("<image>"),
"pattern_5": PromptIndexTargets.end(),
},
{
"pattern_1": [{"start_idx": 0, "end_idx": 0}],
"pattern_2": [],
"pattern_3": [
{"start_idx": 0, "end_idx": 0},
],
"pattern_4": [],
"pattern_5": [
{"start_idx": 0, "end_idx": 0},
],
},
),
(
"<image><image><image><image>",
{
"pattern_1": "<image>",
"pattern_2": "<image><image>",
"pattern_3": "<image><image><image>",
"pattern_4": PromptIndexTargets.start(),
"pattern_5": PromptIndexTargets.prefix("<image>"),
"pattern_6": PromptIndexTargets.end(),
},
{
"pattern_1": [
{"start_idx": 0, "end_idx": 7},
{"start_idx": 7, "end_idx": 14},
{"start_idx": 14, "end_idx": 21},
{"start_idx": 21, "end_idx": 28},
],
"pattern_2": [
{"start_idx": 0, "end_idx": 14},
{"start_idx": 14, "end_idx": 28},
],
"pattern_3": [
{"start_idx": 0, "end_idx": 21},
],
"pattern_4": [
{"start_idx": 0, "end_idx": 0},
],
"pattern_5": [
{"start_idx": 7, "end_idx": 7},
],
"pattern_6": [
{"start_idx": 28, "end_idx": 28},
],
},
),
(
"Image:<image><image><image>Image:<image><image>!",
{
"pattern_1": "Image:<image>",
"pattern_2": "Image:<image><image><image>",
"pattern_3": "Image:<unk><image>",
"pattern_4": PromptIndexTargets.start(),
"pattern_5": PromptIndexTargets.prefix("Image:<image>"),
"pattern_6": PromptIndexTargets.end(),
},
{
"pattern_1": [
{"start_idx": 0, "end_idx": 13},
{"start_idx": 27, "end_idx": 40},
],
"pattern_2": [
{"start_idx": 0, "end_idx": 27},
],
"pattern_3": [],
"pattern_4": [
{"start_idx": 0, "end_idx": 0},
],
"pattern_5": [
{"start_idx": 13, "end_idx": 13},
],
"pattern_6": [
{"start_idx": 48, "end_idx": 48},
],
},
),
# Test regex escape
(
"<|image|><image><|image|><image>",
{
"pattern_1": "<|image|>",
"pattern_2": "<|image|><image>",
"pattern_3": "<|image|><image><|image|>",
},
{
"pattern_1": [
{"start_idx": 0, "end_idx": 9},
{"start_idx": 16, "end_idx": 25},
],
"pattern_2": [
{"start_idx": 0, "end_idx": 16},
{"start_idx": 16, "end_idx": 32},
],
"pattern_3": [
{"start_idx": 0, "end_idx": 25},
],
},
),
],
)
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
def test_find_text_matches(
prompt,
target_by_key,
expected_by_key,
update_type,
):
prompt_updates = {
key: update_type(key, target, []).resolve(0)
for key, target in target_by_key.items()
}
result = {
key: list(update.iter_text_matches(prompt, tokenizer=None))
for key, update in prompt_updates.items()
}
# Only displayed on error
print("result:", result)
# Manually constructed results
assert {
key: [
dict(start_idx=item.start_idx, end_idx=item.end_idx)
for item in result.get(key, [])
]
for key in expected_by_key
} == expected_by_key
@pytest.mark.parametrize(
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
[
(
"Image:<image>Image:<image><image>!",
{
# We use `<image>` before `Image:` to test matches that
# occur out of order
"pattern_1": "<image>",
"pattern_2": "Image:",
"pattern_3": "!",
},
{
# Test whether target is confused with replacement
"pattern_1": "<image><image>",
# Test empty replacement
"pattern_2": "",
# Test dynamic replacement (beyond the form of `unit * count`)
"pattern_3": "?!?",
},
{
PromptInsertion: {
0: "Image:<image>Image:<image><image>!",
1: "Image:<image><image><image>Image:<image><image>!?!?",
2: "Image:<image><image><image><image><image>Image:<image><image>!?!??!?", # noqa: E501
},
PromptReplacement: {
0: "Image:<image>Image:<image><image>!",
1: "<image><image>Image:<image><image>?!?",
2: "<image><image><image><image><image>?!?",
},
},
),
# Test index targets
(
"",
{
"pattern_1": PromptIndexTargets.start(),
"pattern_2": PromptIndexTargets.prefix("<image>"),
"pattern_3": PromptIndexTargets.end(),
},
{
"pattern_1": "1",
"pattern_2": "2",
"pattern_3": "3",
},
{
PromptInsertion: {
0: "",
1: "13",
2: "1133",
},
PromptReplacement: {
0: "",
1: "13",
2: "1133",
},
},
),
(
"<image>",
{
"pattern_1": PromptIndexTargets.start(),
"pattern_2": PromptIndexTargets.prefix("<image>"),
"pattern_3": PromptIndexTargets.end(),
},
{
"pattern_1": "1",
"pattern_2": "2",
"pattern_3": "3",
},
{
PromptInsertion: {
0: "<image>",
1: "1<image>23",
2: "11<image>2233",
},
PromptReplacement: {
0: "<image>",
1: "1<image>23",
2: "11<image>2233",
},
},
),
# Test different replacement per item
(
"<image><image><image>",
{
"pattern_1": "<image>",
},
{
"pattern_1": lambda idx: str(idx + 1),
},
{
PromptInsertion: {
0: "<image><image><image>",
1: "<image>1<image><image>",
2: "<image>12<image><image>",
},
PromptReplacement: {
0: "<image><image><image>",
1: "1<image><image>",
2: "12<image>",
},
},
),
(
"<image><image><image>",
{
"pattern_1": PromptIndexTargets.prefix("<image>"),
},
{
"pattern_1": lambda idx: str(idx + 1),
},
{
PromptInsertion: {
0: "<image><image><image>",
1: "<image>1<image><image>",
2: "<image>12<image><image>",
},
PromptReplacement: {
0: "<image><image><image>",
1: "<image>1<image><image>",
2: "<image>12<image><image>",
},
},
),
],
)
def test_find_update_text(
prompt,
target_by_key,
repl_by_key,
expected_by_update_type_mm_count,
):
for (
update_type,
expected_by_mm_count,
) in expected_by_update_type_mm_count.items():
for mm_count, expected in expected_by_mm_count.items():
mm_prompt_updates = {
key: [
[update_type(key, target, repl_by_key[key]).resolve(i)]
for i in range(mm_count)
]
for key, target in target_by_key.items()
}
new_prompt, result = apply_text_matches(
prompt,
mm_prompt_updates,
tokenizer=None,
)
# Only displayed on error
print("update_type:", update_type)
print("mm_count:", mm_count)
print("mm_prompt_updates:", mm_prompt_updates)
print("new_prompt:", new_prompt)
print("result:", result)
# Manually constructed results
assert new_prompt == expected
FIND_UPDATE_TOKENS_TEST_CASES = [
# Tokenized test cases of `test_find_update_text`
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
@@ -891,11 +583,7 @@ def test_find_update_tokens(
for key, target in target_by_key.items()
}
new_prompt, result = apply_token_matches(
prompt,
mm_prompt_updates,
tokenizer=None,
)
new_prompt, result = apply_token_matches(prompt, mm_prompt_updates)
# Only displayed on error
print("update_type:", update_type)
@@ -945,7 +633,6 @@ def test_apply_token_matches_with_placeholders(
new_prompt, result, placeholders = _apply_token_matches_with_placeholders(
prompt,
mm_prompt_updates,
tokenizer=None,
)
if any(
@@ -1097,7 +784,7 @@ def test_find_mm_placeholders(
for key, repl in repl_by_key.items()
}
result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
result = find_mm_placeholders(prompt, mm_prompt_updates)
# Only displayed on error
print("result:", result)
@@ -1285,24 +972,18 @@ def test_apply_matches_no_match_exits_quickly():
With the fix, it should exit immediately when no match is found.
"""
# Create a long prompt with no placeholder
long_prompt = "x" * 10000
long_prompt = [1] * 10000
# Create update looking for a placeholder that doesn't exist
mm_prompt_updates = {
"image": [[PromptReplacement("image", "<image>", "REPLACED").resolve(0)]]
}
mm_prompt_updates = {"image": [[PromptReplacement("image", [0], [-1]).resolve(0)]]}
start = time.perf_counter()
result, _ = _apply_matches(
long_prompt,
mm_prompt_updates,
tokenizer=None,
)
result, _ = _apply_matches(long_prompt, mm_prompt_updates)
elapsed = time.perf_counter() - start
# Should complete in < 100ms (was taking seconds before the fix)
assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s"
assert "".join(result) == long_prompt
assert flatten_2d_lists(result) == long_prompt
def test_apply_matches_many_shared_targets_scales_linearly():
@@ -1317,11 +998,7 @@ def test_apply_matches_many_shared_targets_scales_linearly():
prompt = [0] * item_count
start = time.perf_counter()
result, match_result = apply_token_matches(
prompt,
mm_prompt_updates,
tokenizer=None,
)
result, match_result = apply_token_matches(prompt, mm_prompt_updates)
elapsed = time.perf_counter() - start
assert len(result) == item_count * len(replacement)
@@ -1360,7 +1037,7 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
}
start = time.perf_counter()
result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
result = find_mm_placeholders(prompt, mm_prompt_updates)
elapsed = time.perf_counter() - start
assert result == {}
@@ -1377,22 +1054,158 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
[1, 2, 3, 4, 5],
],
)
def test_find_mm_placeholders_resolves_content_lazily(prompt):
def test_find_mm_placeholders_stops_at_missing_item(prompt):
"""
Test that content of items the scan never reaches is not resolved.
With `tokenizer=None`, resolving string content raises; the scan must
return no placeholders instead of raising on the second item.
Test that the scan returns no placeholders once it fails to find
an item's placeholder, leaving later items unresolved.
"""
result = find_mm_placeholders(
prompt,
{
"image": [
[PromptReplacement("image", [0], [999]).resolve(0)],
[PromptReplacement("image", [0], "never reached").resolve(1)],
[PromptReplacement("image", [0], [998]).resolve(1)],
]
},
tokenizer=None,
)
assert result == {}
class _FakeTokenizer:
"""
Character-level tokenizer where "foo" merges into one token differently
depending on whether it is followed by "d", like BPE merging "foo" in
"food" across the search-text boundary.
"""
_MERGES = {"food": (1000,), "foo": (101, 111, 111)}
_INVERSE = {ids: text for text, ids in _MERGES.items()}
def encode(self, text: str, add_special_tokens: bool = True) -> list[int]:
token_ids = list[int]()
pos = 0
while pos < len(text):
for length in (4, 3):
word = text[pos : pos + length]
if word in self._MERGES:
token_ids.extend(self._MERGES[word])
pos += length
break
else:
token_ids.append(ord(text[pos]))
pos += 1
return token_ids
def decode(
self,
token_ids: list[int],
skip_special_tokens: bool = False,
) -> str:
chars = list[str]()
pos = 0
while pos < len(token_ids):
for length in (3, 1):
key = tuple(token_ids[pos : pos + length])
if key in self._INVERSE:
chars.append(self._INVERSE[key])
pos += length
break
else:
chars.append(chr(token_ids[pos]))
pos += 1
return "".join(chars)
class _FakeProcessingInfo:
def __init__(self, tokenizer) -> None:
self._tokenizer = tokenizer
def get_tokenizer(self):
return self._tokenizer
class _TextFallbackProcessor(BaseMultiModalProcessor):
"""Only `self.info.get_tokenizer()` is needed by the text fallback."""
def __init__(self, tokenizer: _FakeTokenizer) -> None:
self.info = _FakeProcessingInfo(tokenizer)
def _get_mm_fields_config(self, hf_inputs, hf_processor_mm_kwargs):
raise NotImplementedError
def _get_prompt_updates(self, mm_items, hf_processor_mm_kwargs, out_mm_kwargs):
raise NotImplementedError
def _text_fallback_processor() -> BaseMultiModalProcessor:
return _TextFallbackProcessor(_FakeTokenizer())
def test_apply_prompt_updates_falls_back_to_text_matching():
"""
Test that the fallback in `_apply_prompt_updates` finds targets that
tokenize differently inside the prompt ("foo" in "food").
"""
processor = _text_fallback_processor()
new_token_ids, placeholders = processor._apply_prompt_updates(
[1000], # "food"
{
"image": [
[PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)]
]
},
)
assert new_token_ids == [200, 201, ord("d")]
assert [p.to_range().offset for p in placeholders["image"]] == [0]
assert [p.tokens for p in placeholders["image"]] == [[200, 201]]
def test_apply_prompt_updates_falls_back_with_prefix_target():
"""
Test that `PromptIndexTargets.prefix` targets are resolved against the
decoded text in the fallback path of `_apply_prompt_updates`.
"""
processor = _text_fallback_processor()
new_token_ids, placeholders = processor._apply_prompt_updates(
[1000], # "food"
{
"image": [
[
PromptInsertion(
"image",
PromptIndexTargets.prefix([101, 111, 111]),
[9],
).resolve(0)
]
]
},
)
assert new_token_ids == [101, 111, 111, 9, ord("d")]
assert [p.tokens for p in placeholders["image"]] == [[9]]
def test_apply_prompt_updates_falls_back_with_index_targets():
"""
Test that the text resolvers of `PromptIndexTargets.start`/`end`
match against the decoded text when another item forces the
fallback in `_apply_prompt_updates`.
"""
processor = _text_fallback_processor()
new_token_ids, placeholders = processor._apply_prompt_updates(
[1000], # "food"
{
"image": [
[PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)],
[PromptInsertion("image", PromptIndexTargets.end(), [9]).resolve(1)],
]
},
)
assert new_token_ids == [200, 201, ord("d"), 9]
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
+1 -1
View File
@@ -483,7 +483,7 @@ class AudioFlamingo3MultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_audioflamingo3,
)
]
+6 -2
View File
@@ -40,6 +40,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -277,10 +278,12 @@ class Cohere2VisionMultiModalProcessor(
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
img_tokens_per_tile = int(hf_processor.patch_size**2)
img_line_break_token = hf_processor.img_line_break_token
boi_token = hf_processor.boi_token
eoi_token = hf_processor.eoi_token
tokenizer = self.info.get_tokenizer()
def get_replacement(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -295,12 +298,13 @@ class Cohere2VisionMultiModalProcessor(
patch_tokens = image_token * img_tokens_per_tile + img_line_break_token
repl = f"{boi_token}{patch_tokens * num_patches}{eoi_token}"
return PromptUpdateDetails.select_text(repl, image_token)
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(repl_ids, image_token_id)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement,
)
]
+9 -2
View File
@@ -69,6 +69,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -1197,6 +1198,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
before_placeholder = {
"image": "<|image@placeholder|>",
@@ -1223,12 +1225,17 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
)
else:
num_tokens = int(grid_thw.prod()) // merge_length
return after_placeholder[modality] * num_tokens
placeholder_ids = cached_encode(
tokenizer, after_placeholder[modality], add_special_tokens=False
)
return placeholder_ids * num_tokens
return [
PromptReplacement(
modality=modality,
target=before_placeholder[modality],
target=cached_encode(
tokenizer, before_placeholder[modality], add_special_tokens=False
),
replacement=partial(get_replacement_ernie45vl, modality=modality),
)
for modality in ("image", "video")
+1 -1
View File
@@ -753,7 +753,7 @@ class FunAudioChatMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_funaudiochat,
)
]
+7 -4
View File
@@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
replace_token_matches,
)
from vllm.sequence import IntermediateTensors
@@ -165,7 +166,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
image_height: int,
processor: Gemma3Processor,
mm_kwargs: Mapping[str, object],
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
boi_token = processor.boi_token
num_crops = self.get_num_crops(
@@ -189,8 +190,9 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
tokenizer = processor.tokenizer
vocab = tokenizer.get_vocab()
image_token_id = vocab[tokenizer.image_token]
repl_full_ids = cached_encode(tokenizer, repl_full, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(repl_full, image_token_id)
return PromptUpdateDetails.select_token_id(repl_full_ids, image_token_id)
def get_num_image_tokens(
self,
@@ -313,7 +315,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.boi_token
tokenizer = self.info.get_tokenizer()
boi_token_id = tokenizer.get_vocab()[hf_processor.boi_token]
def get_replacement_gemma3(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -329,7 +332,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=image_token,
target=[boi_token_id],
replacement=get_replacement_gemma3,
)
]
+17 -10
View File
@@ -51,6 +51,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
replace_token_matches,
)
from vllm.sequence import IntermediateTensors
@@ -183,31 +184,37 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
image_width: int,
image_height: int,
processor: Gemma3nProcessor,
) -> str:
) -> PromptUpdateDetails:
"""
Get the replacement text for image tokens.
Get the replacement metadata for image tokens.
For Gemma3n, this should return the full_image_sequence which includes
BOI token, repeated image tokens, and EOI token.
"""
full_token_ids = cached_encode(
processor.tokenizer, processor.full_image_sequence, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
processor.full_image_sequence, processor.image_token_id
full_token_ids, processor.image_token_id
)
def get_audio_repl(
self,
*,
processor: Gemma3nProcessor,
) -> str:
) -> PromptUpdateDetails:
"""
Get the replacement text for audio tokens.
Get the replacement metadata for audio tokens.
For Gemma3n, this should return the full_audio_sequence which includes
BOA token, repeated audio tokens, and EOA token.
"""
# Return the full audio sequence as defined by the processor
full_token_ids = cached_encode(
processor.tokenizer, processor.full_audio_sequence, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
processor.full_audio_sequence, processor.audio_token_id
full_token_ids, processor.audio_token_id
)
@@ -345,7 +352,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
# Handle image tokens
if "image" in mm_items:
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
def get_replacement_image(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -359,14 +366,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt_updates.append(
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement_image,
)
)
# Handle audio tokens
if "audio" in mm_items:
audio_token = hf_processor.audio_token
audio_token_id = hf_processor.audio_token_id
def get_replacement_audio(item_idx: int):
return self.info.get_audio_repl(
@@ -376,7 +383,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt_updates.append(
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_audio,
)
)
+9 -9
View File
@@ -332,7 +332,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
image_height: int,
processor: Gemma4Processor | None,
max_soft_tokens: int | None = None,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Return the dynamic image token sequence for this image.
Computes the exact number of soft tokens the vision tower will
@@ -386,7 +386,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
*,
audio_len: int,
processor: Gemma4Processor | None,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Return the dynamic audio token sequence for this audio.
Computes the number of soft tokens from the audio waveform
@@ -414,7 +414,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
timestamps: list[float],
num_soft_tokens_per_frame: list[int],
processor: Gemma4Processor,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Build the full token replacement for one video.
Produces the same interleaved sequence as the HF Gemma4Processor:
@@ -818,7 +818,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# one image_token exists per image in the token stream.
# The replacement expands it to the full image sequence
# (boi + N×image_token + eoi, where N = max_soft_tokens).
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
def get_replacement_image(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -848,13 +848,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt_updates.append(
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement_image,
)
)
if "video" in mm_items:
video_token = hf_processor.video_token
video_token_id = hf_processor.video_token_id
def get_replacement_video(item_idx: int):
out_item = out_mm_kwargs["video"][item_idx]
@@ -869,13 +869,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt_updates.append(
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=get_replacement_video,
)
)
if "audio" in mm_items:
audio_token = hf_processor.audio_token
audio_token_id = hf_processor.audio_token_id
def get_replacement_audio(item_idx: int):
audios = mm_items.get_items("audio", AudioProcessorItems)
@@ -888,7 +888,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt_updates.append(
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_audio,
)
)
+11 -2
View File
@@ -94,6 +94,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import get_processor_cls_name_from_config
@@ -1694,6 +1695,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
merge_length = image_processor.merge_size**2
@@ -1722,12 +1725,18 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=hf_processor.image_token,
target=cached_encode(
tokenizer, hf_processor.image_token, add_special_tokens=False
),
replacement=get_image_replacement,
),
PromptReplacement(
modality="video",
target="<|begin_of_video|><|video|><|end_of_video|>",
target=[
vocab["<|begin_of_video|>"],
vocab["<|video|>"],
vocab["<|end_of_video|>"],
],
replacement=get_video_replacement,
),
]
+1 -1
View File
@@ -896,7 +896,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_glmasr,
)
]
+4 -1
View File
@@ -25,6 +25,7 @@ from vllm.multimodal.processing.processor import (
ProcessorInputs,
PromptReplacement,
TimingContext,
cached_encode,
)
from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
@@ -90,6 +91,8 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
hf_processor: H2OVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -127,7 +130,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
return PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement_internvl,
)
+10 -5
View File
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -315,7 +316,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token, _, _ = self.info._get_image_token(hf_processor)
image_token_id = hf_processor.image_token_id
tokenizer = self.info.get_tokenizer()
def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:
images = mm_items.get_items("image", ImageProcessorItems)
@@ -329,15 +331,18 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
mm_kwargs=hf_processor_mm_kwargs,
)
return PromptUpdateDetails.select_text(
image_repl,
embed_text=image_token,
image_repl_ids = cached_encode(
tokenizer, image_repl, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
image_repl_ids,
image_token_id,
)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement_idefics3,
)
]
+22 -9
View File
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_video_processor_from_config
@@ -355,11 +356,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = hf_processor.tokenizer
video_token_id = tokenizer.encode(
hf_processor.video_token, add_special_tokens=False
)
assert len(video_token_id) == 1
video_token_id = video_token_id[0]
vocab = tokenizer.get_vocab()
video_token_id = vocab[hf_processor.video_token]
prompt_text = re.sub(
hf_processor.image_token, "<image_placeholder>", prompt_text
@@ -469,10 +468,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
img_context_token = hf_processor.image_token
img_context_token_id = hf_processor.image_token_id
start_image_token = hf_processor.start_image_token
end_image_token = hf_processor.end_image_token
video_token = hf_processor.video_token
tokenizer = self.info.get_tokenizer()
video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False)
assert len(video_token_id) == 1
video_token_id = video_token_id[0]
out_mm_data = out_mm_kwargs.get_data()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
@@ -501,7 +506,12 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
repl_features = img_context_token * feature_size
repl_full = start_image_token + repl_features + end_image_token
return PromptUpdateDetails.select_text(repl_full, img_context_token)
repl_full_ids = cached_encode(
tokenizer, repl_full, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
repl_full_ids, img_context_token_id
)
def get_replacement_interns1_video(item_idx: int):
num_patches = video_num_patches[item_idx]
@@ -512,17 +522,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
)
return PromptUpdateDetails.select_text(repl_full, video_token)
repl_full_ids = cached_encode(
tokenizer, repl_full, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(repl_full_ids, video_token_id)
return [
PromptReplacement(
modality="image",
target=img_context_token,
target=[img_context_token_id],
replacement=get_replacement_interns1_image,
),
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=get_replacement_interns1_video,
),
]
+7 -2
View File
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processors.internvl import (
@@ -264,6 +265,8 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
hf_processor: InternVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -298,7 +301,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
return PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement_internvl,
)
@@ -498,6 +501,8 @@ class InternVLMultiModalProcessor(
hf_processor: InternVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor)
@@ -514,7 +519,7 @@ class InternVLMultiModalProcessor(
return PromptReplacement(
modality="video",
target="<video>",
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
replacement=get_video_replacement_internvl,
)
+8 -3
View File
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
@@ -450,6 +451,10 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
image_pad_token_ids = cached_encode(
tokenizer, "<|image_pad|>", add_special_tokens=False
)
pixel_shuffle_scale = getattr(image_processor, "pixel_shuffle_scale", 2)
merge_length = pixel_shuffle_scale**2
@@ -460,13 +465,13 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
assert isinstance(grid_thw, torch.Tensor)
feature_size = int(grid_thw.prod()) // merge_length
repl_full = "<|image_pad|>" * feature_size
return PromptUpdateDetails.select_text(repl_full, "<|image_pad|>")
repl_full = image_pad_token_ids * feature_size
return PromptUpdateDetails.select_token_ids(repl_full, image_pad_token_ids)
return [
PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement_isaac,
)
]
+4 -1
View File
@@ -33,6 +33,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -541,6 +542,8 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
def get_replacement(idx: int) -> Sequence[int]:
out_item = out_mm_kwargs["image"][idx]
image_token_thw = out_item["image_token_thw"].data
@@ -552,7 +555,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
return [
PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement,
),
]
+10 -5
View File
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdateDetails,
cached_encode,
)
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
@@ -448,7 +449,8 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptReplacement]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
tokenizer = self.info.get_tokenizer()
def get_image_replacement_lfm2vl(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -463,15 +465,18 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
processor=hf_processor,
mm_kwargs=hf_processor_mm_kwargs,
)
return PromptUpdateDetails.select_text(
image_repl,
embed_text=image_token,
image_repl_ids = cached_encode(
tokenizer, image_repl, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
image_repl_ids,
image_token_id,
)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_image_replacement_lfm2vl,
)
]
+1 -1
View File
@@ -662,7 +662,7 @@ class MiDashengLMMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_midashenglm,
)
]
+1 -1
View File
@@ -1112,7 +1112,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
embed_t = torch.tensor(is_embed_mask)
return PromptUpdateDetails(
full=full,
is_embed=lambda _tok, _seq: embed_t,
is_embed=lambda _seq: embed_t,
)
def get_audio_replacement(item_idx: int) -> PromptUpdateDetails:
+15 -4
View File
@@ -58,6 +58,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -502,7 +503,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
out_mm_kwargs=out_mm_kwargs,
)
audio_placeholder = self.info.audio_pattern
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
audio_placeholder = cached_encode(
tokenizer, self.info.audio_pattern, add_special_tokens=False
)
unk_token_ids = [vocab["<unk>"]]
def get_audio_replacement(item_idx: int):
audios = mm_items.get_items(
@@ -517,9 +524,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
else:
audio_len = audios.get_audio_length(item_idx)
return PromptUpdateDetails.select_text(
self.get_audio_prompt_texts(audio_len),
"<unk>",
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_audio_prompt_texts(audio_len),
add_special_tokens=False,
),
unk_token_ids,
)
return [
+40 -29
View File
@@ -88,7 +88,7 @@ from vllm.multimodal.processing.processor import (
PromptUpdate,
PromptUpdateDetails,
ResolvedPromptUpdate,
_seq2text,
cached_encode,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -938,30 +938,24 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Apply multi-modal prompt updates to token IDs."""
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
# If the target does not consist of special tokens, it may be
# tokenized differently inside the prompt, so fall back to
# performing the updates on the decoded text, then encoding the
# result back. The segments are encoded separately to avoid BPE
# merges across segment boundaries.
if not all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
_seq2text(tokenizer, token_ids, use_cache=False),
new_token_ids, match_result = self._apply_prompt_updates_via_text(
token_ids,
mm_prompt_updates,
encode_segments_separately=True,
)
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
@@ -1062,6 +1056,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
additional_placeholders.append((modality, sub_pattern))
placeholders += additional_placeholders
vocab = tokenizer.get_vocab()
unk_token_ids = [vocab["<unk>"]]
def get_image_replacement(item_idx: int):
images = mm_items.get_items(
"image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
@@ -1069,9 +1066,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
image_size = images.get_image_size(item_idx)
return PromptUpdateDetails.select_text(
self.get_image_prompt_texts(image_size, item_idx),
"<unk>",
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_image_prompt_texts(image_size, item_idx),
add_special_tokens=False,
),
unk_token_ids,
)
def get_video_replacement(item_idx: int):
@@ -1082,9 +1083,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
frame_size = videos.get_frame_size(item_idx)
num_frames = videos.get_num_frames(item_idx)
return PromptUpdateDetails.select_text(
self.get_video_prompt_texts(frame_size, num_frames),
"<unk>",
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_video_prompt_texts(frame_size, num_frames),
add_special_tokens=False,
),
unk_token_ids,
)
get_replacement = {
@@ -1094,7 +1099,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
return [
PromptReplacement(
modality=modality, target=pattern, replacement=get_replacement[modality]
modality=modality,
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
replacement=get_replacement[modality],
)
for modality, pattern in placeholders
]
@@ -1114,7 +1121,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
image_processor = self.info.get_image_processor()
version = self.info.get_model_version()
text = _seq2text(tokenizer, cached_update.content.full)
text = tokenizer.decode(cached_update.content.full)
prev_item_idx = cached_update.item_idx
if version == (2, 0) or version == (2, 5):
@@ -1130,13 +1137,17 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
embed_text = getattr(tokenizer, "image_token", "<unk>")
new_update = new_update.with_content(
PromptUpdateDetails.select_text(
text.replace(
f"{im_start}{prev_item_idx}{im_end}",
f"{im_start}{new_item_idx}{im_end}",
1,
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(
f"{im_start}{prev_item_idx}{im_end}",
f"{im_start}{new_item_idx}{im_end}",
1,
),
add_special_tokens=False,
),
embed_text,
cached_encode(tokenizer, embed_text, add_special_tokens=False),
)
)
+48 -29
View File
@@ -36,7 +36,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdateDetails,
ResolvedPromptUpdate,
_seq2text,
cached_encode,
)
from vllm.sequence import IntermediateTensors
@@ -338,6 +338,9 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
# than `<unk>`, so use those tokens as the embedding selector.
image_embed_text = getattr(tokenizer, "image_token", "<|image_pad|>")
video_embed_text = getattr(tokenizer, "video_token", "<|video_pad|>")
vocab = tokenizer.get_vocab()
image_embed_ids = [vocab[image_embed_text]]
video_embed_ids = [vocab[video_embed_text]]
def get_image_replacement(item_idx: int):
images = mm_items.get_items(
@@ -345,13 +348,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
(MiniCPMVImageEmbeddingItems, ImageProcessorItems),
)
image_size = images.get_image_size(item_idx)
return PromptUpdateDetails.select_text(
self.get_image_prompt_texts(
image_size,
item_idx,
downsample_mode=ds_mode,
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_image_prompt_texts(
image_size,
item_idx,
downsample_mode=ds_mode,
),
add_special_tokens=False,
),
image_embed_text,
image_embed_ids,
)
def get_video_replacement(item_idx: int):
@@ -370,14 +377,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
width=int(image_sizes[0, 0].item()),
height=int(image_sizes[0, 1].item()),
)
return PromptUpdateDetails.select_text(
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
),
video_embed_text,
video_embed_ids,
)
videos = mm_items.get_items(
@@ -386,14 +397,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
)
frame_size = videos.get_frame_size(item_idx)
num_frames = videos.get_num_frames(item_idx)
return PromptUpdateDetails.select_text(
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
),
video_embed_text,
video_embed_ids,
)
get_replacement = {
@@ -404,7 +419,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
return [
PromptReplacement(
modality=modality,
target=pattern,
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
replacement=get_replacement[modality],
)
for modality, pattern in placeholders
@@ -424,17 +439,21 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
text = _seq2text(tokenizer, cached_update.content.full)
text = tokenizer.decode(cached_update.content.full)
prev_item_idx = cached_update.item_idx
new_update = new_update.with_content(
PromptUpdateDetails.select_text(
text.replace(
f"{id_start}{prev_item_idx}{id_end}",
f"{id_start}{new_item_idx}{id_end}",
1,
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(
f"{id_start}{prev_item_idx}{id_end}",
f"{id_start}{new_item_idx}{id_end}",
1,
),
add_special_tokens=False,
),
video_token,
cached_encode(tokenizer, video_token, add_special_tokens=False),
)
)
return new_update
+9 -3
View File
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -666,9 +667,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
num_patches_per_chunk = self.info.get_patch_per_chunk(vision_config)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.image_token
img_patch_token = hf_processor.img_patch_token
tokenizer = self.info.get_tokenizer()
img_patch_token_ids = cached_encode(
tokenizer, img_patch_token, add_special_tokens=False
)
def get_replacement(item_idx: int):
out_item = out_mm_kwargs["image"][item_idx]
aspect_ratio = out_item["aspect_ratios"].data
@@ -678,12 +683,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
num_patches_per_chunk=num_patches_per_chunk,
)
return PromptUpdateDetails.select_text(repl, img_patch_token)
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
return PromptUpdateDetails.select_token_ids(repl_ids, img_patch_token_ids)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[hf_processor.image_token_id],
replacement=get_replacement,
)
]
+4 -1
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
PromptInsertion,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -1263,7 +1264,9 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
return [
PromptInsertion(
modality="image",
target=PromptIndexTargets.prefix("<|endoftext|>"),
target=PromptIndexTargets.prefix(
cached_encode(tokenizer, "<|endoftext|>", add_special_tokens=False),
),
insertion=get_insertion_molmo,
)
]
+2 -2
View File
@@ -1393,7 +1393,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
def get_replacement(
item_idx: int,
suffix_token_ids: list[int] | None = None,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
num_tokens = audio_token_lens[item_idx]
if num_tokens == 0:
raise ValueError("The audio is too short to be represented.")
@@ -1410,7 +1410,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
processor.audio_end_id,
*suffix_token_ids,
],
is_embed=lambda _tokenizer, _seq: torch.cat(
is_embed=lambda _seq: torch.cat(
[
torch.tensor([False]),
is_embed,
@@ -64,6 +64,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.multimodal.processing.processor import ProcessorInputs
from vllm.sequence import IntermediateTensors
@@ -535,7 +536,7 @@ class MossTranscribeDiarizeMultiModalProcessor(
raise ValueError("Audio input is too short to produce any tokens.")
return num_tokens
def get_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def get_replacement(item_idx: int) -> PromptUpdateDetails:
num_tokens = get_num_tokens(item_idx)
audio_tokens = processor._audio_span_ids(num_tokens)
return PromptUpdateDetails.select_token_id(
@@ -546,7 +547,9 @@ class MossTranscribeDiarizeMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=AUDIO_PLACEHOLDER,
target=cached_encode(
tokenizer, AUDIO_PLACEHOLDER, add_special_tokens=False
),
replacement=get_replacement,
),
]
+2 -2
View File
@@ -407,7 +407,7 @@ class MuseGlimmerMultiModalProcessor(
video_end_id = vocab["<|vid_end|>"]
video_separator_id = vocab["<|vid_frame_separator|>"]
def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def image_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["image"][item_idx]
num_tokens = int(out_item["image_feature_sizes"].data)
replacement = (
@@ -417,7 +417,7 @@ class MuseGlimmerMultiModalProcessor(
replacement, config.image_token_id
)
def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def video_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["video"][item_idx]
pixel_values = out_item["video_pixel_values"].data
num_groups = len(pixel_values)
+11 -3
View File
@@ -70,6 +70,7 @@ from vllm.multimodal.processing.processor import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.multimodal.video_prune.evs import (
compute_retained_tokens_count,
@@ -429,6 +430,9 @@ class NanoNemotronVLMultiModalProcessor(
hf_processor: NanoNemotronVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -470,7 +474,7 @@ class NanoNemotronVLMultiModalProcessor(
return PromptReplacement(
modality="image",
target="<image>",
target=[vocab["<image>"]],
replacement=get_image_replacement,
)
@@ -480,6 +484,8 @@ class NanoNemotronVLMultiModalProcessor(
hf_processor: NanoNemotronVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor)
@@ -546,7 +552,7 @@ class NanoNemotronVLMultiModalProcessor(
return PromptReplacement(
modality="video",
target="<video>",
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
replacement=get_video_replacement,
)
@@ -556,13 +562,15 @@ class NanoNemotronVLMultiModalProcessor(
hf_processor: NanoNemotronVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
def get_audio_replacement(item_idx: int):
audios = mm_items.get_items("audio", AudioProcessorItems)
return hf_processor.get_audio_repl(audios.get(item_idx))
return PromptReplacement(
modality="audio",
target=AUDIO_CONTEXT,
target=cached_encode(tokenizer, AUDIO_CONTEXT, add_special_tokens=False),
replacement=get_audio_replacement,
)
+10 -3
View File
@@ -26,6 +26,7 @@ from vllm.multimodal.parse import (
from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdateDetails,
cached_encode,
)
from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor
@@ -106,6 +107,8 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
hf_processor: NVLMProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -138,14 +141,18 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
repl = hf_processor.get_image_repl(num_patches, num_features=feature_size)
return PromptUpdateDetails.select_text(
repl.full + "\n", hf_processor.ctx_image_token
# `repl.full` ends in the special `</Image>` token, so there is
# no BPE merge across the boundary with the newline
newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
return PromptUpdateDetails.select_token_id(
repl.full + newline_ids, hf_processor.ctx_image_token_id
)
# See note in dummy data regarding why we have the extra newline
return PromptReplacement(
modality="image",
target="<image>\n",
target=cached_encode(tokenizer, "<image>\n", add_special_tokens=False),
replacement=get_replacement_nvlm,
)
+1 -1
View File
@@ -339,7 +339,7 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
tokenizer = self.info.get_tokenizer()
bos_token_id = tokenizer.bos_token_id
def get_insertion(item_idx: int) -> PromptUpdateDetails[list[int]]:
def get_insertion(item_idx: int) -> PromptUpdateDetails:
images = mm_items.get_items(
"image", (ImageEmbeddingItems, ImageProcessorItems)
)
+4 -1
View File
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
cached_encode,
)
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
@@ -383,6 +384,8 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> list[PromptReplacement]:
tokenizer = self.info.get_tokenizer()
def get_replacement_ovis(item_idx: int):
out_item = out_mm_kwargs["image"][item_idx]
grid = out_item["grids"].data
@@ -393,7 +396,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=IMAGE_TOKEN,
target=cached_encode(tokenizer, IMAGE_TOKEN, add_special_tokens=False),
replacement=get_replacement_ovis,
),
]
+14 -2
View File
@@ -54,6 +54,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
ResolvedPromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -436,8 +437,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
def get_image_token_ids(item_idx: int) -> list[int]:
return cached_encode(
tokenizer, image_tokens[item_idx], add_special_tokens=False
)
def get_replacement_phi3v(item_idx: int):
images = mm_items.get_items(
"image", (ImageEmbeddingItems, ImageProcessorItems)
@@ -458,7 +465,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=image_tokens.__getitem__,
target=get_image_token_ids,
replacement=get_replacement_phi3v,
)
]
@@ -475,8 +482,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
if cached_update.modality == "image":
hf_processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
new_update = new_update.with_target(image_tokens[new_item_idx])
new_update = new_update.with_target(
cached_encode(
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
)
)
return new_update
+27 -4
View File
@@ -47,6 +47,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
ResolvedPromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -934,8 +935,20 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
image_tokens: list[str] = self.info.image_tokens # type: ignore
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
def get_image_token_ids(item_idx: int) -> list[int]:
return cached_encode(
tokenizer, image_tokens[item_idx], add_special_tokens=False
)
def get_audio_token_ids(item_idx: int) -> list[int]:
return cached_encode(
tokenizer, audio_tokens[item_idx], add_special_tokens=False
)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
@@ -970,12 +983,12 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=image_tokens.__getitem__,
target=get_image_token_ids,
replacement=get_image_replacement_phi4mm,
),
PromptReplacement(
modality="audio",
target=audio_tokens.__getitem__,
target=get_audio_token_ids,
replacement=get_audio_replacement_phi4mm,
),
]
@@ -990,12 +1003,22 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
new_item_idx,
)
tokenizer = self.info.get_tokenizer()
if cached_update.modality == "image":
image_tokens: list[str] = self.info.image_tokens # type: ignore
new_update = new_update.with_target(image_tokens[new_item_idx])
new_update = new_update.with_target(
cached_encode(
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
)
)
elif cached_update.modality == "audio":
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
new_update = new_update.with_target(audio_tokens[new_item_idx])
new_update = new_update.with_target(
cached_encode(
tokenizer, audio_tokens[new_item_idx], add_special_tokens=False
)
)
return new_update
+6 -1
View File
@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -193,6 +194,8 @@ class Phi4SiglipMultiModalProcessor(
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
def get_replacement(item_idx: int):
# Read the actual patch grid from the NaFlex processor's
# spatial_shapes output (same pattern as LFM2-VL). This avoids
@@ -207,7 +210,9 @@ class Phi4SiglipMultiModalProcessor(
return [
PromptReplacement(
modality="image",
target=DEFAULT_IMAGE_TOKEN,
target=cached_encode(
tokenizer, DEFAULT_IMAGE_TOKEN, add_special_tokens=False
),
replacement=get_replacement,
),
]
+1 -1
View File
@@ -301,7 +301,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
return [
PromptReplacement(
modality="image",
target="", # Never match the prompt (see below note)
target=[], # Never match the prompt (see below note)
replacement=get_replacement,
),
]
@@ -779,17 +779,17 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_qwen2_audio,
),
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=partial(get_replacement_qwen2_vision, modality="image"),
),
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=video_replacement_fn,
),
]
+1 -1
View File
@@ -328,7 +328,7 @@ class Qwen3ASRMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_qwen2_audio,
),
]
@@ -1493,17 +1493,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_qwen2_audio,
),
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=partial(get_replacement_qwen2_vision, modality="image"),
),
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=video_replacement_fn,
),
]
+8 -7
View File
@@ -1555,17 +1555,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
if self._expands_only_video_token(hf_processor):
# transformers>=5.10 expands only the bare video_token
video_target = hf_processor.video_token
video_target = [video_token_id]
else:
# Old-style processors expand the full placeholder
# NOTE: We match string on purpose since searching sequence of
# token ids takes more time.
video_target = "<|vision_start|><|video_pad|><|vision_end|>"
video_target = [
vision_start_token_id,
video_token_id,
vision_end_token_id,
]
return [
PromptReplacement(
modality="image",
target=hf_processor.image_token,
target=[hf_processor.image_token_id],
replacement=get_image_replacement_qwen3vl,
),
PromptReplacement(
@@ -1585,7 +1587,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
vision_end_token_id: int,
video_token_id: int,
select_token_id: bool = False,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Build prompt replacement for a video in Qwen3VL format.
The replacement structure for each frame is:
@@ -1609,7 +1611,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# Tokenize timestamp strings independently to avoid tokenizer merging
# tokens across boundaries.
# TODO: switch to `_seq2tokens` which has some caching.
timestamp_token_ids = [
tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)
for timestamp in timestamps
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
TimingContext,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -700,6 +701,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"""Replace each modality's placeholder token with the token ids that item's
replacement text encodes to, marking which of them hold embeddings."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
if isinstance(token, str):
return cached_encode(tokenizer, token, add_special_tokens=False)
if isinstance(token, int):
return [token]
return list(token)
updates = []
for modality, items in out_mm_kwargs.items():
# Popped so they are neither cached nor sent to the model; the updates
@@ -711,10 +721,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
for item in items
]
token = getattr(hf_processor, f"{modality}_token")
updates.append(
PromptReplacement(
modality=modality,
target=getattr(hf_processor, f"{modality}_token"),
target=get_target_token_ids(token),
replacement=replacements.__getitem__,
)
)
+11 -1
View File
@@ -39,6 +39,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
@@ -298,6 +299,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
replacement_id = hf_processor.audio_replacement_token_id # type: ignore
@@ -322,7 +324,15 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
return [
PromptReplacement(
modality="audio",
target="<|audio|>",
# `<|audio|>` is paired with `replacement_id` in
# `UltravoxProcessingInfo.get_hf_processor`, but it is not
# guaranteed to be a single vocab entry, so match the encoded
# placeholder text instead of `replacement_id`
target=cached_encode(
tokenizer,
hf_processor.audio_token_replacement, # type: ignore
add_special_tokens=False,
),
replacement=get_replacement_ultravox,
)
]
+1 -1
View File
@@ -310,7 +310,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
return [
PromptReplacement(
modality="audio",
target="", # Never match the prompt (see below note)
target=[], # Never match the prompt (see below note)
replacement=get_replacement,
),
]
+6 -6
View File
@@ -722,7 +722,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
image_end_id = vocab[IMAGE_END]
merge_size = self.info.image_processor.merge_size # type: ignore[union-attr]
def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def image_replacement(item_idx: int) -> PromptUpdateDetails:
grid = out_mm_kwargs["image"][item_idx]["image_grid_thw"].data
assert isinstance(grid, torch.Tensor)
num_tokens = int(grid.prod()) // merge_size**2
@@ -750,7 +750,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
* int(config.get("merge_factor", 1))
)
def audio_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def audio_replacement(item_idx: int) -> PromptUpdateDetails:
length = out_mm_kwargs["audio"][item_idx]["audio_lengths"].data
assert isinstance(length, torch.Tensor)
num_tokens = math.ceil(int(length.item()) / stride)
@@ -768,7 +768,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
)
if "video" in out_mm_kwargs:
def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def video_replacement(item_idx: int) -> PromptUpdateDetails:
item = out_mm_kwargs["video"][item_idx]
input_ids_data = item["video_input_ids"].data
embed_mask_data = item["video_embed_mask"].data
@@ -777,8 +777,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
input_ids = input_ids_data.tolist()
embed_mask = embed_mask_data.bool()
def select_video_embeds(tokenizer, full) -> torch.Tensor:
del tokenizer, full
def select_video_embeds(full) -> torch.Tensor:
del full
return embed_mask
return PromptUpdateDetails(
@@ -789,7 +789,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
updates.append(
PromptReplacement(
modality="video",
target=VIDEO_PLACEHOLDER,
target=[vocab[VIDEO_PLACEHOLDER]],
replacement=video_replacement,
)
)
+10 -3
View File
@@ -25,6 +25,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
from vllm.transformers_utils.processor import cached_get_image_processor
@@ -268,8 +269,9 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
media_token_id = self.info.media_token_id
media_token = self.info.media_token
image_placeholder = self.info.get_hf_config().image_placeholder
tokenizer = self.info.get_tokenizer()
def get_replacement(item_idx: int) -> PromptUpdateDetails[str]:
def get_replacement(item_idx: int) -> PromptUpdateDetails:
images = mm_items.get_items("image", ImageProcessorItems)
image = images.get(item_idx)
if image is None:
@@ -296,12 +298,17 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
f"{pads}<|media_end|>"
)
return PromptUpdateDetails.select_token_id(full, media_token_id)
return PromptUpdateDetails.select_token_id(
cached_encode(tokenizer, full, add_special_tokens=False),
media_token_id,
)
return [
PromptReplacement(
modality="image",
target=image_placeholder,
target=cached_encode(
tokenizer, image_placeholder, add_special_tokens=False
),
replacement=get_replacement,
),
]
+2
View File
@@ -11,6 +11,7 @@ from .processor import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
__all__ = [
@@ -20,6 +21,7 @@ __all__ = [
"BaseDummyInputsBuilder",
"ProcessorInputs",
"BaseMultiModalProcessor",
"cached_encode",
"EncDecMultiModalProcessor",
"PromptUpdate",
"PromptIndexTargets",
+223 -311
View File
@@ -12,10 +12,8 @@ from typing import (
NamedTuple,
Protocol,
TypeAlias,
cast,
)
import regex as re
import torch
from typing_extensions import TypeVar, assert_never
@@ -53,74 +51,31 @@ else:
logger = init_logger(__name__)
_S = TypeVar("_S", str, list[int])
PromptSeq: TypeAlias = str | list[int]
"""A token sequence (list of token IDs) or text."""
@lru_cache(maxsize=2048)
def _cached_encode(
def cached_encode(
tokenizer: TokenizerLike,
text: str,
*,
add_special_tokens: bool = True,
) -> list[int]:
"""Encode text while caching repeated tokenizer calls."""
return tokenizer.encode(text, add_special_tokens=add_special_tokens)
@lru_cache(maxsize=2048)
def _cached_decode(
tokenizer: TokenizerLike,
token_ids: tuple[int, ...],
*,
skip_special_tokens: bool = False,
) -> str:
return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
def _seq2text(
tokenizer: TokenizerLike | None,
seq: PromptSeq,
*,
use_cache: bool = True,
) -> str:
if isinstance(seq, str):
return seq
if tokenizer is None:
raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")
if not use_cache:
return tokenizer.decode(seq)
return _cached_decode(tokenizer, tuple(seq))
def _seq2tokens(
tokenizer: TokenizerLike | None,
seq: PromptSeq,
*,
use_cache: bool = True,
) -> list[int]:
if isinstance(seq, str):
if tokenizer is None:
raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")
if not use_cache:
return tokenizer.encode(seq, add_special_tokens=False)
return _cached_encode(tokenizer, seq, add_special_tokens=False)
return seq
class _GetMatchIndex(Protocol):
def __call__(
self,
tokenizer: TokenizerLike | None,
prompt: PromptSeq,
prompt: list[int],
start_idx: int = 0,
) -> int | None: ...
class _GetTextMatchIndex(Protocol):
def __call__(
self,
tokenizer: TokenizerLike,
prompt: str,
start_idx: int = 0,
) -> int | None: ...
@@ -131,6 +86,14 @@ class PromptIndex:
get_match_index: _GetMatchIndex
get_text_match_index: _GetTextMatchIndex
"""
Resolves the index against a decoded text prompt.
Used only by the private text-based fallback of
[`BaseMultiModalProcessor._apply_prompt_updates`][vllm.multimodal.processing.BaseMultiModalProcessor].
"""
class PromptIndexTargets:
@staticmethod
@@ -140,35 +103,40 @@ class PromptIndexTargets:
This results in a match even if the prompt is empty.
"""
return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
return PromptIndex(
lambda prompt, start_idx=0: 0,
lambda tokenizer, prompt, start_idx=0: 0,
)
@staticmethod
def prefix(seq: PromptSeq) -> PromptIndex:
def prefix(seq: list[int]) -> PromptIndex:
"""
Resolves to a location in the prompt after the given prefix.
"""
def get_match_index(
tokenizer: TokenizerLike | None,
prompt: PromptSeq,
prompt: list[int],
start_idx: int = 0,
) -> int | None:
if start_idx != 0:
return None
prefix = seq
match_idx = len(seq)
return match_idx if prompt[:match_idx] == seq else None
if isinstance(prompt, str):
# Make both `str`
prefix = _seq2text(tokenizer, prefix, use_cache=False)
else:
# Make both `list[int]`
prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
def get_text_match_index(
tokenizer: TokenizerLike,
prompt: str,
start_idx: int = 0,
) -> int | None:
if start_idx != 0:
return None
prefix = tokenizer.decode(seq)
match_idx = len(prefix)
return match_idx if prompt[:match_idx] == prefix else None
return PromptIndex(get_match_index)
return PromptIndex(get_match_index, get_text_match_index)
@staticmethod
def end() -> PromptIndex:
@@ -177,33 +145,36 @@ class PromptIndexTargets:
This results in a match even if the prompt is empty.
"""
return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
return PromptIndex(
lambda prompt, start_idx=0: len(prompt),
lambda tokenizer, prompt, start_idx=0: len(prompt),
)
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
UpdateTarget: TypeAlias = list[int] | PromptIndex
"""
The token sequence or text to update.
The token sequence to update.
"""
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).
output the corresponding token sequence.
For convenience, you can directly pass in the token sequence (or text)
For convenience, you can directly pass in the token sequence
instead of a function if it does not depend on the input.
"""
@dataclass
class PromptUpdateDetails(Generic[_S]):
"""Details about the token sequence or text that are part of the update."""
class PromptUpdateDetails:
"""Details about the token sequence that is part of the update."""
full: _S
full: list[int]
"""The full content."""
is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
is_embed: Callable[[list[int]], torch.Tensor] | None = None
"""
Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
return a boolean mask of shape `(len(full),)` indicating which positions
@@ -216,56 +187,36 @@ class PromptUpdateDetails(Generic[_S]):
"""
@staticmethod
def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
def from_seq(seq: list[int]) -> "PromptUpdateDetails":
return PromptUpdateDetails(full=seq)
@staticmethod
def select_text(
seq: _S,
embed_text: str,
) -> "PromptUpdateDetails[_S]":
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
token_ids = _seq2tokens(tokenizer, full)
return torch.isin(
torch.tensor(token_ids),
torch.tensor(embed_token_ids),
)
return PromptUpdateDetails(full=seq, is_embed=is_embed)
@staticmethod
def select_token_id(
seq: _S,
seq: list[int],
embed_token_id: int,
) -> "PromptUpdateDetails[_S]":
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
token_ids = _seq2tokens(tokenizer, full)
return torch.tensor(token_ids) == embed_token_id
) -> "PromptUpdateDetails":
def is_embed(full: list[int]) -> torch.Tensor:
return torch.tensor(full) == embed_token_id
return PromptUpdateDetails(full=seq, is_embed=is_embed)
@staticmethod
def select_token_ids(
seq: _S,
seq: list[int],
embed_token_ids: list[int],
) -> "PromptUpdateDetails[_S]":
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
token_ids = _seq2tokens(tokenizer, full)
) -> "PromptUpdateDetails":
def is_embed(full: list[int]) -> torch.Tensor:
return torch.isin(
torch.tensor(token_ids),
torch.tensor(full),
torch.tensor(embed_token_ids),
)
return PromptUpdateDetails(full=seq, is_embed=is_embed)
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
PromptUpdateInfo: TypeAlias = list[int] | PromptUpdateDetails
"""
The token sequence or text that are part of the update.
The token sequence that is part of the update.
If only part of the content corresponds to feature placeholders, you can
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
@@ -276,9 +227,9 @@ PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdat
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the corresponding token sequence (or text).
output the corresponding token sequence.
For convenience, you can directly pass in the token sequence (or text)
For convenience, you can directly pass in the token sequence
instead of a function if it does not depend on the input.
"""
@@ -292,13 +243,20 @@ class UpdateMode(str, Enum):
class PromptUpdate(ABC):
"""
Defines how to update a prompt with placeholder tokens.
Note:
The target and content are token sequences. When converting text
to token sequences, remember to encode it with
`add_special_tokens=False` to match how the prompt itself is
tokenized. Otherwise, the updated prompt may contain duplicated
special tokens or fail to match the target.
"""
modality: str
"""The modality for which the update is made."""
target: PromptUpdateTarget
"""The token sequence (or text) to update."""
"""The token sequence to update."""
@property
@abstractmethod
@@ -357,8 +315,8 @@ class PromptInsertion(PromptUpdate):
```python
PromptInsertion(
modality="image",
target="<s>",
insertion="<image>" * image_feature_size,
target=[bos_token_id],
insertion=[image_token_id] * image_feature_size,
)
```
@@ -368,7 +326,7 @@ class PromptInsertion(PromptUpdate):
PromptInsertion(
modality="image",
target=PromptIndexTargets.start(),
insertion="<image>" * image_feature_size,
insertion=[image_token_id] * image_feature_size,
)
```
@@ -377,8 +335,8 @@ class PromptInsertion(PromptUpdate):
```python
PromptInsertion(
modality="image",
target=PromptIndexTargets.prefix("Images:"),
insertion="<image>" * image_feature_size,
target=PromptIndexTargets.prefix(images_prefix_token_ids),
insertion=[image_token_id] * image_feature_size,
)
```
@@ -388,7 +346,7 @@ class PromptInsertion(PromptUpdate):
PromptInsertion(
modality="image",
target=PromptIndexTargets.end(),
insertion="<image>" * image_feature_size,
insertion=[image_token_id] * image_feature_size,
)
```
"""
@@ -397,10 +355,10 @@ class PromptInsertion(PromptUpdate):
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the token sequence (or text) to insert right after
output the token sequence to insert right after
[`target`][vllm.multimodal.processing.PromptUpdate.target].
For convenience, you can directly pass in the token sequence (or text)
For convenience, you can directly pass in the token sequence
instead of a function if it does not depend on the input.
"""
@@ -427,8 +385,8 @@ class PromptReplacement(PromptUpdate):
```python
PromptReplacement(
modality="image",
target="<image>",
replacement="<image>" * image_feature_size,
target=[image_token_id],
replacement=[image_token_id] * image_feature_size,
)
```
@@ -436,26 +394,6 @@ class PromptReplacement(PromptUpdate):
and `<image_eos>`, which are not supposed to be passed to the vision
encoder:
```python
PromptReplacement(
modality="image",
target="<image>",
replacement=PromptUpdateDetails(
full="".join(
[
"<image_bos>",
"<image>" * image_feature_size,
"<image_eos>",
]
),
features="<image>" * image_feature_size,
),
)
```
To avoid unnecessary tokenization during prompt replacement,
we recommended passing token sequences instead of text:
```python
PromptReplacement(
modality="image",
@@ -464,7 +402,7 @@ class PromptReplacement(PromptUpdate):
full=(
[image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
),
features=[image_token_id] * image_feature_size,
is_embed=lambda full: torch.tensor(full) == image_token_id,
),
)
```
@@ -474,10 +412,10 @@ class PromptReplacement(PromptUpdate):
"""
Given the index of the processed item within
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
output the token sequence (or text) to replace
output the token sequence to replace
[`target`][vllm.multimodal.processing.PromptUpdate.target].
For convenience, you can directly pass in the token sequence (or text)
For convenience, you can directly pass in the token sequence
instead of a function if it does not depend on the input.
"""
@@ -520,7 +458,7 @@ class PromptTargetMatch(NamedTuple):
class ResolvedPromptUpdate:
"""
A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
lazy attributes resolved, apart from those related to tokenization.
lazy attributes resolved.
"""
modality: str
@@ -533,7 +471,7 @@ class ResolvedPromptUpdate:
"""Defines how to update the prompt."""
target: UpdateTarget
"""The token sequence (or text) to update."""
"""The token sequence to update."""
content: PromptUpdateDetails = field(repr=False)
"""The placeholder tokens that are part of the update."""
@@ -541,7 +479,6 @@ class ResolvedPromptUpdate:
def iter_token_matches(
self,
prompt: list[int],
tokenizer: TokenizerLike | None,
*,
start_idx: int = 0,
) -> Generator[PromptTargetMatch]:
@@ -549,52 +486,15 @@ class ResolvedPromptUpdate:
target = self.target
if isinstance(target, PromptIndex):
match_idx = target.get_match_index(tokenizer, prompt, start_idx)
match_idx = target.get_match_index(prompt, start_idx)
if match_idx is not None:
yield PromptTargetMatch(match_idx, match_idx)
return
target_token_ids = _seq2tokens(tokenizer, target)
for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
for match in iter_token_matches(prompt, target, start_idx=start_idx):
yield PromptTargetMatch(match.start_idx, match.end_idx)
def iter_text_matches(
self,
prompt: str,
tokenizer: TokenizerLike | None,
*,
start_idx: int = 0,
) -> Generator[PromptTargetMatch]:
"""Yield each instance of `self.target` found in `prompt`."""
target = self.target
if isinstance(target, PromptIndex):
match_idx = target.get_match_index(tokenizer, prompt, start_idx)
if match_idx is not None:
yield PromptTargetMatch(match_idx, match_idx)
return
target_text = _seq2text(tokenizer, target)
for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
yield PromptTargetMatch(match.start(), match.end())
def iter_matches(
self,
prompt: list[int] | str,
tokenizer: TokenizerLike | None,
*,
start_idx: int = 0,
) -> Generator[PromptTargetMatch]:
"""Yield each instance of `self.target` found in `prompt`."""
if isinstance(prompt, str):
return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
def with_target(self, target: UpdateTarget):
return replace(self, target=target)
@@ -729,8 +629,6 @@ def _target_key(target: UpdateTarget) -> tuple[str, object]:
"""Return a hashable key that preserves target matching semantics."""
if isinstance(target, PromptIndex):
return ("index", id(target))
if isinstance(target, str):
return ("text", target)
return ("tokens", tuple(target))
@@ -761,10 +659,16 @@ def _compile_prompt_update_queues(
return queues_by_signature
_IterMatches: TypeAlias = Callable[
[ResolvedPromptUpdate, int],
Generator[PromptTargetMatch, None, None],
]
"""Yields each match of an update's target, given the search start index."""
def _find_queue_match(
queue: _UpdateQueue,
prompt: _S,
tokenizer: TokenizerLike | None,
iter_matches: _IterMatches,
*,
start_idx: int,
mode: UpdateMode | None = None,
@@ -775,10 +679,7 @@ def _find_queue_match(
if mode is not None and update.mode != mode:
continue
match = next(
update.iter_matches(prompt, tokenizer, start_idx=start_idx),
None,
)
match = next(iter_matches(update, start_idx), None)
if match is not None:
return match, update_idx
@@ -791,10 +692,9 @@ def _next_priority(queue: _UpdateQueue) -> int:
return priority
def _plan_prompt_updates(
prompt: _S,
def _plan_prompt_updates_with(
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
iter_matches: _IterMatches,
) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
"""Plan non-overlapping prompt updates before rendering the output."""
queues = list(_compile_prompt_update_queues(mm_prompt_updates).values())
@@ -809,8 +709,7 @@ def _plan_prompt_updates(
for queue in queues:
queue_match = _find_queue_match(
queue,
prompt,
tokenizer,
iter_matches,
start_idx=prev_end_idx,
)
if queue_match is not None:
@@ -835,8 +734,7 @@ def _plan_prompt_updates(
queue_match = _find_queue_match(
queue,
prompt,
tokenizer,
iter_matches,
start_idx=prev_end_idx,
mode=mode,
)
@@ -887,6 +785,19 @@ def _plan_prompt_updates(
return planned_updates, result
def _plan_prompt_updates(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
"""Plan non-overlapping prompt updates before rendering the output."""
return _plan_prompt_updates_with(
mm_prompt_updates,
lambda update, start_idx: update.iter_token_matches(
prompt, start_idx=start_idx
),
)
def _all_items_found(
mm_item_counts: dict[str, int],
mm_found_counts: dict[str, int],
@@ -898,22 +809,16 @@ def _all_items_found(
def _apply_matches(
prompt: _S,
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
out_seqs = list[str | list[int]]()
matched_updates, result = _plan_prompt_updates(
prompt,
mm_prompt_updates,
tokenizer,
)
) -> tuple[list[list[int]], "MultiModalPromptUpdatesApplyResult"]:
out_seqs = list[list[int]]()
matched_updates, result = _plan_prompt_updates(prompt, mm_prompt_updates)
prev_end_idx = 0
for matched_update in matched_updates:
update = matched_update.update
match = matched_update.match
matched_content = update.content.full
if update.mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
@@ -923,22 +828,17 @@ def _apply_matches(
assert_never(update.mode)
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
out_seqs.append(
_seq2text(tokenizer, matched_content)
if isinstance(prompt, str)
else _seq2tokens(tokenizer, matched_content)
)
out_seqs.append(update.content.full)
prev_end_idx = match.end_idx
out_seqs.append(prompt[prev_end_idx:])
return cast(list[_S], out_seqs), result
return out_seqs, result
def apply_token_matches(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
"""
Apply the updates in `mm_prompt_updates` to `prompt`.
@@ -947,7 +847,7 @@ def apply_token_matches(
the same placeholder tokens. In that case, the modality that
appears earlier in `mm_prompt_updates` takes priority.
"""
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates)
return flatten_2d_lists(token_id_seqs), result
@@ -955,17 +855,12 @@ def apply_token_matches(
def _apply_token_matches_with_placeholders(
token_ids: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[
list[int],
"MultiModalPromptUpdatesApplyResult",
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
matched_updates, result = _plan_prompt_updates(
token_ids,
mm_prompt_updates,
tokenizer,
)
matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
modality: [] for modality in mm_prompt_updates
}
@@ -975,7 +870,6 @@ def _apply_token_matches_with_placeholders(
for matched_update in matched_updates:
update = matched_update.update
match = matched_update.match
matched_content = update.content.full
if update.mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
@@ -987,11 +881,11 @@ def _apply_token_matches_with_placeholders(
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
start_idx = len(new_token_ids)
tokens = _seq2tokens(tokenizer, matched_content)
tokens = update.content.full
if tokens:
content_is_embed = update.content.is_embed
if content_is_embed is not None:
content_is_embed = content_is_embed(tokenizer, matched_content)
content_is_embed = content_is_embed(tokens)
placeholders[update.modality].append(
PlaceholderFeaturesInfo(
@@ -1010,50 +904,9 @@ def _apply_token_matches_with_placeholders(
return new_token_ids, result, placeholders
def apply_text_matches(
prompt: str,
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
"""
Apply the updates in `mm_prompt_updates` to `prompt`.
Matches are exclusive even when multiple modalities share
the same placeholder tokens. In that case, the modality that
appears earlier in `mm_prompt_updates` takes priority.
"""
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
return "".join(texts), result
def apply_text_matches_as_segmented_tokens(
prompt: str,
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
"""
Apply the updates in `mm_prompt_updates` to `prompt`.
Matches are exclusive even when multiple modalities share
the same placeholder tokens. In that case, the modality that
appears earlier in `mm_prompt_updates` takes priority.
Each segment is encoded separately instead of being joined into one
string and encoded in a single pass. Joining first would let BPE merge
tokens across a segment boundary, silently change how a text
(non-special-token) placeholder is tokenized.
"""
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
return flatten_2d_lists(token_id_seqs), result
def _iter_placeholders(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> Iterable[PlaceholderFeaturesInfo]:
"""
Yield each set of placeholder tokens found in `prompt`.
@@ -1073,15 +926,15 @@ def _iter_placeholders(
prompt_len = len(prompt)
start_idx = 0
# The current (unfound) item's updates for each modality, with their
# content resolved to token ids; rebuilt whenever an item is found.
# Items are only resolved once the scan reaches them.
# The current (unfound) item's updates for each modality; rebuilt
# whenever an item is found. Items are only resolved once the scan
# reaches them.
candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None
while start_idx < prompt_len:
if candidates is None:
candidates = [
(modality, update, _seq2tokens(tokenizer, update.content.full))
(modality, update, update.content.full)
for modality, modality_updates in mm_prompt_updates.items()
if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0)
for update in modality_updates[item_idx_by_modality[modality]]
@@ -1107,7 +960,7 @@ def _iter_placeholders(
content = update.content
content_is_embed = content.is_embed
if content_is_embed is not None:
content_is_embed = content_is_embed(tokenizer, content.full)
content_is_embed = content_is_embed(content.full)
yield PlaceholderFeaturesInfo(
modality=modality,
@@ -1134,9 +987,8 @@ def _iter_placeholders(
def find_mm_placeholders(
prompt: list[int],
mm_prompt_updates: "MultiModalPromptUpdates",
tokenizer: TokenizerLike | None,
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
it = _iter_placeholders(prompt, mm_prompt_updates)
return dict(full_groupby_modality(it))
@@ -1146,7 +998,7 @@ A collection of the `is_cached` flag for each item, with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
"""
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
MultiModalPromptUpdates = Mapping[str, Sequence[Sequence[ResolvedPromptUpdate]]]
"""
A collection of prompt updates with a similar structure as
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
@@ -1283,9 +1135,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
new_token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
tokenizer = self.info.get_tokenizer()
return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
return find_mm_placeholders(new_token_ids, mm_prompt_updates)
def _get_hf_mm_data(
self,
@@ -1606,8 +1456,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
tokenizer = self.info.get_tokenizer()
return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
return apply_token_matches(prompt, mm_prompt_updates)
def _apply_token_matches_with_placeholders(
self,
@@ -1618,31 +1467,105 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
MultiModalPromptUpdatesApplyResult,
Mapping[str, list[PlaceholderFeaturesInfo]],
]:
tokenizer = self.info.get_tokenizer()
return _apply_token_matches_with_placeholders(
token_ids,
mm_prompt_updates,
tokenizer,
)
def _apply_text_matches(
def _iter_text_matches(
self,
prompt: str,
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
tokenizer = self.info.get_tokenizer()
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
update: ResolvedPromptUpdate,
prompt_text: str,
tokenizer: TokenizerLike,
*,
start_idx: int = 0,
) -> Generator[PromptTargetMatch, None, None]:
"""Yield each instance of `update.target` found in `prompt_text`."""
target = update.target
def _apply_text_matches_as_segmented_tokens(
if isinstance(target, PromptIndex):
match_idx = target.get_text_match_index(tokenizer, prompt_text, start_idx)
if match_idx is not None:
yield PromptTargetMatch(match_idx, match_idx)
return
target_text = tokenizer.decode(target)
if not target_text:
return
idx = prompt_text.find(target_text, start_idx)
while idx != -1:
end_idx = idx + len(target_text)
yield PromptTargetMatch(idx, end_idx)
idx = prompt_text.find(target_text, end_idx)
def _apply_prompt_updates_via_text(
self,
prompt: str,
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
*,
encode_segments_separately: bool = False,
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
"""
Apply multi-modal prompt updates in text space, then encode the
result.
A non-special-token target may be tokenized differently inside the
prompt because the tokens may go across the boundaries of the target
(e.g. when searching for "foo" in "food", if "food" itself makes up
a token, then the token ID of "foo" will not appear at all). Since
it is inefficient to search for all possible tokenizations of the
target in the prompt, we instead perform text-based updates on the
decoded token IDs, then encode them back.
"""
tokenizer = self.info.get_tokenizer()
return apply_text_matches_as_segmented_tokens(
prompt, mm_prompt_updates, tokenizer
prompt_text = tokenizer.decode(token_ids)
matched_updates, result = _plan_prompt_updates_with(
mm_prompt_updates,
lambda update, start_idx: self._iter_text_matches(
update, prompt_text, tokenizer, start_idx=start_idx
),
)
out_texts = list[str]()
prev_end_idx = 0
for matched_update in matched_updates:
update = matched_update.update
match = matched_update.match
if update.mode == UpdateMode.INSERT:
end_idx_to_insert = match.end_idx
elif update.mode == UpdateMode.REPLACE:
end_idx_to_insert = match.start_idx
else:
assert_never(update.mode)
out_texts.append(prompt_text[prev_end_idx:end_idx_to_insert])
out_texts.append(tokenizer.decode(update.content.full))
prev_end_idx = match.end_idx
out_texts.append(prompt_text[prev_end_idx:])
if encode_segments_separately:
# Encode each segment separately instead of joining into one
# text and encoding it in a single pass. Joining first would
# let BPE merge tokens across a segment boundary, silently
# changing how a non-special-token placeholder is tokenized.
new_token_ids = flatten_2d_lists(
[
cached_encode(tokenizer, text, add_special_tokens=False)
for text in out_texts
]
)
else:
new_token_ids = cached_encode(
tokenizer, "".join(out_texts), add_special_tokens=False
)
return new_token_ids, result
def _matched_updates_from_result(
self,
mm_prompt_updates: MultiModalPromptUpdates,
@@ -1668,8 +1591,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Apply multi-modal prompt updates to token IDs."""
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result, placeholders = (
self._apply_token_matches_with_placeholders(
token_ids,
@@ -1688,23 +1609,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
}
return new_token_ids, placeholders
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
new_text, match_result = self._apply_text_matches(
_seq2text(tokenizer, token_ids, use_cache=False),
# A non-special-token target may be tokenized differently inside
# the prompt, so fall back to performing the updates on the
# decoded text, then encoding the result back.
new_token_ids, match_result = self._apply_prompt_updates_via_text(
token_ids,
mm_prompt_updates,
)
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
placeholders = self._find_mm_placeholders(
new_token_ids,
self._matched_updates_from_result(mm_prompt_updates, match_result),
+2 -6
View File
@@ -157,7 +157,7 @@ def _expand_prompt_embeds_placeholders(
`token_ids` is replaced with a consecutive span of
`tensor.shape[0]` copies, following tensors in order.
"""
expanded, _ = apply_token_matches(token_ids, mm_prompt_updates, tokenizer=None)
expanded, _ = apply_token_matches(token_ids, mm_prompt_updates)
return expanded
@@ -171,11 +171,7 @@ def _build_prompt_embeds_positions(
Expects `token_ids` to already contain expanded N-token spans.
Returns `[(start_idx, length), ...]` aligned with the tensors.
"""
placeholders = find_mm_placeholders(
prompt=token_ids,
mm_prompt_updates=mm_prompt_updates,
tokenizer=None,
)
placeholders = find_mm_placeholders(token_ids, mm_prompt_updates)
features = placeholders.get("prompt_embeds", [])
if len(features) != num_tensors:
+24 -7
View File
@@ -21,7 +21,7 @@ from transformers import (
from transformers.processing_utils import ProcessorMixin
from vllm.multimodal.image import convert_image_mode
from vllm.multimodal.processing import PromptUpdateDetails
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
from vllm.tokenizers.hf import HfTokenizer
IMAGENET_MEAN = (0.485, 0.456, 0.406)
@@ -444,7 +444,7 @@ class InternVLProcessor(ProcessorMixin):
self,
num_patches: int | None,
num_features: int | None = None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
if num_patches is None:
assert num_features is not None
else:
@@ -453,9 +453,11 @@ class InternVLProcessor(ProcessorMixin):
repl_features = self.ctx_image_token * num_features
repl_full = self.start_image_token + repl_features + self.end_image_token
return PromptUpdateDetails.select_text(repl_full, self.ctx_image_token)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails[str]:
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails:
assert self.ctx_video_token is not None
repl_features = self.ctx_video_token * self.image_seq_length
@@ -467,7 +469,10 @@ class InternVLProcessor(ProcessorMixin):
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
)
return PromptUpdateDetails.select_text(repl_full, self.ctx_video_token)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
assert self.ctx_video_token_id is not None
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id)
def __call__(
self,
@@ -523,7 +528,13 @@ class InternVLProcessor(ProcessorMixin):
while image_token in new_prompt:
new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
image_repl = self.get_image_repl(image_num_patches[image_index])
replace_strings.append(image_repl.full)
# Convert the token IDs back to text for the
# string-based replacement below
replace_strings.append(
self.tokenizer.decode(
image_repl.full, skip_special_tokens=False
)
)
image_index += 1
while "<placeholder>" in new_prompt:
@@ -548,7 +559,13 @@ class InternVLProcessor(ProcessorMixin):
while video_token in new_prompt:
new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
video_repl = self.get_video_repl(video_num_patches[video_index])
replace_strings.append(video_repl.full)
# Convert the token IDs back to text for the
# string-based replacement below
replace_strings.append(
self.tokenizer.decode(
video_repl.full, skip_special_tokens=False
)
)
video_index += 1
while "<placeholder>" in new_prompt:
@@ -24,7 +24,7 @@ from transformers import BatchFeature, PretrainedConfig, TensorType
from vllm.model_executor.models.parakeet import ParakeetExtractor
from vllm.multimodal.inputs import AudioItem
from vllm.multimodal.processing.processor import PromptUpdateDetails
from vllm.multimodal.processing.processor import PromptUpdateDetails, cached_encode
from vllm.multimodal.video_prune.evs import compute_retained_tokens_count
from vllm.platforms import current_platform
from vllm.tokenizers.hf import HfTokenizer
@@ -633,7 +633,7 @@ class BaseNanoNemotronVLProcessor(ABC):
self,
feature_size: int,
num_patches: int | None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
raise NotImplementedError
def get_num_image_tokens(
@@ -730,7 +730,12 @@ class BaseNanoNemotronVLProcessor(ABC):
zip(num_tokens_per_image, image_num_patches, strict=True)
):
image_repl = self.get_image_repl(feature_size, num_patches)
parts[i] = parts[i].replace("<image>", image_repl.full)
# image_repl.full is a list of token IDs
# Convert token IDs back to text for the HF processor flow
image_repl_text = self.tokenizer.decode(
image_repl.full, skip_special_tokens=False
)
parts[i] = parts[i].replace("<image>", image_repl_text)
text = ["".join(parts)]
return text, image_inputs
@@ -1001,7 +1006,11 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
for idx, part in enumerate(parts):
if part == AUDIO_CONTEXT:
audio_repl = self.get_audio_repl(audios[audio_index])
parts[idx] = audio_repl.full
# audio_repl.full is a list of token IDs
# Convert token IDs back to text for the HF processor flow
parts[idx] = self.tokenizer.decode(
audio_repl.full, skip_special_tokens=False
)
audio_index += 1
text = ["".join(parts)]
audio_inputs = extractor(audios)
@@ -1078,20 +1087,30 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
self,
feature_size: int,
num_patches: int | None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
repl_features = IMG_CONTEXT * feature_size
repl_full = IMG_START + repl_features + IMG_END
return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
return PromptUpdateDetails.select_token_ids(
full_ids, self._img_context_token_ids
)
def get_audio_repl(
self,
audio: npt.NDArray,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
assert self.audio_extractor is not None
num_tokens = self.audio_extractor.audio_token_count(len(audio))
repl_full = f"{AUDIO_START}{AUDIO_CONTEXT * num_tokens}{AUDIO_END}"
return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
ctx_token_ids = cached_encode(
self.tokenizer, AUDIO_CONTEXT, add_special_tokens=False
)
return PromptUpdateDetails.select_token_ids(full_ids, ctx_token_ids)
@classmethod
def get_video_repl(
@@ -1105,7 +1124,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
img_end_token_ids: list[int],
img_context_token_ids: list[int],
video_temporal_patch_size: int = 1,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""
Build prompt replacement for a video.
The replacement returned is not actually used to replace the placeholder
+5 -3
View File
@@ -7,7 +7,7 @@
# Copyright (c) 2024 NVIDIA
# Licensed under Apache 2.0 License [see LICENSE for details]
# --------------------------------------------------------
from vllm.multimodal.processing import PromptUpdateDetails
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
from vllm.tokenizers.hf import HfTokenizer
from .internvl import InternVLImageProcessor, InternVLProcessor
@@ -37,7 +37,7 @@ class NVLMProcessor(InternVLProcessor):
self,
num_patches: int | None,
num_features: int | None = None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
if num_patches is None:
raise NotImplementedError("Embedding inputs are not supported")
@@ -58,4 +58,6 @@ class NVLMProcessor(InternVLProcessor):
# when trying to find "<tile" as a subsequence of "<Image><tile"
repl = self.start_image_token + features + self.end_image_token
return PromptUpdateDetails.select_text(repl, self.ctx_image_token)
full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)