[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
@@ -532,6 +532,12 @@ return a list of [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instanc
|
||||
Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies an update operation
|
||||
(e.g.: insertion, replacement) performed by the HF processor.
|
||||
|
||||
!!! note
|
||||
The target and content of each update are token sequences. When converting
|
||||
text to token sequences, remember to encode it with
|
||||
`cached_encode(..., add_special_tokens=False)`. Otherwise, the updated prompt
|
||||
may contain duplicated special tokens or fail to match the target.
|
||||
|
||||
=== "Basic example: LLaVA"
|
||||
|
||||
Looking at HF's `LlavaProcessor`:
|
||||
|
||||
@@ -192,11 +192,11 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
|
||||
assert resolved.modality == "image"
|
||||
assert [
|
||||
(match.start_idx, match.end_idx)
|
||||
for match in resolved.iter_matches([1, 10, 11], _FakeTokenizer())
|
||||
for match in resolved.iter_token_matches([1, 10, 11])
|
||||
] == [(1, 1)]
|
||||
assert content.full == [32000] * 256
|
||||
|
||||
is_embed = content.is_embed(None, content.full)
|
||||
is_embed = content.is_embed(content.full)
|
||||
assert is_embed.dtype == torch.bool
|
||||
assert is_embed.tolist() == [True] * 256
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ def test_cache_item_size(item, expected_size):
|
||||
cache[""] = item
|
||||
assert cache.currsize == expected_size
|
||||
|
||||
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
|
||||
prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0)
|
||||
|
||||
cache[""] = MultiModalProcessorCacheItem(item, [prompt_update])
|
||||
assert cache.currsize == expected_size
|
||||
@@ -151,7 +151,7 @@ def _compare_caches(
|
||||
for item in all_items
|
||||
]
|
||||
|
||||
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
|
||||
prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0)
|
||||
|
||||
for it in range(n_iter):
|
||||
num_items_to_select = rng.randint(0, max_items_per_iter)
|
||||
|
||||
+154
-341
@@ -12,18 +12,19 @@ from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.processing.context import InputProcessingContext
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptIndexTargets,
|
||||
PromptInsertion,
|
||||
PromptReplacement,
|
||||
_apply_matches,
|
||||
_apply_token_matches_with_placeholders,
|
||||
apply_text_matches,
|
||||
apply_token_matches,
|
||||
find_mm_placeholders,
|
||||
iter_token_matches,
|
||||
replace_token_matches,
|
||||
)
|
||||
from vllm.utils.collection_utils import flatten_2d_lists
|
||||
|
||||
from .utils import random_image
|
||||
|
||||
@@ -242,7 +243,7 @@ def test_find_token_matches(
|
||||
for key, target in target_by_key.items()
|
||||
}
|
||||
result = {
|
||||
key: list(update.iter_token_matches(prompt, tokenizer=None))
|
||||
key: list(update.iter_token_matches(prompt))
|
||||
for key, update in prompt_updates.items()
|
||||
}
|
||||
|
||||
@@ -259,315 +260,6 @@ def test_find_token_matches(
|
||||
} == expected_by_key
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "expected_by_key"),
|
||||
[
|
||||
# Detokenized test cases of `test_find_token_matches`
|
||||
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
|
||||
(
|
||||
"",
|
||||
{
|
||||
"pattern_1": "",
|
||||
"pattern_2": "<image>",
|
||||
"pattern_3": PromptIndexTargets.start(),
|
||||
"pattern_4": PromptIndexTargets.prefix("<image>"),
|
||||
"pattern_5": PromptIndexTargets.end(),
|
||||
},
|
||||
{
|
||||
"pattern_1": [{"start_idx": 0, "end_idx": 0}],
|
||||
"pattern_2": [],
|
||||
"pattern_3": [
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_4": [],
|
||||
"pattern_5": [
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
"<image><image><image><image>",
|
||||
{
|
||||
"pattern_1": "<image>",
|
||||
"pattern_2": "<image><image>",
|
||||
"pattern_3": "<image><image><image>",
|
||||
"pattern_4": PromptIndexTargets.start(),
|
||||
"pattern_5": PromptIndexTargets.prefix("<image>"),
|
||||
"pattern_6": PromptIndexTargets.end(),
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{"start_idx": 0, "end_idx": 7},
|
||||
{"start_idx": 7, "end_idx": 14},
|
||||
{"start_idx": 14, "end_idx": 21},
|
||||
{"start_idx": 21, "end_idx": 28},
|
||||
],
|
||||
"pattern_2": [
|
||||
{"start_idx": 0, "end_idx": 14},
|
||||
{"start_idx": 14, "end_idx": 28},
|
||||
],
|
||||
"pattern_3": [
|
||||
{"start_idx": 0, "end_idx": 21},
|
||||
],
|
||||
"pattern_4": [
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_5": [
|
||||
{"start_idx": 7, "end_idx": 7},
|
||||
],
|
||||
"pattern_6": [
|
||||
{"start_idx": 28, "end_idx": 28},
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
"Image:<image><image><image>Image:<image><image>!",
|
||||
{
|
||||
"pattern_1": "Image:<image>",
|
||||
"pattern_2": "Image:<image><image><image>",
|
||||
"pattern_3": "Image:<unk><image>",
|
||||
"pattern_4": PromptIndexTargets.start(),
|
||||
"pattern_5": PromptIndexTargets.prefix("Image:<image>"),
|
||||
"pattern_6": PromptIndexTargets.end(),
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{"start_idx": 0, "end_idx": 13},
|
||||
{"start_idx": 27, "end_idx": 40},
|
||||
],
|
||||
"pattern_2": [
|
||||
{"start_idx": 0, "end_idx": 27},
|
||||
],
|
||||
"pattern_3": [],
|
||||
"pattern_4": [
|
||||
{"start_idx": 0, "end_idx": 0},
|
||||
],
|
||||
"pattern_5": [
|
||||
{"start_idx": 13, "end_idx": 13},
|
||||
],
|
||||
"pattern_6": [
|
||||
{"start_idx": 48, "end_idx": 48},
|
||||
],
|
||||
},
|
||||
),
|
||||
# Test regex escape
|
||||
(
|
||||
"<|image|><image><|image|><image>",
|
||||
{
|
||||
"pattern_1": "<|image|>",
|
||||
"pattern_2": "<|image|><image>",
|
||||
"pattern_3": "<|image|><image><|image|>",
|
||||
},
|
||||
{
|
||||
"pattern_1": [
|
||||
{"start_idx": 0, "end_idx": 9},
|
||||
{"start_idx": 16, "end_idx": 25},
|
||||
],
|
||||
"pattern_2": [
|
||||
{"start_idx": 0, "end_idx": 16},
|
||||
{"start_idx": 16, "end_idx": 32},
|
||||
],
|
||||
"pattern_3": [
|
||||
{"start_idx": 0, "end_idx": 25},
|
||||
],
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
|
||||
def test_find_text_matches(
|
||||
prompt,
|
||||
target_by_key,
|
||||
expected_by_key,
|
||||
update_type,
|
||||
):
|
||||
prompt_updates = {
|
||||
key: update_type(key, target, []).resolve(0)
|
||||
for key, target in target_by_key.items()
|
||||
}
|
||||
result = {
|
||||
key: list(update.iter_text_matches(prompt, tokenizer=None))
|
||||
for key, update in prompt_updates.items()
|
||||
}
|
||||
|
||||
# Only displayed on error
|
||||
print("result:", result)
|
||||
|
||||
# Manually constructed results
|
||||
assert {
|
||||
key: [
|
||||
dict(start_idx=item.start_idx, end_idx=item.end_idx)
|
||||
for item in result.get(key, [])
|
||||
]
|
||||
for key in expected_by_key
|
||||
} == expected_by_key
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
|
||||
[
|
||||
(
|
||||
"Image:<image>Image:<image><image>!",
|
||||
{
|
||||
# We use `<image>` before `Image:` to test matches that
|
||||
# occur out of order
|
||||
"pattern_1": "<image>",
|
||||
"pattern_2": "Image:",
|
||||
"pattern_3": "!",
|
||||
},
|
||||
{
|
||||
# Test whether target is confused with replacement
|
||||
"pattern_1": "<image><image>",
|
||||
# Test empty replacement
|
||||
"pattern_2": "",
|
||||
# Test dynamic replacement (beyond the form of `unit * count`)
|
||||
"pattern_3": "?!?",
|
||||
},
|
||||
{
|
||||
PromptInsertion: {
|
||||
0: "Image:<image>Image:<image><image>!",
|
||||
1: "Image:<image><image><image>Image:<image><image>!?!?",
|
||||
2: "Image:<image><image><image><image><image>Image:<image><image>!?!??!?", # noqa: E501
|
||||
},
|
||||
PromptReplacement: {
|
||||
0: "Image:<image>Image:<image><image>!",
|
||||
1: "<image><image>Image:<image><image>?!?",
|
||||
2: "<image><image><image><image><image>?!?",
|
||||
},
|
||||
},
|
||||
),
|
||||
# Test index targets
|
||||
(
|
||||
"",
|
||||
{
|
||||
"pattern_1": PromptIndexTargets.start(),
|
||||
"pattern_2": PromptIndexTargets.prefix("<image>"),
|
||||
"pattern_3": PromptIndexTargets.end(),
|
||||
},
|
||||
{
|
||||
"pattern_1": "1",
|
||||
"pattern_2": "2",
|
||||
"pattern_3": "3",
|
||||
},
|
||||
{
|
||||
PromptInsertion: {
|
||||
0: "",
|
||||
1: "13",
|
||||
2: "1133",
|
||||
},
|
||||
PromptReplacement: {
|
||||
0: "",
|
||||
1: "13",
|
||||
2: "1133",
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
"<image>",
|
||||
{
|
||||
"pattern_1": PromptIndexTargets.start(),
|
||||
"pattern_2": PromptIndexTargets.prefix("<image>"),
|
||||
"pattern_3": PromptIndexTargets.end(),
|
||||
},
|
||||
{
|
||||
"pattern_1": "1",
|
||||
"pattern_2": "2",
|
||||
"pattern_3": "3",
|
||||
},
|
||||
{
|
||||
PromptInsertion: {
|
||||
0: "<image>",
|
||||
1: "1<image>23",
|
||||
2: "11<image>2233",
|
||||
},
|
||||
PromptReplacement: {
|
||||
0: "<image>",
|
||||
1: "1<image>23",
|
||||
2: "11<image>2233",
|
||||
},
|
||||
},
|
||||
),
|
||||
# Test different replacement per item
|
||||
(
|
||||
"<image><image><image>",
|
||||
{
|
||||
"pattern_1": "<image>",
|
||||
},
|
||||
{
|
||||
"pattern_1": lambda idx: str(idx + 1),
|
||||
},
|
||||
{
|
||||
PromptInsertion: {
|
||||
0: "<image><image><image>",
|
||||
1: "<image>1<image><image>",
|
||||
2: "<image>12<image><image>",
|
||||
},
|
||||
PromptReplacement: {
|
||||
0: "<image><image><image>",
|
||||
1: "1<image><image>",
|
||||
2: "12<image>",
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
"<image><image><image>",
|
||||
{
|
||||
"pattern_1": PromptIndexTargets.prefix("<image>"),
|
||||
},
|
||||
{
|
||||
"pattern_1": lambda idx: str(idx + 1),
|
||||
},
|
||||
{
|
||||
PromptInsertion: {
|
||||
0: "<image><image><image>",
|
||||
1: "<image>1<image><image>",
|
||||
2: "<image>12<image><image>",
|
||||
},
|
||||
PromptReplacement: {
|
||||
0: "<image><image><image>",
|
||||
1: "<image>1<image><image>",
|
||||
2: "<image>12<image><image>",
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_find_update_text(
|
||||
prompt,
|
||||
target_by_key,
|
||||
repl_by_key,
|
||||
expected_by_update_type_mm_count,
|
||||
):
|
||||
for (
|
||||
update_type,
|
||||
expected_by_mm_count,
|
||||
) in expected_by_update_type_mm_count.items():
|
||||
for mm_count, expected in expected_by_mm_count.items():
|
||||
mm_prompt_updates = {
|
||||
key: [
|
||||
[update_type(key, target, repl_by_key[key]).resolve(i)]
|
||||
for i in range(mm_count)
|
||||
]
|
||||
for key, target in target_by_key.items()
|
||||
}
|
||||
|
||||
new_prompt, result = apply_text_matches(
|
||||
prompt,
|
||||
mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
|
||||
# Only displayed on error
|
||||
print("update_type:", update_type)
|
||||
print("mm_count:", mm_count)
|
||||
print("mm_prompt_updates:", mm_prompt_updates)
|
||||
print("new_prompt:", new_prompt)
|
||||
print("result:", result)
|
||||
|
||||
# Manually constructed results
|
||||
assert new_prompt == expected
|
||||
|
||||
|
||||
FIND_UPDATE_TOKENS_TEST_CASES = [
|
||||
# Tokenized test cases of `test_find_update_text`
|
||||
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
|
||||
@@ -891,11 +583,7 @@ def test_find_update_tokens(
|
||||
for key, target in target_by_key.items()
|
||||
}
|
||||
|
||||
new_prompt, result = apply_token_matches(
|
||||
prompt,
|
||||
mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
new_prompt, result = apply_token_matches(prompt, mm_prompt_updates)
|
||||
|
||||
# Only displayed on error
|
||||
print("update_type:", update_type)
|
||||
@@ -945,7 +633,6 @@ def test_apply_token_matches_with_placeholders(
|
||||
new_prompt, result, placeholders = _apply_token_matches_with_placeholders(
|
||||
prompt,
|
||||
mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
|
||||
if any(
|
||||
@@ -1097,7 +784,7 @@ def test_find_mm_placeholders(
|
||||
for key, repl in repl_by_key.items()
|
||||
}
|
||||
|
||||
result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
|
||||
result = find_mm_placeholders(prompt, mm_prompt_updates)
|
||||
|
||||
# Only displayed on error
|
||||
print("result:", result)
|
||||
@@ -1285,24 +972,18 @@ def test_apply_matches_no_match_exits_quickly():
|
||||
With the fix, it should exit immediately when no match is found.
|
||||
"""
|
||||
# Create a long prompt with no placeholder
|
||||
long_prompt = "x" * 10000
|
||||
long_prompt = [1] * 10000
|
||||
|
||||
# Create update looking for a placeholder that doesn't exist
|
||||
mm_prompt_updates = {
|
||||
"image": [[PromptReplacement("image", "<image>", "REPLACED").resolve(0)]]
|
||||
}
|
||||
mm_prompt_updates = {"image": [[PromptReplacement("image", [0], [-1]).resolve(0)]]}
|
||||
|
||||
start = time.perf_counter()
|
||||
result, _ = _apply_matches(
|
||||
long_prompt,
|
||||
mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
result, _ = _apply_matches(long_prompt, mm_prompt_updates)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
# Should complete in < 100ms (was taking seconds before the fix)
|
||||
assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s"
|
||||
assert "".join(result) == long_prompt
|
||||
assert flatten_2d_lists(result) == long_prompt
|
||||
|
||||
|
||||
def test_apply_matches_many_shared_targets_scales_linearly():
|
||||
@@ -1317,11 +998,7 @@ def test_apply_matches_many_shared_targets_scales_linearly():
|
||||
prompt = [0] * item_count
|
||||
|
||||
start = time.perf_counter()
|
||||
result, match_result = apply_token_matches(
|
||||
prompt,
|
||||
mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
result, match_result = apply_token_matches(prompt, mm_prompt_updates)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert len(result) == item_count * len(replacement)
|
||||
@@ -1360,7 +1037,7 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
|
||||
}
|
||||
|
||||
start = time.perf_counter()
|
||||
result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
|
||||
result = find_mm_placeholders(prompt, mm_prompt_updates)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert result == {}
|
||||
@@ -1377,22 +1054,158 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
|
||||
[1, 2, 3, 4, 5],
|
||||
],
|
||||
)
|
||||
def test_find_mm_placeholders_resolves_content_lazily(prompt):
|
||||
def test_find_mm_placeholders_stops_at_missing_item(prompt):
|
||||
"""
|
||||
Test that content of items the scan never reaches is not resolved.
|
||||
|
||||
With `tokenizer=None`, resolving string content raises; the scan must
|
||||
return no placeholders instead of raising on the second item.
|
||||
Test that the scan returns no placeholders once it fails to find
|
||||
an item's placeholder, leaving later items unresolved.
|
||||
"""
|
||||
result = find_mm_placeholders(
|
||||
prompt,
|
||||
{
|
||||
"image": [
|
||||
[PromptReplacement("image", [0], [999]).resolve(0)],
|
||||
[PromptReplacement("image", [0], "never reached").resolve(1)],
|
||||
[PromptReplacement("image", [0], [998]).resolve(1)],
|
||||
]
|
||||
},
|
||||
tokenizer=None,
|
||||
)
|
||||
|
||||
assert result == {}
|
||||
|
||||
|
||||
class _FakeTokenizer:
|
||||
"""
|
||||
Character-level tokenizer where "foo" merges into one token differently
|
||||
depending on whether it is followed by "d", like BPE merging "foo" in
|
||||
"food" across the search-text boundary.
|
||||
"""
|
||||
|
||||
_MERGES = {"food": (1000,), "foo": (101, 111, 111)}
|
||||
_INVERSE = {ids: text for text, ids in _MERGES.items()}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = True) -> list[int]:
|
||||
token_ids = list[int]()
|
||||
pos = 0
|
||||
while pos < len(text):
|
||||
for length in (4, 3):
|
||||
word = text[pos : pos + length]
|
||||
if word in self._MERGES:
|
||||
token_ids.extend(self._MERGES[word])
|
||||
pos += length
|
||||
break
|
||||
else:
|
||||
token_ids.append(ord(text[pos]))
|
||||
pos += 1
|
||||
return token_ids
|
||||
|
||||
def decode(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
skip_special_tokens: bool = False,
|
||||
) -> str:
|
||||
chars = list[str]()
|
||||
pos = 0
|
||||
while pos < len(token_ids):
|
||||
for length in (3, 1):
|
||||
key = tuple(token_ids[pos : pos + length])
|
||||
if key in self._INVERSE:
|
||||
chars.append(self._INVERSE[key])
|
||||
pos += length
|
||||
break
|
||||
else:
|
||||
chars.append(chr(token_ids[pos]))
|
||||
pos += 1
|
||||
return "".join(chars)
|
||||
|
||||
|
||||
class _FakeProcessingInfo:
|
||||
def __init__(self, tokenizer) -> None:
|
||||
self._tokenizer = tokenizer
|
||||
|
||||
def get_tokenizer(self):
|
||||
return self._tokenizer
|
||||
|
||||
|
||||
class _TextFallbackProcessor(BaseMultiModalProcessor):
|
||||
"""Only `self.info.get_tokenizer()` is needed by the text fallback."""
|
||||
|
||||
def __init__(self, tokenizer: _FakeTokenizer) -> None:
|
||||
self.info = _FakeProcessingInfo(tokenizer)
|
||||
|
||||
def _get_mm_fields_config(self, hf_inputs, hf_processor_mm_kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_prompt_updates(self, mm_items, hf_processor_mm_kwargs, out_mm_kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _text_fallback_processor() -> BaseMultiModalProcessor:
|
||||
return _TextFallbackProcessor(_FakeTokenizer())
|
||||
|
||||
|
||||
def test_apply_prompt_updates_falls_back_to_text_matching():
|
||||
"""
|
||||
Test that the fallback in `_apply_prompt_updates` finds targets that
|
||||
tokenize differently inside the prompt ("foo" in "food").
|
||||
"""
|
||||
processor = _text_fallback_processor()
|
||||
|
||||
new_token_ids, placeholders = processor._apply_prompt_updates(
|
||||
[1000], # "food"
|
||||
{
|
||||
"image": [
|
||||
[PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)]
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
assert new_token_ids == [200, 201, ord("d")]
|
||||
assert [p.to_range().offset for p in placeholders["image"]] == [0]
|
||||
assert [p.tokens for p in placeholders["image"]] == [[200, 201]]
|
||||
|
||||
|
||||
def test_apply_prompt_updates_falls_back_with_prefix_target():
|
||||
"""
|
||||
Test that `PromptIndexTargets.prefix` targets are resolved against the
|
||||
decoded text in the fallback path of `_apply_prompt_updates`.
|
||||
"""
|
||||
processor = _text_fallback_processor()
|
||||
|
||||
new_token_ids, placeholders = processor._apply_prompt_updates(
|
||||
[1000], # "food"
|
||||
{
|
||||
"image": [
|
||||
[
|
||||
PromptInsertion(
|
||||
"image",
|
||||
PromptIndexTargets.prefix([101, 111, 111]),
|
||||
[9],
|
||||
).resolve(0)
|
||||
]
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
assert new_token_ids == [101, 111, 111, 9, ord("d")]
|
||||
assert [p.tokens for p in placeholders["image"]] == [[9]]
|
||||
|
||||
|
||||
def test_apply_prompt_updates_falls_back_with_index_targets():
|
||||
"""
|
||||
Test that the text resolvers of `PromptIndexTargets.start`/`end`
|
||||
match against the decoded text when another item forces the
|
||||
fallback in `_apply_prompt_updates`.
|
||||
"""
|
||||
processor = _text_fallback_processor()
|
||||
|
||||
new_token_ids, placeholders = processor._apply_prompt_updates(
|
||||
[1000], # "food"
|
||||
{
|
||||
"image": [
|
||||
[PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)],
|
||||
[PromptInsertion("image", PromptIndexTargets.end(), [9]).resolve(1)],
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
assert new_token_ids == [200, 201, ord("d"), 9]
|
||||
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
|
||||
|
||||
@@ -483,7 +483,7 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_audioflamingo3,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -277,10 +278,12 @@ class Cohere2VisionMultiModalProcessor(
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
img_tokens_per_tile = int(hf_processor.patch_size**2)
|
||||
img_line_break_token = hf_processor.img_line_break_token
|
||||
boi_token = hf_processor.boi_token
|
||||
eoi_token = hf_processor.eoi_token
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -295,12 +298,13 @@ class Cohere2VisionMultiModalProcessor(
|
||||
patch_tokens = image_token * img_tokens_per_tile + img_line_break_token
|
||||
repl = f"{boi_token}{patch_tokens * num_patches}{eoi_token}"
|
||||
|
||||
return PromptUpdateDetails.select_text(repl, image_token)
|
||||
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
|
||||
return PromptUpdateDetails.select_token_id(repl_ids, image_token_id)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -69,6 +69,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -1197,6 +1198,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
before_placeholder = {
|
||||
"image": "<|image@placeholder|>",
|
||||
@@ -1223,12 +1225,17 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
)
|
||||
else:
|
||||
num_tokens = int(grid_thw.prod()) // merge_length
|
||||
return after_placeholder[modality] * num_tokens
|
||||
placeholder_ids = cached_encode(
|
||||
tokenizer, after_placeholder[modality], add_special_tokens=False
|
||||
)
|
||||
return placeholder_ids * num_tokens
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=before_placeholder[modality],
|
||||
target=cached_encode(
|
||||
tokenizer, before_placeholder[modality], add_special_tokens=False
|
||||
),
|
||||
replacement=partial(get_replacement_ernie45vl, modality=modality),
|
||||
)
|
||||
for modality in ("image", "video")
|
||||
|
||||
@@ -753,7 +753,7 @@ class FunAudioChatMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_funaudiochat,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
replace_token_matches,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -165,7 +166,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
|
||||
image_height: int,
|
||||
processor: Gemma3Processor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
boi_token = processor.boi_token
|
||||
|
||||
num_crops = self.get_num_crops(
|
||||
@@ -189,8 +190,9 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
|
||||
tokenizer = processor.tokenizer
|
||||
vocab = tokenizer.get_vocab()
|
||||
image_token_id = vocab[tokenizer.image_token]
|
||||
repl_full_ids = cached_encode(tokenizer, repl_full, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(repl_full, image_token_id)
|
||||
return PromptUpdateDetails.select_token_id(repl_full_ids, image_token_id)
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
@@ -313,7 +315,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.boi_token
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
boi_token_id = tokenizer.get_vocab()[hf_processor.boi_token]
|
||||
|
||||
def get_replacement_gemma3(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -329,7 +332,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[boi_token_id],
|
||||
replacement=get_replacement_gemma3,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -51,6 +51,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
replace_token_matches,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -183,31 +184,37 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
processor: Gemma3nProcessor,
|
||||
) -> str:
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Get the replacement text for image tokens.
|
||||
Get the replacement metadata for image tokens.
|
||||
|
||||
For Gemma3n, this should return the full_image_sequence which includes
|
||||
BOI token, repeated image tokens, and EOI token.
|
||||
"""
|
||||
full_token_ids = cached_encode(
|
||||
processor.tokenizer, processor.full_image_sequence, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
processor.full_image_sequence, processor.image_token_id
|
||||
full_token_ids, processor.image_token_id
|
||||
)
|
||||
|
||||
def get_audio_repl(
|
||||
self,
|
||||
*,
|
||||
processor: Gemma3nProcessor,
|
||||
) -> str:
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Get the replacement text for audio tokens.
|
||||
Get the replacement metadata for audio tokens.
|
||||
|
||||
For Gemma3n, this should return the full_audio_sequence which includes
|
||||
BOA token, repeated audio tokens, and EOA token.
|
||||
"""
|
||||
# Return the full audio sequence as defined by the processor
|
||||
full_token_ids = cached_encode(
|
||||
processor.tokenizer, processor.full_audio_sequence, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
processor.full_audio_sequence, processor.audio_token_id
|
||||
full_token_ids, processor.audio_token_id
|
||||
)
|
||||
|
||||
|
||||
@@ -345,7 +352,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
|
||||
# Handle image tokens
|
||||
if "image" in mm_items:
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
|
||||
def get_replacement_image(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -359,14 +366,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_image,
|
||||
)
|
||||
)
|
||||
|
||||
# Handle audio tokens
|
||||
if "audio" in mm_items:
|
||||
audio_token = hf_processor.audio_token
|
||||
audio_token_id = hf_processor.audio_token_id
|
||||
|
||||
def get_replacement_audio(item_idx: int):
|
||||
return self.info.get_audio_repl(
|
||||
@@ -376,7 +383,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_audio,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -332,7 +332,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
image_height: int,
|
||||
processor: Gemma4Processor | None,
|
||||
max_soft_tokens: int | None = None,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Return the dynamic image token sequence for this image.
|
||||
|
||||
Computes the exact number of soft tokens the vision tower will
|
||||
@@ -386,7 +386,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
audio_len: int,
|
||||
processor: Gemma4Processor | None,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Return the dynamic audio token sequence for this audio.
|
||||
|
||||
Computes the number of soft tokens from the audio waveform
|
||||
@@ -414,7 +414,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
timestamps: list[float],
|
||||
num_soft_tokens_per_frame: list[int],
|
||||
processor: Gemma4Processor,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Build the full token replacement for one video.
|
||||
|
||||
Produces the same interleaved sequence as the HF Gemma4Processor:
|
||||
@@ -818,7 +818,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
# one image_token exists per image in the token stream.
|
||||
# The replacement expands it to the full image sequence
|
||||
# (boi + N×image_token + eoi, where N = max_soft_tokens).
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
|
||||
def get_replacement_image(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -848,13 +848,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_image,
|
||||
)
|
||||
)
|
||||
|
||||
if "video" in mm_items:
|
||||
video_token = hf_processor.video_token
|
||||
video_token_id = hf_processor.video_token_id
|
||||
|
||||
def get_replacement_video(item_idx: int):
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
@@ -869,13 +869,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=get_replacement_video,
|
||||
)
|
||||
)
|
||||
|
||||
if "audio" in mm_items:
|
||||
audio_token = hf_processor.audio_token
|
||||
audio_token_id = hf_processor.audio_token_id
|
||||
|
||||
def get_replacement_audio(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
@@ -888,7 +888,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_audio,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -94,6 +94,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import get_processor_cls_name_from_config
|
||||
@@ -1694,6 +1695,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
merge_length = image_processor.merge_size**2
|
||||
|
||||
@@ -1722,12 +1725,18 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=hf_processor.image_token,
|
||||
target=cached_encode(
|
||||
tokenizer, hf_processor.image_token, add_special_tokens=False
|
||||
),
|
||||
replacement=get_image_replacement,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<|begin_of_video|><|video|><|end_of_video|>",
|
||||
target=[
|
||||
vocab["<|begin_of_video|>"],
|
||||
vocab["<|video|>"],
|
||||
vocab["<|end_of_video|>"],
|
||||
],
|
||||
replacement=get_video_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -896,7 +896,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_glmasr,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.multimodal.processing.processor import (
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
TimingContext,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
|
||||
|
||||
@@ -90,6 +91,8 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
||||
hf_processor: H2OVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -127,7 +130,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
||||
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement_internvl,
|
||||
)
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -315,7 +316,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token, _, _ = self.info._get_image_token(hf_processor)
|
||||
image_token_id = hf_processor.image_token_id
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -329,15 +331,18 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
image_repl,
|
||||
embed_text=image_token,
|
||||
image_repl_ids = cached_encode(
|
||||
tokenizer, image_repl, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
image_repl_ids,
|
||||
image_token_id,
|
||||
)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_idefics3,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_video_processor_from_config
|
||||
@@ -355,11 +356,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = hf_processor.tokenizer
|
||||
video_token_id = tokenizer.encode(
|
||||
hf_processor.video_token, add_special_tokens=False
|
||||
)
|
||||
assert len(video_token_id) == 1
|
||||
video_token_id = video_token_id[0]
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
video_token_id = vocab[hf_processor.video_token]
|
||||
|
||||
prompt_text = re.sub(
|
||||
hf_processor.image_token, "<image_placeholder>", prompt_text
|
||||
@@ -469,10 +468,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
img_context_token = hf_processor.image_token
|
||||
img_context_token_id = hf_processor.image_token_id
|
||||
start_image_token = hf_processor.start_image_token
|
||||
end_image_token = hf_processor.end_image_token
|
||||
video_token = hf_processor.video_token
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False)
|
||||
assert len(video_token_id) == 1
|
||||
video_token_id = video_token_id[0]
|
||||
|
||||
out_mm_data = out_mm_kwargs.get_data()
|
||||
if "video_num_patches" in out_mm_data:
|
||||
video_num_patches = out_mm_data["video_num_patches"]
|
||||
@@ -501,7 +506,12 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
|
||||
repl_features = img_context_token * feature_size
|
||||
repl_full = start_image_token + repl_features + end_image_token
|
||||
return PromptUpdateDetails.select_text(repl_full, img_context_token)
|
||||
repl_full_ids = cached_encode(
|
||||
tokenizer, repl_full, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
repl_full_ids, img_context_token_id
|
||||
)
|
||||
|
||||
def get_replacement_interns1_video(item_idx: int):
|
||||
num_patches = video_num_patches[item_idx]
|
||||
@@ -512,17 +522,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, video_token)
|
||||
repl_full_ids = cached_encode(
|
||||
tokenizer, repl_full, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(repl_full_ids, video_token_id)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=img_context_token,
|
||||
target=[img_context_token_id],
|
||||
replacement=get_replacement_interns1_image,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=get_replacement_interns1_video,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
@@ -264,6 +265,8 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
hf_processor: InternVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -298,7 +301,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement_internvl,
|
||||
)
|
||||
|
||||
@@ -498,6 +501,8 @@ class InternVLMultiModalProcessor(
|
||||
hf_processor: InternVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "video_num_patches" in out_mm_data:
|
||||
video_num_patches = out_mm_data["video_num_patches"]
|
||||
assert isinstance(video_num_patches, torch.Tensor)
|
||||
@@ -514,7 +519,7 @@ class InternVLMultiModalProcessor(
|
||||
|
||||
return PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
|
||||
replacement=get_video_replacement_internvl,
|
||||
)
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
@@ -450,6 +451,10 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_pad_token_ids = cached_encode(
|
||||
tokenizer, "<|image_pad|>", add_special_tokens=False
|
||||
)
|
||||
|
||||
pixel_shuffle_scale = getattr(image_processor, "pixel_shuffle_scale", 2)
|
||||
merge_length = pixel_shuffle_scale**2
|
||||
@@ -460,13 +465,13 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
|
||||
feature_size = int(grid_thw.prod()) // merge_length
|
||||
repl_full = "<|image_pad|>" * feature_size
|
||||
return PromptUpdateDetails.select_text(repl_full, "<|image_pad|>")
|
||||
repl_full = image_pad_token_ids * feature_size
|
||||
return PromptUpdateDetails.select_token_ids(repl_full, image_pad_token_ids)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement_isaac,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
@@ -541,6 +542,8 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement(idx: int) -> Sequence[int]:
|
||||
out_item = out_mm_kwargs["image"][idx]
|
||||
image_token_thw = out_item["image_token_thw"].data
|
||||
@@ -552,7 +555,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -448,7 +449,8 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptReplacement]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_image_replacement_lfm2vl(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -463,15 +465,18 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
processor=hf_processor,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
return PromptUpdateDetails.select_text(
|
||||
image_repl,
|
||||
embed_text=image_token,
|
||||
image_repl_ids = cached_encode(
|
||||
tokenizer, image_repl, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
image_repl_ids,
|
||||
image_token_id,
|
||||
)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_image_replacement_lfm2vl,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -662,7 +662,7 @@ class MiDashengLMMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_midashenglm,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -1112,7 +1112,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
embed_t = torch.tensor(is_embed_mask)
|
||||
return PromptUpdateDetails(
|
||||
full=full,
|
||||
is_embed=lambda _tok, _seq: embed_t,
|
||||
is_embed=lambda _seq: embed_t,
|
||||
)
|
||||
|
||||
def get_audio_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -502,7 +503,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
out_mm_kwargs=out_mm_kwargs,
|
||||
)
|
||||
|
||||
audio_placeholder = self.info.audio_pattern
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
audio_placeholder = cached_encode(
|
||||
tokenizer, self.info.audio_pattern, add_special_tokens=False
|
||||
)
|
||||
unk_token_ids = [vocab["<unk>"]]
|
||||
|
||||
def get_audio_replacement(item_idx: int):
|
||||
audios = mm_items.get_items(
|
||||
@@ -517,9 +524,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
else:
|
||||
audio_len = audios.get_audio_length(item_idx)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_audio_prompt_texts(audio_len),
|
||||
"<unk>",
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_audio_prompt_texts(audio_len),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
unk_token_ids,
|
||||
)
|
||||
|
||||
return [
|
||||
|
||||
@@ -88,7 +88,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
ResolvedPromptUpdate,
|
||||
_seq2text,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -938,30 +938,24 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Apply multi-modal prompt updates to token IDs."""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
new_token_ids, match_result = self._apply_token_matches(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
# If the search text does not represent a special token,
|
||||
# it may have different token IDs in the prompt, because
|
||||
# the tokens may go across the boundaries of the search text.
|
||||
# ----
|
||||
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||
# up a token, then the token ID of "foo" will not appear at all
|
||||
# ----
|
||||
# Since it is inefficient to search for all possible tokenizations
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
# If the target does not consist of special tokens, it may be
|
||||
# tokenized differently inside the prompt, so fall back to
|
||||
# performing the updates on the decoded text, then encoding the
|
||||
# result back. The segments are encoded separately to avoid BPE
|
||||
# merges across segment boundaries.
|
||||
if not all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
|
||||
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||
new_token_ids, match_result = self._apply_prompt_updates_via_text(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
encode_segments_separately=True,
|
||||
)
|
||||
|
||||
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
@@ -1062,6 +1056,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
additional_placeholders.append((modality, sub_pattern))
|
||||
placeholders += additional_placeholders
|
||||
|
||||
vocab = tokenizer.get_vocab()
|
||||
unk_token_ids = [vocab["<unk>"]]
|
||||
|
||||
def get_image_replacement(item_idx: int):
|
||||
images = mm_items.get_items(
|
||||
"image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
|
||||
@@ -1069,9 +1066,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
image_size = images.get_image_size(item_idx)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_image_prompt_texts(image_size, item_idx),
|
||||
"<unk>",
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_image_prompt_texts(image_size, item_idx),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
unk_token_ids,
|
||||
)
|
||||
|
||||
def get_video_replacement(item_idx: int):
|
||||
@@ -1082,9 +1083,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
frame_size = videos.get_frame_size(item_idx)
|
||||
num_frames = videos.get_num_frames(item_idx)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_video_prompt_texts(frame_size, num_frames),
|
||||
"<unk>",
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_video_prompt_texts(frame_size, num_frames),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
unk_token_ids,
|
||||
)
|
||||
|
||||
get_replacement = {
|
||||
@@ -1094,7 +1099,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality, target=pattern, replacement=get_replacement[modality]
|
||||
modality=modality,
|
||||
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
|
||||
replacement=get_replacement[modality],
|
||||
)
|
||||
for modality, pattern in placeholders
|
||||
]
|
||||
@@ -1114,7 +1121,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
image_processor = self.info.get_image_processor()
|
||||
version = self.info.get_model_version()
|
||||
|
||||
text = _seq2text(tokenizer, cached_update.content.full)
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
|
||||
if version == (2, 0) or version == (2, 5):
|
||||
@@ -1130,13 +1137,17 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
embed_text = getattr(tokenizer, "image_token", "<unk>")
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_text(
|
||||
text.replace(
|
||||
f"{im_start}{prev_item_idx}{im_end}",
|
||||
f"{im_start}{new_item_idx}{im_end}",
|
||||
1,
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{im_start}{prev_item_idx}{im_end}",
|
||||
f"{im_start}{new_item_idx}{im_end}",
|
||||
1,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
embed_text,
|
||||
cached_encode(tokenizer, embed_text, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
ResolvedPromptUpdate,
|
||||
_seq2text,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -338,6 +338,9 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
# than `<unk>`, so use those tokens as the embedding selector.
|
||||
image_embed_text = getattr(tokenizer, "image_token", "<|image_pad|>")
|
||||
video_embed_text = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
vocab = tokenizer.get_vocab()
|
||||
image_embed_ids = [vocab[image_embed_text]]
|
||||
video_embed_ids = [vocab[video_embed_text]]
|
||||
|
||||
def get_image_replacement(item_idx: int):
|
||||
images = mm_items.get_items(
|
||||
@@ -345,13 +348,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
(MiniCPMVImageEmbeddingItems, ImageProcessorItems),
|
||||
)
|
||||
image_size = images.get_image_size(item_idx)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
image_embed_text,
|
||||
image_embed_ids,
|
||||
)
|
||||
|
||||
def get_video_replacement(item_idx: int):
|
||||
@@ -370,14 +377,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
width=int(image_sizes[0, 0].item()),
|
||||
height=int(image_sizes[0, 1].item()),
|
||||
)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
video_embed_text,
|
||||
video_embed_ids,
|
||||
)
|
||||
|
||||
videos = mm_items.get_items(
|
||||
@@ -386,14 +397,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
)
|
||||
frame_size = videos.get_frame_size(item_idx)
|
||||
num_frames = videos.get_num_frames(item_idx)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
video_embed_text,
|
||||
video_embed_ids,
|
||||
)
|
||||
|
||||
get_replacement = {
|
||||
@@ -404,7 +419,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=pattern,
|
||||
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
|
||||
replacement=get_replacement[modality],
|
||||
)
|
||||
for modality, pattern in placeholders
|
||||
@@ -424,17 +439,21 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = _seq2text(tokenizer, cached_update.content.full)
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_text(
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
video_token,
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -666,9 +667,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
|
||||
|
||||
num_patches_per_chunk = self.info.get_patch_per_chunk(vision_config)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.image_token
|
||||
img_patch_token = hf_processor.img_patch_token
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
img_patch_token_ids = cached_encode(
|
||||
tokenizer, img_patch_token, add_special_tokens=False
|
||||
)
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
aspect_ratio = out_item["aspect_ratios"].data
|
||||
@@ -678,12 +683,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
|
||||
num_patches_per_chunk=num_patches_per_chunk,
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl, img_patch_token)
|
||||
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
|
||||
return PromptUpdateDetails.select_token_ids(repl_ids, img_patch_token_ids)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[hf_processor.image_token_id],
|
||||
replacement=get_replacement,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
|
||||
PromptInsertion,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -1263,7 +1264,9 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
return [
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.prefix("<|endoftext|>"),
|
||||
target=PromptIndexTargets.prefix(
|
||||
cached_encode(tokenizer, "<|endoftext|>", add_special_tokens=False),
|
||||
),
|
||||
insertion=get_insertion_molmo,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -1393,7 +1393,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
||||
def get_replacement(
|
||||
item_idx: int,
|
||||
suffix_token_ids: list[int] | None = None,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
num_tokens = audio_token_lens[item_idx]
|
||||
if num_tokens == 0:
|
||||
raise ValueError("The audio is too short to be represented.")
|
||||
@@ -1410,7 +1410,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
||||
processor.audio_end_id,
|
||||
*suffix_token_ids,
|
||||
],
|
||||
is_embed=lambda _tokenizer, _seq: torch.cat(
|
||||
is_embed=lambda _seq: torch.cat(
|
||||
[
|
||||
torch.tensor([False]),
|
||||
is_embed,
|
||||
|
||||
@@ -64,6 +64,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import ProcessorInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -535,7 +536,7 @@ class MossTranscribeDiarizeMultiModalProcessor(
|
||||
raise ValueError("Audio input is too short to produce any tokens.")
|
||||
return num_tokens
|
||||
|
||||
def get_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def get_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
num_tokens = get_num_tokens(item_idx)
|
||||
audio_tokens = processor._audio_span_ids(num_tokens)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
@@ -546,7 +547,9 @@ class MossTranscribeDiarizeMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=AUDIO_PLACEHOLDER,
|
||||
target=cached_encode(
|
||||
tokenizer, AUDIO_PLACEHOLDER, add_special_tokens=False
|
||||
),
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -407,7 +407,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
video_end_id = vocab["<|vid_end|>"]
|
||||
video_separator_id = vocab["<|vid_frame_separator|>"]
|
||||
|
||||
def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def image_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
num_tokens = int(out_item["image_feature_sizes"].data)
|
||||
replacement = (
|
||||
@@ -417,7 +417,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
replacement, config.image_token_id
|
||||
)
|
||||
|
||||
def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def video_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
pixel_values = out_item["video_pixel_values"].data
|
||||
num_groups = len(pixel_values)
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.multimodal.processing.processor import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.video_prune.evs import (
|
||||
compute_retained_tokens_count,
|
||||
@@ -429,6 +430,9 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
hf_processor: NanoNemotronVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -470,7 +474,7 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=[vocab["<image>"]],
|
||||
replacement=get_image_replacement,
|
||||
)
|
||||
|
||||
@@ -480,6 +484,8 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
hf_processor: NanoNemotronVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "video_num_patches" in out_mm_data:
|
||||
video_num_patches = out_mm_data["video_num_patches"]
|
||||
assert isinstance(video_num_patches, torch.Tensor)
|
||||
@@ -546,7 +552,7 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
|
||||
return PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
|
||||
replacement=get_video_replacement,
|
||||
)
|
||||
|
||||
@@ -556,13 +562,15 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
hf_processor: NanoNemotronVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_audio_replacement(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
return hf_processor.get_audio_repl(audios.get(item_idx))
|
||||
|
||||
return PromptReplacement(
|
||||
modality="audio",
|
||||
target=AUDIO_CONTEXT,
|
||||
target=cached_encode(tokenizer, AUDIO_CONTEXT, add_special_tokens=False),
|
||||
replacement=get_audio_replacement,
|
||||
)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.multimodal.parse import (
|
||||
from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
|
||||
from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor
|
||||
@@ -106,6 +107,8 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
|
||||
hf_processor: NVLMProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -138,14 +141,18 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
|
||||
|
||||
repl = hf_processor.get_image_repl(num_patches, num_features=feature_size)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
repl.full + "\n", hf_processor.ctx_image_token
|
||||
# `repl.full` ends in the special `</Image>` token, so there is
|
||||
# no BPE merge across the boundary with the newline
|
||||
newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
repl.full + newline_ids, hf_processor.ctx_image_token_id
|
||||
)
|
||||
|
||||
# See note in dummy data regarding why we have the extra newline
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>\n",
|
||||
target=cached_encode(tokenizer, "<image>\n", add_special_tokens=False),
|
||||
replacement=get_replacement_nvlm,
|
||||
)
|
||||
|
||||
|
||||
@@ -339,7 +339,7 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
bos_token_id = tokenizer.bos_token_id
|
||||
|
||||
def get_insertion(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def get_insertion(item_idx: int) -> PromptUpdateDetails:
|
||||
images = mm_items.get_items(
|
||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||
)
|
||||
|
||||
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -383,6 +384,8 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> list[PromptReplacement]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement_ovis(item_idx: int):
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
grid = out_item["grids"].data
|
||||
@@ -393,7 +396,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=IMAGE_TOKEN,
|
||||
target=cached_encode(tokenizer, IMAGE_TOKEN, add_special_tokens=False),
|
||||
replacement=get_replacement_ovis,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
ResolvedPromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -436,8 +437,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
|
||||
|
||||
def get_image_token_ids(item_idx: int) -> list[int]:
|
||||
return cached_encode(
|
||||
tokenizer, image_tokens[item_idx], add_special_tokens=False
|
||||
)
|
||||
|
||||
def get_replacement_phi3v(item_idx: int):
|
||||
images = mm_items.get_items(
|
||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||
@@ -458,7 +465,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_tokens.__getitem__,
|
||||
target=get_image_token_ids,
|
||||
replacement=get_replacement_phi3v,
|
||||
)
|
||||
]
|
||||
@@ -475,8 +482,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
|
||||
if cached_update.modality == "image":
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
|
||||
new_update = new_update.with_target(image_tokens[new_item_idx])
|
||||
new_update = new_update.with_target(
|
||||
cached_encode(
|
||||
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
|
||||
)
|
||||
)
|
||||
|
||||
return new_update
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
ResolvedPromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -934,8 +935,20 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_tokens: list[str] = self.info.image_tokens # type: ignore
|
||||
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
|
||||
|
||||
def get_image_token_ids(item_idx: int) -> list[int]:
|
||||
return cached_encode(
|
||||
tokenizer, image_tokens[item_idx], add_special_tokens=False
|
||||
)
|
||||
|
||||
def get_audio_token_ids(item_idx: int) -> list[int]:
|
||||
return cached_encode(
|
||||
tokenizer, audio_tokens[item_idx], add_special_tokens=False
|
||||
)
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
@@ -970,12 +983,12 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_tokens.__getitem__,
|
||||
target=get_image_token_ids,
|
||||
replacement=get_image_replacement_phi4mm,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_tokens.__getitem__,
|
||||
target=get_audio_token_ids,
|
||||
replacement=get_audio_replacement_phi4mm,
|
||||
),
|
||||
]
|
||||
@@ -990,12 +1003,22 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
new_item_idx,
|
||||
)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if cached_update.modality == "image":
|
||||
image_tokens: list[str] = self.info.image_tokens # type: ignore
|
||||
new_update = new_update.with_target(image_tokens[new_item_idx])
|
||||
new_update = new_update.with_target(
|
||||
cached_encode(
|
||||
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
|
||||
)
|
||||
)
|
||||
elif cached_update.modality == "audio":
|
||||
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
|
||||
new_update = new_update.with_target(audio_tokens[new_item_idx])
|
||||
new_update = new_update.with_target(
|
||||
cached_encode(
|
||||
tokenizer, audio_tokens[new_item_idx], add_special_tokens=False
|
||||
)
|
||||
)
|
||||
|
||||
return new_update
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -193,6 +194,8 @@ class Phi4SiglipMultiModalProcessor(
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
# Read the actual patch grid from the NaFlex processor's
|
||||
# spatial_shapes output (same pattern as LFM2-VL). This avoids
|
||||
@@ -207,7 +210,9 @@ class Phi4SiglipMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=DEFAULT_IMAGE_TOKEN,
|
||||
target=cached_encode(
|
||||
tokenizer, DEFAULT_IMAGE_TOKEN, add_special_tokens=False
|
||||
),
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -301,7 +301,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="", # Never match the prompt (see below note)
|
||||
target=[], # Never match the prompt (see below note)
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -779,17 +779,17 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_qwen2_audio,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=partial(get_replacement_qwen2_vision, modality="image"),
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=video_replacement_fn,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -328,7 +328,7 @@ class Qwen3ASRMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_qwen2_audio,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -1493,17 +1493,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_qwen2_audio,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=partial(get_replacement_qwen2_vision, modality="image"),
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=video_replacement_fn,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -1555,17 +1555,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
|
||||
if self._expands_only_video_token(hf_processor):
|
||||
# transformers>=5.10 expands only the bare video_token
|
||||
video_target = hf_processor.video_token
|
||||
video_target = [video_token_id]
|
||||
else:
|
||||
# Old-style processors expand the full placeholder
|
||||
# NOTE: We match string on purpose since searching sequence of
|
||||
# token ids takes more time.
|
||||
video_target = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
video_target = [
|
||||
vision_start_token_id,
|
||||
video_token_id,
|
||||
vision_end_token_id,
|
||||
]
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=hf_processor.image_token,
|
||||
target=[hf_processor.image_token_id],
|
||||
replacement=get_image_replacement_qwen3vl,
|
||||
),
|
||||
PromptReplacement(
|
||||
@@ -1585,7 +1587,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
vision_end_token_id: int,
|
||||
video_token_id: int,
|
||||
select_token_id: bool = False,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Build prompt replacement for a video in Qwen3VL format.
|
||||
|
||||
The replacement structure for each frame is:
|
||||
@@ -1609,7 +1611,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
|
||||
# Tokenize timestamp strings independently to avoid tokenizer merging
|
||||
# tokens across boundaries.
|
||||
# TODO: switch to `_seq2tokens` which has some caching.
|
||||
timestamp_token_ids = [
|
||||
tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)
|
||||
for timestamp in timestamps
|
||||
|
||||
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
TimingContext,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -700,6 +701,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Replace each modality's placeholder token with the token ids that item's
|
||||
replacement text encodes to, marking which of them hold embeddings."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
|
||||
if isinstance(token, str):
|
||||
return cached_encode(tokenizer, token, add_special_tokens=False)
|
||||
if isinstance(token, int):
|
||||
return [token]
|
||||
return list(token)
|
||||
|
||||
updates = []
|
||||
for modality, items in out_mm_kwargs.items():
|
||||
# Popped so they are neither cached nor sent to the model; the updates
|
||||
@@ -711,10 +721,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
token = getattr(hf_processor, f"{modality}_token")
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=getattr(hf_processor, f"{modality}_token"),
|
||||
target=get_target_token_ids(token),
|
||||
replacement=replacements.__getitem__,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -298,6 +299,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
replacement_id = hf_processor.audio_replacement_token_id # type: ignore
|
||||
|
||||
@@ -322,7 +324,15 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target="<|audio|>",
|
||||
# `<|audio|>` is paired with `replacement_id` in
|
||||
# `UltravoxProcessingInfo.get_hf_processor`, but it is not
|
||||
# guaranteed to be a single vocab entry, so match the encoded
|
||||
# placeholder text instead of `replacement_id`
|
||||
target=cached_encode(
|
||||
tokenizer,
|
||||
hf_processor.audio_token_replacement, # type: ignore
|
||||
add_special_tokens=False,
|
||||
),
|
||||
replacement=get_replacement_ultravox,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -310,7 +310,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target="", # Never match the prompt (see below note)
|
||||
target=[], # Never match the prompt (see below note)
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -722,7 +722,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
||||
image_end_id = vocab[IMAGE_END]
|
||||
merge_size = self.info.image_processor.merge_size # type: ignore[union-attr]
|
||||
|
||||
def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def image_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
grid = out_mm_kwargs["image"][item_idx]["image_grid_thw"].data
|
||||
assert isinstance(grid, torch.Tensor)
|
||||
num_tokens = int(grid.prod()) // merge_size**2
|
||||
@@ -750,7 +750,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
||||
* int(config.get("merge_factor", 1))
|
||||
)
|
||||
|
||||
def audio_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def audio_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
length = out_mm_kwargs["audio"][item_idx]["audio_lengths"].data
|
||||
assert isinstance(length, torch.Tensor)
|
||||
num_tokens = math.ceil(int(length.item()) / stride)
|
||||
@@ -768,7 +768,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
||||
)
|
||||
if "video" in out_mm_kwargs:
|
||||
|
||||
def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def video_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
item = out_mm_kwargs["video"][item_idx]
|
||||
input_ids_data = item["video_input_ids"].data
|
||||
embed_mask_data = item["video_embed_mask"].data
|
||||
@@ -777,8 +777,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
||||
input_ids = input_ids_data.tolist()
|
||||
embed_mask = embed_mask_data.bool()
|
||||
|
||||
def select_video_embeds(tokenizer, full) -> torch.Tensor:
|
||||
del tokenizer, full
|
||||
def select_video_embeds(full) -> torch.Tensor:
|
||||
del full
|
||||
return embed_mask
|
||||
|
||||
return PromptUpdateDetails(
|
||||
@@ -789,7 +789,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=VIDEO_PLACEHOLDER,
|
||||
target=[vocab[VIDEO_PLACEHOLDER]],
|
||||
replacement=video_replacement,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||
from vllm.transformers_utils.processor import cached_get_image_processor
|
||||
@@ -268,8 +269,9 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
|
||||
media_token_id = self.info.media_token_id
|
||||
media_token = self.info.media_token
|
||||
image_placeholder = self.info.get_hf_config().image_placeholder
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement(item_idx: int) -> PromptUpdateDetails[str]:
|
||||
def get_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image = images.get(item_idx)
|
||||
if image is None:
|
||||
@@ -296,12 +298,17 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
|
||||
f"{pads}<|media_end|>"
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full, media_token_id)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
cached_encode(tokenizer, full, add_special_tokens=False),
|
||||
media_token_id,
|
||||
)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_placeholder,
|
||||
target=cached_encode(
|
||||
tokenizer, image_placeholder, add_special_tokens=False
|
||||
),
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -11,6 +11,7 @@ from .processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -20,6 +21,7 @@ __all__ = [
|
||||
"BaseDummyInputsBuilder",
|
||||
"ProcessorInputs",
|
||||
"BaseMultiModalProcessor",
|
||||
"cached_encode",
|
||||
"EncDecMultiModalProcessor",
|
||||
"PromptUpdate",
|
||||
"PromptIndexTargets",
|
||||
|
||||
@@ -12,10 +12,8 @@ from typing import (
|
||||
NamedTuple,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
cast,
|
||||
)
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
@@ -53,74 +51,31 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_S = TypeVar("_S", str, list[int])
|
||||
|
||||
|
||||
PromptSeq: TypeAlias = str | list[int]
|
||||
"""A token sequence (list of token IDs) or text."""
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
def _cached_encode(
|
||||
def cached_encode(
|
||||
tokenizer: TokenizerLike,
|
||||
text: str,
|
||||
*,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[int]:
|
||||
"""Encode text while caching repeated tokenizer calls."""
|
||||
return tokenizer.encode(text, add_special_tokens=add_special_tokens)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
def _cached_decode(
|
||||
tokenizer: TokenizerLike,
|
||||
token_ids: tuple[int, ...],
|
||||
*,
|
||||
skip_special_tokens: bool = False,
|
||||
) -> str:
|
||||
return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
|
||||
|
||||
|
||||
def _seq2text(
|
||||
tokenizer: TokenizerLike | None,
|
||||
seq: PromptSeq,
|
||||
*,
|
||||
use_cache: bool = True,
|
||||
) -> str:
|
||||
if isinstance(seq, str):
|
||||
return seq
|
||||
|
||||
if tokenizer is None:
|
||||
raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")
|
||||
|
||||
if not use_cache:
|
||||
return tokenizer.decode(seq)
|
||||
|
||||
return _cached_decode(tokenizer, tuple(seq))
|
||||
|
||||
|
||||
def _seq2tokens(
|
||||
tokenizer: TokenizerLike | None,
|
||||
seq: PromptSeq,
|
||||
*,
|
||||
use_cache: bool = True,
|
||||
) -> list[int]:
|
||||
if isinstance(seq, str):
|
||||
if tokenizer is None:
|
||||
raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")
|
||||
|
||||
if not use_cache:
|
||||
return tokenizer.encode(seq, add_special_tokens=False)
|
||||
|
||||
return _cached_encode(tokenizer, seq, add_special_tokens=False)
|
||||
|
||||
return seq
|
||||
|
||||
|
||||
class _GetMatchIndex(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
tokenizer: TokenizerLike | None,
|
||||
prompt: PromptSeq,
|
||||
prompt: list[int],
|
||||
start_idx: int = 0,
|
||||
) -> int | None: ...
|
||||
|
||||
|
||||
class _GetTextMatchIndex(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
prompt: str,
|
||||
start_idx: int = 0,
|
||||
) -> int | None: ...
|
||||
|
||||
@@ -131,6 +86,14 @@ class PromptIndex:
|
||||
|
||||
get_match_index: _GetMatchIndex
|
||||
|
||||
get_text_match_index: _GetTextMatchIndex
|
||||
"""
|
||||
Resolves the index against a decoded text prompt.
|
||||
|
||||
Used only by the private text-based fallback of
|
||||
[`BaseMultiModalProcessor._apply_prompt_updates`][vllm.multimodal.processing.BaseMultiModalProcessor].
|
||||
"""
|
||||
|
||||
|
||||
class PromptIndexTargets:
|
||||
@staticmethod
|
||||
@@ -140,35 +103,40 @@ class PromptIndexTargets:
|
||||
|
||||
This results in a match even if the prompt is empty.
|
||||
"""
|
||||
return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
|
||||
return PromptIndex(
|
||||
lambda prompt, start_idx=0: 0,
|
||||
lambda tokenizer, prompt, start_idx=0: 0,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def prefix(seq: PromptSeq) -> PromptIndex:
|
||||
def prefix(seq: list[int]) -> PromptIndex:
|
||||
"""
|
||||
Resolves to a location in the prompt after the given prefix.
|
||||
"""
|
||||
|
||||
def get_match_index(
|
||||
tokenizer: TokenizerLike | None,
|
||||
prompt: PromptSeq,
|
||||
prompt: list[int],
|
||||
start_idx: int = 0,
|
||||
) -> int | None:
|
||||
if start_idx != 0:
|
||||
return None
|
||||
|
||||
prefix = seq
|
||||
match_idx = len(seq)
|
||||
return match_idx if prompt[:match_idx] == seq else None
|
||||
|
||||
if isinstance(prompt, str):
|
||||
# Make both `str`
|
||||
prefix = _seq2text(tokenizer, prefix, use_cache=False)
|
||||
else:
|
||||
# Make both `list[int]`
|
||||
prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
|
||||
def get_text_match_index(
|
||||
tokenizer: TokenizerLike,
|
||||
prompt: str,
|
||||
start_idx: int = 0,
|
||||
) -> int | None:
|
||||
if start_idx != 0:
|
||||
return None
|
||||
|
||||
prefix = tokenizer.decode(seq)
|
||||
match_idx = len(prefix)
|
||||
return match_idx if prompt[:match_idx] == prefix else None
|
||||
|
||||
return PromptIndex(get_match_index)
|
||||
return PromptIndex(get_match_index, get_text_match_index)
|
||||
|
||||
@staticmethod
|
||||
def end() -> PromptIndex:
|
||||
@@ -177,33 +145,36 @@ class PromptIndexTargets:
|
||||
|
||||
This results in a match even if the prompt is empty.
|
||||
"""
|
||||
return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
|
||||
return PromptIndex(
|
||||
lambda prompt, start_idx=0: len(prompt),
|
||||
lambda tokenizer, prompt, start_idx=0: len(prompt),
|
||||
)
|
||||
|
||||
|
||||
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
|
||||
UpdateTarget: TypeAlias = list[int] | PromptIndex
|
||||
"""
|
||||
The token sequence or text to update.
|
||||
The token sequence to update.
|
||||
"""
|
||||
|
||||
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the corresponding token sequence (or text).
|
||||
output the corresponding token sequence.
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromptUpdateDetails(Generic[_S]):
|
||||
"""Details about the token sequence or text that are part of the update."""
|
||||
class PromptUpdateDetails:
|
||||
"""Details about the token sequence that is part of the update."""
|
||||
|
||||
full: _S
|
||||
full: list[int]
|
||||
"""The full content."""
|
||||
|
||||
is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
|
||||
is_embed: Callable[[list[int]], torch.Tensor] | None = None
|
||||
"""
|
||||
Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
|
||||
return a boolean mask of shape `(len(full),)` indicating which positions
|
||||
@@ -216,56 +187,36 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
|
||||
def from_seq(seq: list[int]) -> "PromptUpdateDetails":
|
||||
return PromptUpdateDetails(full=seq)
|
||||
|
||||
@staticmethod
|
||||
def select_text(
|
||||
seq: _S,
|
||||
embed_text: str,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
|
||||
embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
return torch.isin(
|
||||
torch.tensor(token_ids),
|
||||
torch.tensor(embed_token_ids),
|
||||
)
|
||||
|
||||
return PromptUpdateDetails(full=seq, is_embed=is_embed)
|
||||
|
||||
@staticmethod
|
||||
def select_token_id(
|
||||
seq: _S,
|
||||
seq: list[int],
|
||||
embed_token_id: int,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
return torch.tensor(token_ids) == embed_token_id
|
||||
) -> "PromptUpdateDetails":
|
||||
def is_embed(full: list[int]) -> torch.Tensor:
|
||||
return torch.tensor(full) == embed_token_id
|
||||
|
||||
return PromptUpdateDetails(full=seq, is_embed=is_embed)
|
||||
|
||||
@staticmethod
|
||||
def select_token_ids(
|
||||
seq: _S,
|
||||
seq: list[int],
|
||||
embed_token_ids: list[int],
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
) -> "PromptUpdateDetails":
|
||||
def is_embed(full: list[int]) -> torch.Tensor:
|
||||
return torch.isin(
|
||||
torch.tensor(token_ids),
|
||||
torch.tensor(full),
|
||||
torch.tensor(embed_token_ids),
|
||||
)
|
||||
|
||||
return PromptUpdateDetails(full=seq, is_embed=is_embed)
|
||||
|
||||
|
||||
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
|
||||
PromptUpdateInfo: TypeAlias = list[int] | PromptUpdateDetails
|
||||
"""
|
||||
The token sequence or text that are part of the update.
|
||||
The token sequence that is part of the update.
|
||||
|
||||
If only part of the content corresponds to feature placeholders, you can
|
||||
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
|
||||
@@ -276,9 +227,9 @@ PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdat
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the corresponding token sequence (or text).
|
||||
output the corresponding token sequence.
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
@@ -292,13 +243,20 @@ class UpdateMode(str, Enum):
|
||||
class PromptUpdate(ABC):
|
||||
"""
|
||||
Defines how to update a prompt with placeholder tokens.
|
||||
|
||||
Note:
|
||||
The target and content are token sequences. When converting text
|
||||
to token sequences, remember to encode it with
|
||||
`add_special_tokens=False` to match how the prompt itself is
|
||||
tokenized. Otherwise, the updated prompt may contain duplicated
|
||||
special tokens or fail to match the target.
|
||||
"""
|
||||
|
||||
modality: str
|
||||
"""The modality for which the update is made."""
|
||||
|
||||
target: PromptUpdateTarget
|
||||
"""The token sequence (or text) to update."""
|
||||
"""The token sequence to update."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
@@ -357,8 +315,8 @@ class PromptInsertion(PromptUpdate):
|
||||
```python
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target="<s>",
|
||||
insertion="<image>" * image_feature_size,
|
||||
target=[bos_token_id],
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -368,7 +326,7 @@ class PromptInsertion(PromptUpdate):
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.start(),
|
||||
insertion="<image>" * image_feature_size,
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -377,8 +335,8 @@ class PromptInsertion(PromptUpdate):
|
||||
```python
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.prefix("Images:"),
|
||||
insertion="<image>" * image_feature_size,
|
||||
target=PromptIndexTargets.prefix(images_prefix_token_ids),
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -388,7 +346,7 @@ class PromptInsertion(PromptUpdate):
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.end(),
|
||||
insertion="<image>" * image_feature_size,
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
"""
|
||||
@@ -397,10 +355,10 @@ class PromptInsertion(PromptUpdate):
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the token sequence (or text) to insert right after
|
||||
output the token sequence to insert right after
|
||||
[`target`][vllm.multimodal.processing.PromptUpdate.target].
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
@@ -427,8 +385,8 @@ class PromptReplacement(PromptUpdate):
|
||||
```python
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
replacement="<image>" * image_feature_size,
|
||||
target=[image_token_id],
|
||||
replacement=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -436,26 +394,6 @@ class PromptReplacement(PromptUpdate):
|
||||
and `<image_eos>`, which are not supposed to be passed to the vision
|
||||
encoder:
|
||||
|
||||
```python
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
replacement=PromptUpdateDetails(
|
||||
full="".join(
|
||||
[
|
||||
"<image_bos>",
|
||||
"<image>" * image_feature_size,
|
||||
"<image_eos>",
|
||||
]
|
||||
),
|
||||
features="<image>" * image_feature_size,
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
To avoid unnecessary tokenization during prompt replacement,
|
||||
we recommended passing token sequences instead of text:
|
||||
|
||||
```python
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
@@ -464,7 +402,7 @@ class PromptReplacement(PromptUpdate):
|
||||
full=(
|
||||
[image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
|
||||
),
|
||||
features=[image_token_id] * image_feature_size,
|
||||
is_embed=lambda full: torch.tensor(full) == image_token_id,
|
||||
),
|
||||
)
|
||||
```
|
||||
@@ -474,10 +412,10 @@ class PromptReplacement(PromptUpdate):
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the token sequence (or text) to replace
|
||||
output the token sequence to replace
|
||||
[`target`][vllm.multimodal.processing.PromptUpdate.target].
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
@@ -520,7 +458,7 @@ class PromptTargetMatch(NamedTuple):
|
||||
class ResolvedPromptUpdate:
|
||||
"""
|
||||
A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
|
||||
lazy attributes resolved, apart from those related to tokenization.
|
||||
lazy attributes resolved.
|
||||
"""
|
||||
|
||||
modality: str
|
||||
@@ -533,7 +471,7 @@ class ResolvedPromptUpdate:
|
||||
"""Defines how to update the prompt."""
|
||||
|
||||
target: UpdateTarget
|
||||
"""The token sequence (or text) to update."""
|
||||
"""The token sequence to update."""
|
||||
|
||||
content: PromptUpdateDetails = field(repr=False)
|
||||
"""The placeholder tokens that are part of the update."""
|
||||
@@ -541,7 +479,6 @@ class ResolvedPromptUpdate:
|
||||
def iter_token_matches(
|
||||
self,
|
||||
prompt: list[int],
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
@@ -549,52 +486,15 @@ class ResolvedPromptUpdate:
|
||||
target = self.target
|
||||
|
||||
if isinstance(target, PromptIndex):
|
||||
match_idx = target.get_match_index(tokenizer, prompt, start_idx)
|
||||
match_idx = target.get_match_index(prompt, start_idx)
|
||||
if match_idx is not None:
|
||||
yield PromptTargetMatch(match_idx, match_idx)
|
||||
|
||||
return
|
||||
|
||||
target_token_ids = _seq2tokens(tokenizer, target)
|
||||
|
||||
for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
|
||||
for match in iter_token_matches(prompt, target, start_idx=start_idx):
|
||||
yield PromptTargetMatch(match.start_idx, match.end_idx)
|
||||
|
||||
def iter_text_matches(
|
||||
self,
|
||||
prompt: str,
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
"""Yield each instance of `self.target` found in `prompt`."""
|
||||
target = self.target
|
||||
|
||||
if isinstance(target, PromptIndex):
|
||||
match_idx = target.get_match_index(tokenizer, prompt, start_idx)
|
||||
if match_idx is not None:
|
||||
yield PromptTargetMatch(match_idx, match_idx)
|
||||
|
||||
return
|
||||
|
||||
target_text = _seq2text(tokenizer, target)
|
||||
|
||||
for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
|
||||
yield PromptTargetMatch(match.start(), match.end())
|
||||
|
||||
def iter_matches(
|
||||
self,
|
||||
prompt: list[int] | str,
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
"""Yield each instance of `self.target` found in `prompt`."""
|
||||
if isinstance(prompt, str):
|
||||
return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
def with_target(self, target: UpdateTarget):
|
||||
return replace(self, target=target)
|
||||
|
||||
@@ -729,8 +629,6 @@ def _target_key(target: UpdateTarget) -> tuple[str, object]:
|
||||
"""Return a hashable key that preserves target matching semantics."""
|
||||
if isinstance(target, PromptIndex):
|
||||
return ("index", id(target))
|
||||
if isinstance(target, str):
|
||||
return ("text", target)
|
||||
|
||||
return ("tokens", tuple(target))
|
||||
|
||||
@@ -761,10 +659,16 @@ def _compile_prompt_update_queues(
|
||||
return queues_by_signature
|
||||
|
||||
|
||||
_IterMatches: TypeAlias = Callable[
|
||||
[ResolvedPromptUpdate, int],
|
||||
Generator[PromptTargetMatch, None, None],
|
||||
]
|
||||
"""Yields each match of an update's target, given the search start index."""
|
||||
|
||||
|
||||
def _find_queue_match(
|
||||
queue: _UpdateQueue,
|
||||
prompt: _S,
|
||||
tokenizer: TokenizerLike | None,
|
||||
iter_matches: _IterMatches,
|
||||
*,
|
||||
start_idx: int,
|
||||
mode: UpdateMode | None = None,
|
||||
@@ -775,10 +679,7 @@ def _find_queue_match(
|
||||
if mode is not None and update.mode != mode:
|
||||
continue
|
||||
|
||||
match = next(
|
||||
update.iter_matches(prompt, tokenizer, start_idx=start_idx),
|
||||
None,
|
||||
)
|
||||
match = next(iter_matches(update, start_idx), None)
|
||||
if match is not None:
|
||||
return match, update_idx
|
||||
|
||||
@@ -791,10 +692,9 @@ def _next_priority(queue: _UpdateQueue) -> int:
|
||||
return priority
|
||||
|
||||
|
||||
def _plan_prompt_updates(
|
||||
prompt: _S,
|
||||
def _plan_prompt_updates_with(
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
iter_matches: _IterMatches,
|
||||
) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""Plan non-overlapping prompt updates before rendering the output."""
|
||||
queues = list(_compile_prompt_update_queues(mm_prompt_updates).values())
|
||||
@@ -809,8 +709,7 @@ def _plan_prompt_updates(
|
||||
for queue in queues:
|
||||
queue_match = _find_queue_match(
|
||||
queue,
|
||||
prompt,
|
||||
tokenizer,
|
||||
iter_matches,
|
||||
start_idx=prev_end_idx,
|
||||
)
|
||||
if queue_match is not None:
|
||||
@@ -835,8 +734,7 @@ def _plan_prompt_updates(
|
||||
|
||||
queue_match = _find_queue_match(
|
||||
queue,
|
||||
prompt,
|
||||
tokenizer,
|
||||
iter_matches,
|
||||
start_idx=prev_end_idx,
|
||||
mode=mode,
|
||||
)
|
||||
@@ -887,6 +785,19 @@ def _plan_prompt_updates(
|
||||
return planned_updates, result
|
||||
|
||||
|
||||
def _plan_prompt_updates(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""Plan non-overlapping prompt updates before rendering the output."""
|
||||
return _plan_prompt_updates_with(
|
||||
mm_prompt_updates,
|
||||
lambda update, start_idx: update.iter_token_matches(
|
||||
prompt, start_idx=start_idx
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _all_items_found(
|
||||
mm_item_counts: dict[str, int],
|
||||
mm_found_counts: dict[str, int],
|
||||
@@ -898,22 +809,16 @@ def _all_items_found(
|
||||
|
||||
|
||||
def _apply_matches(
|
||||
prompt: _S,
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
|
||||
out_seqs = list[str | list[int]]()
|
||||
matched_updates, result = _plan_prompt_updates(
|
||||
prompt,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
) -> tuple[list[list[int]], "MultiModalPromptUpdatesApplyResult"]:
|
||||
out_seqs = list[list[int]]()
|
||||
matched_updates, result = _plan_prompt_updates(prompt, mm_prompt_updates)
|
||||
|
||||
prev_end_idx = 0
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
matched_content = update.content.full
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
@@ -923,22 +828,17 @@ def _apply_matches(
|
||||
assert_never(update.mode)
|
||||
|
||||
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
|
||||
out_seqs.append(
|
||||
_seq2text(tokenizer, matched_content)
|
||||
if isinstance(prompt, str)
|
||||
else _seq2tokens(tokenizer, matched_content)
|
||||
)
|
||||
out_seqs.append(update.content.full)
|
||||
prev_end_idx = match.end_idx
|
||||
|
||||
out_seqs.append(prompt[prev_end_idx:])
|
||||
|
||||
return cast(list[_S], out_seqs), result
|
||||
return out_seqs, result
|
||||
|
||||
|
||||
def apply_token_matches(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""
|
||||
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
@@ -947,7 +847,7 @@ def apply_token_matches(
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
"""
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates)
|
||||
|
||||
return flatten_2d_lists(token_id_seqs), result
|
||||
|
||||
@@ -955,17 +855,12 @@ def apply_token_matches(
|
||||
def _apply_token_matches_with_placeholders(
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[
|
||||
list[int],
|
||||
"MultiModalPromptUpdatesApplyResult",
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
matched_updates, result = _plan_prompt_updates(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
|
||||
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||
modality: [] for modality in mm_prompt_updates
|
||||
}
|
||||
@@ -975,7 +870,6 @@ def _apply_token_matches_with_placeholders(
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
matched_content = update.content.full
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
@@ -987,11 +881,11 @@ def _apply_token_matches_with_placeholders(
|
||||
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
|
||||
start_idx = len(new_token_ids)
|
||||
|
||||
tokens = _seq2tokens(tokenizer, matched_content)
|
||||
tokens = update.content.full
|
||||
if tokens:
|
||||
content_is_embed = update.content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(tokenizer, matched_content)
|
||||
content_is_embed = content_is_embed(tokens)
|
||||
|
||||
placeholders[update.modality].append(
|
||||
PlaceholderFeaturesInfo(
|
||||
@@ -1010,50 +904,9 @@ def _apply_token_matches_with_placeholders(
|
||||
return new_token_ids, result, placeholders
|
||||
|
||||
|
||||
def apply_text_matches(
|
||||
prompt: str,
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""
|
||||
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
|
||||
Matches are exclusive even when multiple modalities share
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
"""
|
||||
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
|
||||
return "".join(texts), result
|
||||
|
||||
|
||||
def apply_text_matches_as_segmented_tokens(
|
||||
prompt: str,
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""
|
||||
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
|
||||
Matches are exclusive even when multiple modalities share
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
|
||||
Each segment is encoded separately instead of being joined into one
|
||||
string and encoded in a single pass. Joining first would let BPE merge
|
||||
tokens across a segment boundary, silently change how a text
|
||||
(non-special-token) placeholder is tokenized.
|
||||
"""
|
||||
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
|
||||
|
||||
return flatten_2d_lists(token_id_seqs), result
|
||||
|
||||
|
||||
def _iter_placeholders(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> Iterable[PlaceholderFeaturesInfo]:
|
||||
"""
|
||||
Yield each set of placeholder tokens found in `prompt`.
|
||||
@@ -1073,15 +926,15 @@ def _iter_placeholders(
|
||||
prompt_len = len(prompt)
|
||||
start_idx = 0
|
||||
|
||||
# The current (unfound) item's updates for each modality, with their
|
||||
# content resolved to token ids; rebuilt whenever an item is found.
|
||||
# Items are only resolved once the scan reaches them.
|
||||
# The current (unfound) item's updates for each modality; rebuilt
|
||||
# whenever an item is found. Items are only resolved once the scan
|
||||
# reaches them.
|
||||
candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None
|
||||
|
||||
while start_idx < prompt_len:
|
||||
if candidates is None:
|
||||
candidates = [
|
||||
(modality, update, _seq2tokens(tokenizer, update.content.full))
|
||||
(modality, update, update.content.full)
|
||||
for modality, modality_updates in mm_prompt_updates.items()
|
||||
if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0)
|
||||
for update in modality_updates[item_idx_by_modality[modality]]
|
||||
@@ -1107,7 +960,7 @@ def _iter_placeholders(
|
||||
content = update.content
|
||||
content_is_embed = content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(tokenizer, content.full)
|
||||
content_is_embed = content_is_embed(content.full)
|
||||
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
@@ -1134,9 +987,8 @@ def _iter_placeholders(
|
||||
def find_mm_placeholders(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
|
||||
it = _iter_placeholders(prompt, mm_prompt_updates)
|
||||
return dict(full_groupby_modality(it))
|
||||
|
||||
|
||||
@@ -1146,7 +998,7 @@ A collection of the `is_cached` flag for each item, with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
|
||||
MultiModalPromptUpdates = Mapping[str, Sequence[Sequence[ResolvedPromptUpdate]]]
|
||||
"""
|
||||
A collection of prompt updates with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
@@ -1283,9 +1135,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
new_token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates)
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
@@ -1606,8 +1456,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
return apply_token_matches(prompt, mm_prompt_updates)
|
||||
|
||||
def _apply_token_matches_with_placeholders(
|
||||
self,
|
||||
@@ -1618,31 +1467,105 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return _apply_token_matches_with_placeholders(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
|
||||
def _apply_text_matches(
|
||||
def _iter_text_matches(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
update: ResolvedPromptUpdate,
|
||||
prompt_text: str,
|
||||
tokenizer: TokenizerLike,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch, None, None]:
|
||||
"""Yield each instance of `update.target` found in `prompt_text`."""
|
||||
target = update.target
|
||||
|
||||
def _apply_text_matches_as_segmented_tokens(
|
||||
if isinstance(target, PromptIndex):
|
||||
match_idx = target.get_text_match_index(tokenizer, prompt_text, start_idx)
|
||||
if match_idx is not None:
|
||||
yield PromptTargetMatch(match_idx, match_idx)
|
||||
|
||||
return
|
||||
|
||||
target_text = tokenizer.decode(target)
|
||||
if not target_text:
|
||||
return
|
||||
|
||||
idx = prompt_text.find(target_text, start_idx)
|
||||
while idx != -1:
|
||||
end_idx = idx + len(target_text)
|
||||
yield PromptTargetMatch(idx, end_idx)
|
||||
idx = prompt_text.find(target_text, end_idx)
|
||||
|
||||
def _apply_prompt_updates_via_text(
|
||||
self,
|
||||
prompt: str,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
*,
|
||||
encode_segments_separately: bool = False,
|
||||
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||
"""
|
||||
Apply multi-modal prompt updates in text space, then encode the
|
||||
result.
|
||||
|
||||
A non-special-token target may be tokenized differently inside the
|
||||
prompt because the tokens may go across the boundaries of the target
|
||||
(e.g. when searching for "foo" in "food", if "food" itself makes up
|
||||
a token, then the token ID of "foo" will not appear at all). Since
|
||||
it is inefficient to search for all possible tokenizations of the
|
||||
target in the prompt, we instead perform text-based updates on the
|
||||
decoded token IDs, then encode them back.
|
||||
"""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return apply_text_matches_as_segmented_tokens(
|
||||
prompt, mm_prompt_updates, tokenizer
|
||||
prompt_text = tokenizer.decode(token_ids)
|
||||
|
||||
matched_updates, result = _plan_prompt_updates_with(
|
||||
mm_prompt_updates,
|
||||
lambda update, start_idx: self._iter_text_matches(
|
||||
update, prompt_text, tokenizer, start_idx=start_idx
|
||||
),
|
||||
)
|
||||
|
||||
out_texts = list[str]()
|
||||
prev_end_idx = 0
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
elif update.mode == UpdateMode.REPLACE:
|
||||
end_idx_to_insert = match.start_idx
|
||||
else:
|
||||
assert_never(update.mode)
|
||||
|
||||
out_texts.append(prompt_text[prev_end_idx:end_idx_to_insert])
|
||||
out_texts.append(tokenizer.decode(update.content.full))
|
||||
prev_end_idx = match.end_idx
|
||||
|
||||
out_texts.append(prompt_text[prev_end_idx:])
|
||||
|
||||
if encode_segments_separately:
|
||||
# Encode each segment separately instead of joining into one
|
||||
# text and encoding it in a single pass. Joining first would
|
||||
# let BPE merge tokens across a segment boundary, silently
|
||||
# changing how a non-special-token placeholder is tokenized.
|
||||
new_token_ids = flatten_2d_lists(
|
||||
[
|
||||
cached_encode(tokenizer, text, add_special_tokens=False)
|
||||
for text in out_texts
|
||||
]
|
||||
)
|
||||
else:
|
||||
new_token_ids = cached_encode(
|
||||
tokenizer, "".join(out_texts), add_special_tokens=False
|
||||
)
|
||||
|
||||
return new_token_ids, result
|
||||
|
||||
def _matched_updates_from_result(
|
||||
self,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
@@ -1668,8 +1591,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Apply multi-modal prompt updates to token IDs."""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
new_token_ids, match_result, placeholders = (
|
||||
self._apply_token_matches_with_placeholders(
|
||||
token_ids,
|
||||
@@ -1688,23 +1609,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
}
|
||||
return new_token_ids, placeholders
|
||||
|
||||
# If the search text does not represent a special token,
|
||||
# it may have different token IDs in the prompt, because
|
||||
# the tokens may go across the boundaries of the search text.
|
||||
# ----
|
||||
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||
# up a token, then the token ID of "foo" will not appear at all
|
||||
# ----
|
||||
# Since it is inefficient to search for all possible tokenizations
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
new_text, match_result = self._apply_text_matches(
|
||||
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||
# A non-special-token target may be tokenized differently inside
|
||||
# the prompt, so fall back to performing the updates on the
|
||||
# decoded text, then encoding the result back.
|
||||
new_token_ids, match_result = self._apply_prompt_updates_via_text(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
|
||||
|
||||
placeholders = self._find_mm_placeholders(
|
||||
new_token_ids,
|
||||
self._matched_updates_from_result(mm_prompt_updates, match_result),
|
||||
|
||||
@@ -157,7 +157,7 @@ def _expand_prompt_embeds_placeholders(
|
||||
`token_ids` is replaced with a consecutive span of
|
||||
`tensor.shape[0]` copies, following tensors in order.
|
||||
"""
|
||||
expanded, _ = apply_token_matches(token_ids, mm_prompt_updates, tokenizer=None)
|
||||
expanded, _ = apply_token_matches(token_ids, mm_prompt_updates)
|
||||
return expanded
|
||||
|
||||
|
||||
@@ -171,11 +171,7 @@ def _build_prompt_embeds_positions(
|
||||
Expects `token_ids` to already contain expanded N-token spans.
|
||||
Returns `[(start_idx, length), ...]` aligned with the tensors.
|
||||
"""
|
||||
placeholders = find_mm_placeholders(
|
||||
prompt=token_ids,
|
||||
mm_prompt_updates=mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
placeholders = find_mm_placeholders(token_ids, mm_prompt_updates)
|
||||
features = placeholders.get("prompt_embeds", [])
|
||||
|
||||
if len(features) != num_tensors:
|
||||
|
||||
@@ -21,7 +21,7 @@ from transformers import (
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.processing import PromptUpdateDetails
|
||||
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||
@@ -444,7 +444,7 @@ class InternVLProcessor(ProcessorMixin):
|
||||
self,
|
||||
num_patches: int | None,
|
||||
num_features: int | None = None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
if num_patches is None:
|
||||
assert num_features is not None
|
||||
else:
|
||||
@@ -453,9 +453,11 @@ class InternVLProcessor(ProcessorMixin):
|
||||
repl_features = self.ctx_image_token * num_features
|
||||
repl_full = self.start_image_token + repl_features + self.end_image_token
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, self.ctx_image_token)
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
|
||||
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails[str]:
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
|
||||
|
||||
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails:
|
||||
assert self.ctx_video_token is not None
|
||||
|
||||
repl_features = self.ctx_video_token * self.image_seq_length
|
||||
@@ -467,7 +469,10 @@ class InternVLProcessor(ProcessorMixin):
|
||||
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, self.ctx_video_token)
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
assert self.ctx_video_token_id is not None
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -523,7 +528,13 @@ class InternVLProcessor(ProcessorMixin):
|
||||
while image_token in new_prompt:
|
||||
new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
|
||||
image_repl = self.get_image_repl(image_num_patches[image_index])
|
||||
replace_strings.append(image_repl.full)
|
||||
# Convert the token IDs back to text for the
|
||||
# string-based replacement below
|
||||
replace_strings.append(
|
||||
self.tokenizer.decode(
|
||||
image_repl.full, skip_special_tokens=False
|
||||
)
|
||||
)
|
||||
image_index += 1
|
||||
|
||||
while "<placeholder>" in new_prompt:
|
||||
@@ -548,7 +559,13 @@ class InternVLProcessor(ProcessorMixin):
|
||||
while video_token in new_prompt:
|
||||
new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
|
||||
video_repl = self.get_video_repl(video_num_patches[video_index])
|
||||
replace_strings.append(video_repl.full)
|
||||
# Convert the token IDs back to text for the
|
||||
# string-based replacement below
|
||||
replace_strings.append(
|
||||
self.tokenizer.decode(
|
||||
video_repl.full, skip_special_tokens=False
|
||||
)
|
||||
)
|
||||
video_index += 1
|
||||
|
||||
while "<placeholder>" in new_prompt:
|
||||
|
||||
@@ -24,7 +24,7 @@ from transformers import BatchFeature, PretrainedConfig, TensorType
|
||||
|
||||
from vllm.model_executor.models.parakeet import ParakeetExtractor
|
||||
from vllm.multimodal.inputs import AudioItem
|
||||
from vllm.multimodal.processing.processor import PromptUpdateDetails
|
||||
from vllm.multimodal.processing.processor import PromptUpdateDetails, cached_encode
|
||||
from vllm.multimodal.video_prune.evs import compute_retained_tokens_count
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
@@ -633,7 +633,7 @@ class BaseNanoNemotronVLProcessor(ABC):
|
||||
self,
|
||||
feature_size: int,
|
||||
num_patches: int | None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_num_image_tokens(
|
||||
@@ -730,7 +730,12 @@ class BaseNanoNemotronVLProcessor(ABC):
|
||||
zip(num_tokens_per_image, image_num_patches, strict=True)
|
||||
):
|
||||
image_repl = self.get_image_repl(feature_size, num_patches)
|
||||
parts[i] = parts[i].replace("<image>", image_repl.full)
|
||||
# image_repl.full is a list of token IDs
|
||||
# Convert token IDs back to text for the HF processor flow
|
||||
image_repl_text = self.tokenizer.decode(
|
||||
image_repl.full, skip_special_tokens=False
|
||||
)
|
||||
parts[i] = parts[i].replace("<image>", image_repl_text)
|
||||
text = ["".join(parts)]
|
||||
|
||||
return text, image_inputs
|
||||
@@ -1001,7 +1006,11 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
for idx, part in enumerate(parts):
|
||||
if part == AUDIO_CONTEXT:
|
||||
audio_repl = self.get_audio_repl(audios[audio_index])
|
||||
parts[idx] = audio_repl.full
|
||||
# audio_repl.full is a list of token IDs
|
||||
# Convert token IDs back to text for the HF processor flow
|
||||
parts[idx] = self.tokenizer.decode(
|
||||
audio_repl.full, skip_special_tokens=False
|
||||
)
|
||||
audio_index += 1
|
||||
text = ["".join(parts)]
|
||||
audio_inputs = extractor(audios)
|
||||
@@ -1078,20 +1087,30 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
self,
|
||||
feature_size: int,
|
||||
num_patches: int | None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
repl_features = IMG_CONTEXT * feature_size
|
||||
repl_full = IMG_START + repl_features + IMG_END
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
full_ids, self._img_context_token_ids
|
||||
)
|
||||
|
||||
def get_audio_repl(
|
||||
self,
|
||||
audio: npt.NDArray,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
assert self.audio_extractor is not None
|
||||
num_tokens = self.audio_extractor.audio_token_count(len(audio))
|
||||
repl_full = f"{AUDIO_START}{AUDIO_CONTEXT * num_tokens}{AUDIO_END}"
|
||||
return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
|
||||
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
ctx_token_ids = cached_encode(
|
||||
self.tokenizer, AUDIO_CONTEXT, add_special_tokens=False
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_token_ids(full_ids, ctx_token_ids)
|
||||
|
||||
@classmethod
|
||||
def get_video_repl(
|
||||
@@ -1105,7 +1124,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
img_end_token_ids: list[int],
|
||||
img_context_token_ids: list[int],
|
||||
video_temporal_patch_size: int = 1,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Build prompt replacement for a video.
|
||||
The replacement returned is not actually used to replace the placeholder
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
# Copyright (c) 2024 NVIDIA
|
||||
# Licensed under Apache 2.0 License [see LICENSE for details]
|
||||
# --------------------------------------------------------
|
||||
from vllm.multimodal.processing import PromptUpdateDetails
|
||||
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
from .internvl import InternVLImageProcessor, InternVLProcessor
|
||||
@@ -37,7 +37,7 @@ class NVLMProcessor(InternVLProcessor):
|
||||
self,
|
||||
num_patches: int | None,
|
||||
num_features: int | None = None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
if num_patches is None:
|
||||
raise NotImplementedError("Embedding inputs are not supported")
|
||||
|
||||
@@ -58,4 +58,6 @@ class NVLMProcessor(InternVLProcessor):
|
||||
# when trying to find "<tile" as a subsequence of "<Image><tile"
|
||||
repl = self.start_image_token + features + self.end_image_token
|
||||
|
||||
return PromptUpdateDetails.select_text(repl, self.ctx_image_token)
|
||||
full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
|
||||
|
||||
Reference in New Issue
Block a user