[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
@@ -12,10 +12,8 @@ from typing import (
|
||||
NamedTuple,
|
||||
Protocol,
|
||||
TypeAlias,
|
||||
cast,
|
||||
)
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
@@ -53,74 +51,31 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_S = TypeVar("_S", str, list[int])
|
||||
|
||||
|
||||
PromptSeq: TypeAlias = str | list[int]
|
||||
"""A token sequence (list of token IDs) or text."""
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
def _cached_encode(
|
||||
def cached_encode(
|
||||
tokenizer: TokenizerLike,
|
||||
text: str,
|
||||
*,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[int]:
|
||||
"""Encode text while caching repeated tokenizer calls."""
|
||||
return tokenizer.encode(text, add_special_tokens=add_special_tokens)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
def _cached_decode(
|
||||
tokenizer: TokenizerLike,
|
||||
token_ids: tuple[int, ...],
|
||||
*,
|
||||
skip_special_tokens: bool = False,
|
||||
) -> str:
|
||||
return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
|
||||
|
||||
|
||||
def _seq2text(
|
||||
tokenizer: TokenizerLike | None,
|
||||
seq: PromptSeq,
|
||||
*,
|
||||
use_cache: bool = True,
|
||||
) -> str:
|
||||
if isinstance(seq, str):
|
||||
return seq
|
||||
|
||||
if tokenizer is None:
|
||||
raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")
|
||||
|
||||
if not use_cache:
|
||||
return tokenizer.decode(seq)
|
||||
|
||||
return _cached_decode(tokenizer, tuple(seq))
|
||||
|
||||
|
||||
def _seq2tokens(
|
||||
tokenizer: TokenizerLike | None,
|
||||
seq: PromptSeq,
|
||||
*,
|
||||
use_cache: bool = True,
|
||||
) -> list[int]:
|
||||
if isinstance(seq, str):
|
||||
if tokenizer is None:
|
||||
raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")
|
||||
|
||||
if not use_cache:
|
||||
return tokenizer.encode(seq, add_special_tokens=False)
|
||||
|
||||
return _cached_encode(tokenizer, seq, add_special_tokens=False)
|
||||
|
||||
return seq
|
||||
|
||||
|
||||
class _GetMatchIndex(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
tokenizer: TokenizerLike | None,
|
||||
prompt: PromptSeq,
|
||||
prompt: list[int],
|
||||
start_idx: int = 0,
|
||||
) -> int | None: ...
|
||||
|
||||
|
||||
class _GetTextMatchIndex(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
prompt: str,
|
||||
start_idx: int = 0,
|
||||
) -> int | None: ...
|
||||
|
||||
@@ -131,6 +86,14 @@ class PromptIndex:
|
||||
|
||||
get_match_index: _GetMatchIndex
|
||||
|
||||
get_text_match_index: _GetTextMatchIndex
|
||||
"""
|
||||
Resolves the index against a decoded text prompt.
|
||||
|
||||
Used only by the private text-based fallback of
|
||||
[`BaseMultiModalProcessor._apply_prompt_updates`][vllm.multimodal.processing.BaseMultiModalProcessor].
|
||||
"""
|
||||
|
||||
|
||||
class PromptIndexTargets:
|
||||
@staticmethod
|
||||
@@ -140,35 +103,40 @@ class PromptIndexTargets:
|
||||
|
||||
This results in a match even if the prompt is empty.
|
||||
"""
|
||||
return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
|
||||
return PromptIndex(
|
||||
lambda prompt, start_idx=0: 0,
|
||||
lambda tokenizer, prompt, start_idx=0: 0,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def prefix(seq: PromptSeq) -> PromptIndex:
|
||||
def prefix(seq: list[int]) -> PromptIndex:
|
||||
"""
|
||||
Resolves to a location in the prompt after the given prefix.
|
||||
"""
|
||||
|
||||
def get_match_index(
|
||||
tokenizer: TokenizerLike | None,
|
||||
prompt: PromptSeq,
|
||||
prompt: list[int],
|
||||
start_idx: int = 0,
|
||||
) -> int | None:
|
||||
if start_idx != 0:
|
||||
return None
|
||||
|
||||
prefix = seq
|
||||
match_idx = len(seq)
|
||||
return match_idx if prompt[:match_idx] == seq else None
|
||||
|
||||
if isinstance(prompt, str):
|
||||
# Make both `str`
|
||||
prefix = _seq2text(tokenizer, prefix, use_cache=False)
|
||||
else:
|
||||
# Make both `list[int]`
|
||||
prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
|
||||
def get_text_match_index(
|
||||
tokenizer: TokenizerLike,
|
||||
prompt: str,
|
||||
start_idx: int = 0,
|
||||
) -> int | None:
|
||||
if start_idx != 0:
|
||||
return None
|
||||
|
||||
prefix = tokenizer.decode(seq)
|
||||
match_idx = len(prefix)
|
||||
return match_idx if prompt[:match_idx] == prefix else None
|
||||
|
||||
return PromptIndex(get_match_index)
|
||||
return PromptIndex(get_match_index, get_text_match_index)
|
||||
|
||||
@staticmethod
|
||||
def end() -> PromptIndex:
|
||||
@@ -177,33 +145,36 @@ class PromptIndexTargets:
|
||||
|
||||
This results in a match even if the prompt is empty.
|
||||
"""
|
||||
return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
|
||||
return PromptIndex(
|
||||
lambda prompt, start_idx=0: len(prompt),
|
||||
lambda tokenizer, prompt, start_idx=0: len(prompt),
|
||||
)
|
||||
|
||||
|
||||
UpdateTarget: TypeAlias = PromptSeq | PromptIndex
|
||||
UpdateTarget: TypeAlias = list[int] | PromptIndex
|
||||
"""
|
||||
The token sequence or text to update.
|
||||
The token sequence to update.
|
||||
"""
|
||||
|
||||
PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the corresponding token sequence (or text).
|
||||
output the corresponding token sequence.
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromptUpdateDetails(Generic[_S]):
|
||||
"""Details about the token sequence or text that are part of the update."""
|
||||
class PromptUpdateDetails:
|
||||
"""Details about the token sequence that is part of the update."""
|
||||
|
||||
full: _S
|
||||
full: list[int]
|
||||
"""The full content."""
|
||||
|
||||
is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
|
||||
is_embed: Callable[[list[int]], torch.Tensor] | None = None
|
||||
"""
|
||||
Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
|
||||
return a boolean mask of shape `(len(full),)` indicating which positions
|
||||
@@ -216,56 +187,36 @@ class PromptUpdateDetails(Generic[_S]):
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
|
||||
def from_seq(seq: list[int]) -> "PromptUpdateDetails":
|
||||
return PromptUpdateDetails(full=seq)
|
||||
|
||||
@staticmethod
|
||||
def select_text(
|
||||
seq: _S,
|
||||
embed_text: str,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
|
||||
embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
return torch.isin(
|
||||
torch.tensor(token_ids),
|
||||
torch.tensor(embed_token_ids),
|
||||
)
|
||||
|
||||
return PromptUpdateDetails(full=seq, is_embed=is_embed)
|
||||
|
||||
@staticmethod
|
||||
def select_token_id(
|
||||
seq: _S,
|
||||
seq: list[int],
|
||||
embed_token_id: int,
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
return torch.tensor(token_ids) == embed_token_id
|
||||
) -> "PromptUpdateDetails":
|
||||
def is_embed(full: list[int]) -> torch.Tensor:
|
||||
return torch.tensor(full) == embed_token_id
|
||||
|
||||
return PromptUpdateDetails(full=seq, is_embed=is_embed)
|
||||
|
||||
@staticmethod
|
||||
def select_token_ids(
|
||||
seq: _S,
|
||||
seq: list[int],
|
||||
embed_token_ids: list[int],
|
||||
) -> "PromptUpdateDetails[_S]":
|
||||
def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
|
||||
token_ids = _seq2tokens(tokenizer, full)
|
||||
|
||||
) -> "PromptUpdateDetails":
|
||||
def is_embed(full: list[int]) -> torch.Tensor:
|
||||
return torch.isin(
|
||||
torch.tensor(token_ids),
|
||||
torch.tensor(full),
|
||||
torch.tensor(embed_token_ids),
|
||||
)
|
||||
|
||||
return PromptUpdateDetails(full=seq, is_embed=is_embed)
|
||||
|
||||
|
||||
PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
|
||||
PromptUpdateInfo: TypeAlias = list[int] | PromptUpdateDetails
|
||||
"""
|
||||
The token sequence or text that are part of the update.
|
||||
The token sequence that is part of the update.
|
||||
|
||||
If only part of the content corresponds to feature placeholders, you can
|
||||
use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
|
||||
@@ -276,9 +227,9 @@ PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdat
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the corresponding token sequence (or text).
|
||||
output the corresponding token sequence.
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
@@ -292,13 +243,20 @@ class UpdateMode(str, Enum):
|
||||
class PromptUpdate(ABC):
|
||||
"""
|
||||
Defines how to update a prompt with placeholder tokens.
|
||||
|
||||
Note:
|
||||
The target and content are token sequences. When converting text
|
||||
to token sequences, remember to encode it with
|
||||
`add_special_tokens=False` to match how the prompt itself is
|
||||
tokenized. Otherwise, the updated prompt may contain duplicated
|
||||
special tokens or fail to match the target.
|
||||
"""
|
||||
|
||||
modality: str
|
||||
"""The modality for which the update is made."""
|
||||
|
||||
target: PromptUpdateTarget
|
||||
"""The token sequence (or text) to update."""
|
||||
"""The token sequence to update."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
@@ -357,8 +315,8 @@ class PromptInsertion(PromptUpdate):
|
||||
```python
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target="<s>",
|
||||
insertion="<image>" * image_feature_size,
|
||||
target=[bos_token_id],
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -368,7 +326,7 @@ class PromptInsertion(PromptUpdate):
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.start(),
|
||||
insertion="<image>" * image_feature_size,
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -377,8 +335,8 @@ class PromptInsertion(PromptUpdate):
|
||||
```python
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.prefix("Images:"),
|
||||
insertion="<image>" * image_feature_size,
|
||||
target=PromptIndexTargets.prefix(images_prefix_token_ids),
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -388,7 +346,7 @@ class PromptInsertion(PromptUpdate):
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.end(),
|
||||
insertion="<image>" * image_feature_size,
|
||||
insertion=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
"""
|
||||
@@ -397,10 +355,10 @@ class PromptInsertion(PromptUpdate):
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the token sequence (or text) to insert right after
|
||||
output the token sequence to insert right after
|
||||
[`target`][vllm.multimodal.processing.PromptUpdate.target].
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
@@ -427,8 +385,8 @@ class PromptReplacement(PromptUpdate):
|
||||
```python
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
replacement="<image>" * image_feature_size,
|
||||
target=[image_token_id],
|
||||
replacement=[image_token_id] * image_feature_size,
|
||||
)
|
||||
```
|
||||
|
||||
@@ -436,26 +394,6 @@ class PromptReplacement(PromptUpdate):
|
||||
and `<image_eos>`, which are not supposed to be passed to the vision
|
||||
encoder:
|
||||
|
||||
```python
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
replacement=PromptUpdateDetails(
|
||||
full="".join(
|
||||
[
|
||||
"<image_bos>",
|
||||
"<image>" * image_feature_size,
|
||||
"<image_eos>",
|
||||
]
|
||||
),
|
||||
features="<image>" * image_feature_size,
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
To avoid unnecessary tokenization during prompt replacement,
|
||||
we recommended passing token sequences instead of text:
|
||||
|
||||
```python
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
@@ -464,7 +402,7 @@ class PromptReplacement(PromptUpdate):
|
||||
full=(
|
||||
[image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
|
||||
),
|
||||
features=[image_token_id] * image_feature_size,
|
||||
is_embed=lambda full: torch.tensor(full) == image_token_id,
|
||||
),
|
||||
)
|
||||
```
|
||||
@@ -474,10 +412,10 @@ class PromptReplacement(PromptUpdate):
|
||||
"""
|
||||
Given the index of the processed item within
|
||||
[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
|
||||
output the token sequence (or text) to replace
|
||||
output the token sequence to replace
|
||||
[`target`][vllm.multimodal.processing.PromptUpdate.target].
|
||||
|
||||
For convenience, you can directly pass in the token sequence (or text)
|
||||
For convenience, you can directly pass in the token sequence
|
||||
instead of a function if it does not depend on the input.
|
||||
"""
|
||||
|
||||
@@ -520,7 +458,7 @@ class PromptTargetMatch(NamedTuple):
|
||||
class ResolvedPromptUpdate:
|
||||
"""
|
||||
A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
|
||||
lazy attributes resolved, apart from those related to tokenization.
|
||||
lazy attributes resolved.
|
||||
"""
|
||||
|
||||
modality: str
|
||||
@@ -533,7 +471,7 @@ class ResolvedPromptUpdate:
|
||||
"""Defines how to update the prompt."""
|
||||
|
||||
target: UpdateTarget
|
||||
"""The token sequence (or text) to update."""
|
||||
"""The token sequence to update."""
|
||||
|
||||
content: PromptUpdateDetails = field(repr=False)
|
||||
"""The placeholder tokens that are part of the update."""
|
||||
@@ -541,7 +479,6 @@ class ResolvedPromptUpdate:
|
||||
def iter_token_matches(
|
||||
self,
|
||||
prompt: list[int],
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
@@ -549,52 +486,15 @@ class ResolvedPromptUpdate:
|
||||
target = self.target
|
||||
|
||||
if isinstance(target, PromptIndex):
|
||||
match_idx = target.get_match_index(tokenizer, prompt, start_idx)
|
||||
match_idx = target.get_match_index(prompt, start_idx)
|
||||
if match_idx is not None:
|
||||
yield PromptTargetMatch(match_idx, match_idx)
|
||||
|
||||
return
|
||||
|
||||
target_token_ids = _seq2tokens(tokenizer, target)
|
||||
|
||||
for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
|
||||
for match in iter_token_matches(prompt, target, start_idx=start_idx):
|
||||
yield PromptTargetMatch(match.start_idx, match.end_idx)
|
||||
|
||||
def iter_text_matches(
|
||||
self,
|
||||
prompt: str,
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
"""Yield each instance of `self.target` found in `prompt`."""
|
||||
target = self.target
|
||||
|
||||
if isinstance(target, PromptIndex):
|
||||
match_idx = target.get_match_index(tokenizer, prompt, start_idx)
|
||||
if match_idx is not None:
|
||||
yield PromptTargetMatch(match_idx, match_idx)
|
||||
|
||||
return
|
||||
|
||||
target_text = _seq2text(tokenizer, target)
|
||||
|
||||
for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
|
||||
yield PromptTargetMatch(match.start(), match.end())
|
||||
|
||||
def iter_matches(
|
||||
self,
|
||||
prompt: list[int] | str,
|
||||
tokenizer: TokenizerLike | None,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch]:
|
||||
"""Yield each instance of `self.target` found in `prompt`."""
|
||||
if isinstance(prompt, str):
|
||||
return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
|
||||
|
||||
def with_target(self, target: UpdateTarget):
|
||||
return replace(self, target=target)
|
||||
|
||||
@@ -729,8 +629,6 @@ def _target_key(target: UpdateTarget) -> tuple[str, object]:
|
||||
"""Return a hashable key that preserves target matching semantics."""
|
||||
if isinstance(target, PromptIndex):
|
||||
return ("index", id(target))
|
||||
if isinstance(target, str):
|
||||
return ("text", target)
|
||||
|
||||
return ("tokens", tuple(target))
|
||||
|
||||
@@ -761,10 +659,16 @@ def _compile_prompt_update_queues(
|
||||
return queues_by_signature
|
||||
|
||||
|
||||
_IterMatches: TypeAlias = Callable[
|
||||
[ResolvedPromptUpdate, int],
|
||||
Generator[PromptTargetMatch, None, None],
|
||||
]
|
||||
"""Yields each match of an update's target, given the search start index."""
|
||||
|
||||
|
||||
def _find_queue_match(
|
||||
queue: _UpdateQueue,
|
||||
prompt: _S,
|
||||
tokenizer: TokenizerLike | None,
|
||||
iter_matches: _IterMatches,
|
||||
*,
|
||||
start_idx: int,
|
||||
mode: UpdateMode | None = None,
|
||||
@@ -775,10 +679,7 @@ def _find_queue_match(
|
||||
if mode is not None and update.mode != mode:
|
||||
continue
|
||||
|
||||
match = next(
|
||||
update.iter_matches(prompt, tokenizer, start_idx=start_idx),
|
||||
None,
|
||||
)
|
||||
match = next(iter_matches(update, start_idx), None)
|
||||
if match is not None:
|
||||
return match, update_idx
|
||||
|
||||
@@ -791,10 +692,9 @@ def _next_priority(queue: _UpdateQueue) -> int:
|
||||
return priority
|
||||
|
||||
|
||||
def _plan_prompt_updates(
|
||||
prompt: _S,
|
||||
def _plan_prompt_updates_with(
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
iter_matches: _IterMatches,
|
||||
) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""Plan non-overlapping prompt updates before rendering the output."""
|
||||
queues = list(_compile_prompt_update_queues(mm_prompt_updates).values())
|
||||
@@ -809,8 +709,7 @@ def _plan_prompt_updates(
|
||||
for queue in queues:
|
||||
queue_match = _find_queue_match(
|
||||
queue,
|
||||
prompt,
|
||||
tokenizer,
|
||||
iter_matches,
|
||||
start_idx=prev_end_idx,
|
||||
)
|
||||
if queue_match is not None:
|
||||
@@ -835,8 +734,7 @@ def _plan_prompt_updates(
|
||||
|
||||
queue_match = _find_queue_match(
|
||||
queue,
|
||||
prompt,
|
||||
tokenizer,
|
||||
iter_matches,
|
||||
start_idx=prev_end_idx,
|
||||
mode=mode,
|
||||
)
|
||||
@@ -887,6 +785,19 @@ def _plan_prompt_updates(
|
||||
return planned_updates, result
|
||||
|
||||
|
||||
def _plan_prompt_updates(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""Plan non-overlapping prompt updates before rendering the output."""
|
||||
return _plan_prompt_updates_with(
|
||||
mm_prompt_updates,
|
||||
lambda update, start_idx: update.iter_token_matches(
|
||||
prompt, start_idx=start_idx
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _all_items_found(
|
||||
mm_item_counts: dict[str, int],
|
||||
mm_found_counts: dict[str, int],
|
||||
@@ -898,22 +809,16 @@ def _all_items_found(
|
||||
|
||||
|
||||
def _apply_matches(
|
||||
prompt: _S,
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
|
||||
out_seqs = list[str | list[int]]()
|
||||
matched_updates, result = _plan_prompt_updates(
|
||||
prompt,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
) -> tuple[list[list[int]], "MultiModalPromptUpdatesApplyResult"]:
|
||||
out_seqs = list[list[int]]()
|
||||
matched_updates, result = _plan_prompt_updates(prompt, mm_prompt_updates)
|
||||
|
||||
prev_end_idx = 0
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
matched_content = update.content.full
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
@@ -923,22 +828,17 @@ def _apply_matches(
|
||||
assert_never(update.mode)
|
||||
|
||||
out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
|
||||
out_seqs.append(
|
||||
_seq2text(tokenizer, matched_content)
|
||||
if isinstance(prompt, str)
|
||||
else _seq2tokens(tokenizer, matched_content)
|
||||
)
|
||||
out_seqs.append(update.content.full)
|
||||
prev_end_idx = match.end_idx
|
||||
|
||||
out_seqs.append(prompt[prev_end_idx:])
|
||||
|
||||
return cast(list[_S], out_seqs), result
|
||||
return out_seqs, result
|
||||
|
||||
|
||||
def apply_token_matches(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""
|
||||
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
@@ -947,7 +847,7 @@ def apply_token_matches(
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
"""
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates)
|
||||
|
||||
return flatten_2d_lists(token_id_seqs), result
|
||||
|
||||
@@ -955,17 +855,12 @@ def apply_token_matches(
|
||||
def _apply_token_matches_with_placeholders(
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[
|
||||
list[int],
|
||||
"MultiModalPromptUpdatesApplyResult",
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
matched_updates, result = _plan_prompt_updates(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
|
||||
placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
|
||||
modality: [] for modality in mm_prompt_updates
|
||||
}
|
||||
@@ -975,7 +870,6 @@ def _apply_token_matches_with_placeholders(
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
matched_content = update.content.full
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
@@ -987,11 +881,11 @@ def _apply_token_matches_with_placeholders(
|
||||
new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
|
||||
start_idx = len(new_token_ids)
|
||||
|
||||
tokens = _seq2tokens(tokenizer, matched_content)
|
||||
tokens = update.content.full
|
||||
if tokens:
|
||||
content_is_embed = update.content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(tokenizer, matched_content)
|
||||
content_is_embed = content_is_embed(tokens)
|
||||
|
||||
placeholders[update.modality].append(
|
||||
PlaceholderFeaturesInfo(
|
||||
@@ -1010,50 +904,9 @@ def _apply_token_matches_with_placeholders(
|
||||
return new_token_ids, result, placeholders
|
||||
|
||||
|
||||
def apply_text_matches(
|
||||
prompt: str,
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""
|
||||
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
|
||||
Matches are exclusive even when multiple modalities share
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
"""
|
||||
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
|
||||
return "".join(texts), result
|
||||
|
||||
|
||||
def apply_text_matches_as_segmented_tokens(
|
||||
prompt: str,
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
|
||||
"""
|
||||
Apply the updates in `mm_prompt_updates` to `prompt`.
|
||||
|
||||
Matches are exclusive even when multiple modalities share
|
||||
the same placeholder tokens. In that case, the modality that
|
||||
appears earlier in `mm_prompt_updates` takes priority.
|
||||
|
||||
Each segment is encoded separately instead of being joined into one
|
||||
string and encoded in a single pass. Joining first would let BPE merge
|
||||
tokens across a segment boundary, silently change how a text
|
||||
(non-special-token) placeholder is tokenized.
|
||||
"""
|
||||
texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
|
||||
|
||||
return flatten_2d_lists(token_id_seqs), result
|
||||
|
||||
|
||||
def _iter_placeholders(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> Iterable[PlaceholderFeaturesInfo]:
|
||||
"""
|
||||
Yield each set of placeholder tokens found in `prompt`.
|
||||
@@ -1073,15 +926,15 @@ def _iter_placeholders(
|
||||
prompt_len = len(prompt)
|
||||
start_idx = 0
|
||||
|
||||
# The current (unfound) item's updates for each modality, with their
|
||||
# content resolved to token ids; rebuilt whenever an item is found.
|
||||
# Items are only resolved once the scan reaches them.
|
||||
# The current (unfound) item's updates for each modality; rebuilt
|
||||
# whenever an item is found. Items are only resolved once the scan
|
||||
# reaches them.
|
||||
candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None
|
||||
|
||||
while start_idx < prompt_len:
|
||||
if candidates is None:
|
||||
candidates = [
|
||||
(modality, update, _seq2tokens(tokenizer, update.content.full))
|
||||
(modality, update, update.content.full)
|
||||
for modality, modality_updates in mm_prompt_updates.items()
|
||||
if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0)
|
||||
for update in modality_updates[item_idx_by_modality[modality]]
|
||||
@@ -1107,7 +960,7 @@ def _iter_placeholders(
|
||||
content = update.content
|
||||
content_is_embed = content.is_embed
|
||||
if content_is_embed is not None:
|
||||
content_is_embed = content_is_embed(tokenizer, content.full)
|
||||
content_is_embed = content_is_embed(content.full)
|
||||
|
||||
yield PlaceholderFeaturesInfo(
|
||||
modality=modality,
|
||||
@@ -1134,9 +987,8 @@ def _iter_placeholders(
|
||||
def find_mm_placeholders(
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: "MultiModalPromptUpdates",
|
||||
tokenizer: TokenizerLike | None,
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
|
||||
it = _iter_placeholders(prompt, mm_prompt_updates)
|
||||
return dict(full_groupby_modality(it))
|
||||
|
||||
|
||||
@@ -1146,7 +998,7 @@ A collection of the `is_cached` flag for each item, with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
"""
|
||||
|
||||
MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
|
||||
MultiModalPromptUpdates = Mapping[str, Sequence[Sequence[ResolvedPromptUpdate]]]
|
||||
"""
|
||||
A collection of prompt updates with a similar structure as
|
||||
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
|
||||
@@ -1283,9 +1135,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
new_token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates)
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
@@ -1606,8 +1456,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
prompt: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
return apply_token_matches(prompt, mm_prompt_updates)
|
||||
|
||||
def _apply_token_matches_with_placeholders(
|
||||
self,
|
||||
@@ -1618,31 +1467,105 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return _apply_token_matches_with_placeholders(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
tokenizer,
|
||||
)
|
||||
|
||||
def _apply_text_matches(
|
||||
def _iter_text_matches(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
|
||||
update: ResolvedPromptUpdate,
|
||||
prompt_text: str,
|
||||
tokenizer: TokenizerLike,
|
||||
*,
|
||||
start_idx: int = 0,
|
||||
) -> Generator[PromptTargetMatch, None, None]:
|
||||
"""Yield each instance of `update.target` found in `prompt_text`."""
|
||||
target = update.target
|
||||
|
||||
def _apply_text_matches_as_segmented_tokens(
|
||||
if isinstance(target, PromptIndex):
|
||||
match_idx = target.get_text_match_index(tokenizer, prompt_text, start_idx)
|
||||
if match_idx is not None:
|
||||
yield PromptTargetMatch(match_idx, match_idx)
|
||||
|
||||
return
|
||||
|
||||
target_text = tokenizer.decode(target)
|
||||
if not target_text:
|
||||
return
|
||||
|
||||
idx = prompt_text.find(target_text, start_idx)
|
||||
while idx != -1:
|
||||
end_idx = idx + len(target_text)
|
||||
yield PromptTargetMatch(idx, end_idx)
|
||||
idx = prompt_text.find(target_text, end_idx)
|
||||
|
||||
def _apply_prompt_updates_via_text(
|
||||
self,
|
||||
prompt: str,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
*,
|
||||
encode_segments_separately: bool = False,
|
||||
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||
"""
|
||||
Apply multi-modal prompt updates in text space, then encode the
|
||||
result.
|
||||
|
||||
A non-special-token target may be tokenized differently inside the
|
||||
prompt because the tokens may go across the boundaries of the target
|
||||
(e.g. when searching for "foo" in "food", if "food" itself makes up
|
||||
a token, then the token ID of "foo" will not appear at all). Since
|
||||
it is inefficient to search for all possible tokenizations of the
|
||||
target in the prompt, we instead perform text-based updates on the
|
||||
decoded token IDs, then encode them back.
|
||||
"""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
return apply_text_matches_as_segmented_tokens(
|
||||
prompt, mm_prompt_updates, tokenizer
|
||||
prompt_text = tokenizer.decode(token_ids)
|
||||
|
||||
matched_updates, result = _plan_prompt_updates_with(
|
||||
mm_prompt_updates,
|
||||
lambda update, start_idx: self._iter_text_matches(
|
||||
update, prompt_text, tokenizer, start_idx=start_idx
|
||||
),
|
||||
)
|
||||
|
||||
out_texts = list[str]()
|
||||
prev_end_idx = 0
|
||||
for matched_update in matched_updates:
|
||||
update = matched_update.update
|
||||
match = matched_update.match
|
||||
|
||||
if update.mode == UpdateMode.INSERT:
|
||||
end_idx_to_insert = match.end_idx
|
||||
elif update.mode == UpdateMode.REPLACE:
|
||||
end_idx_to_insert = match.start_idx
|
||||
else:
|
||||
assert_never(update.mode)
|
||||
|
||||
out_texts.append(prompt_text[prev_end_idx:end_idx_to_insert])
|
||||
out_texts.append(tokenizer.decode(update.content.full))
|
||||
prev_end_idx = match.end_idx
|
||||
|
||||
out_texts.append(prompt_text[prev_end_idx:])
|
||||
|
||||
if encode_segments_separately:
|
||||
# Encode each segment separately instead of joining into one
|
||||
# text and encoding it in a single pass. Joining first would
|
||||
# let BPE merge tokens across a segment boundary, silently
|
||||
# changing how a non-special-token placeholder is tokenized.
|
||||
new_token_ids = flatten_2d_lists(
|
||||
[
|
||||
cached_encode(tokenizer, text, add_special_tokens=False)
|
||||
for text in out_texts
|
||||
]
|
||||
)
|
||||
else:
|
||||
new_token_ids = cached_encode(
|
||||
tokenizer, "".join(out_texts), add_special_tokens=False
|
||||
)
|
||||
|
||||
return new_token_ids, result
|
||||
|
||||
def _matched_updates_from_result(
|
||||
self,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
@@ -1668,8 +1591,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Apply multi-modal prompt updates to token IDs."""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
new_token_ids, match_result, placeholders = (
|
||||
self._apply_token_matches_with_placeholders(
|
||||
token_ids,
|
||||
@@ -1688,23 +1609,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
}
|
||||
return new_token_ids, placeholders
|
||||
|
||||
# If the search text does not represent a special token,
|
||||
# it may have different token IDs in the prompt, because
|
||||
# the tokens may go across the boundaries of the search text.
|
||||
# ----
|
||||
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||
# up a token, then the token ID of "foo" will not appear at all
|
||||
# ----
|
||||
# Since it is inefficient to search for all possible tokenizations
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
new_text, match_result = self._apply_text_matches(
|
||||
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||
# A non-special-token target may be tokenized differently inside
|
||||
# the prompt, so fall back to performing the updates on the
|
||||
# decoded text, then encoding the result back.
|
||||
new_token_ids, match_result = self._apply_prompt_updates_via_text(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
|
||||
|
||||
placeholders = self._find_mm_placeholders(
|
||||
new_token_ids,
|
||||
self._matched_updates_from_result(mm_prompt_updates, match_result),
|
||||
|
||||
Reference in New Issue
Block a user