[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
@@ -21,7 +21,7 @@ from transformers import (
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.processing import PromptUpdateDetails
|
||||
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||
@@ -444,7 +444,7 @@ class InternVLProcessor(ProcessorMixin):
|
||||
self,
|
||||
num_patches: int | None,
|
||||
num_features: int | None = None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
if num_patches is None:
|
||||
assert num_features is not None
|
||||
else:
|
||||
@@ -453,9 +453,11 @@ class InternVLProcessor(ProcessorMixin):
|
||||
repl_features = self.ctx_image_token * num_features
|
||||
repl_full = self.start_image_token + repl_features + self.end_image_token
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, self.ctx_image_token)
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
|
||||
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails[str]:
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
|
||||
|
||||
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails:
|
||||
assert self.ctx_video_token is not None
|
||||
|
||||
repl_features = self.ctx_video_token * self.image_seq_length
|
||||
@@ -467,7 +469,10 @@ class InternVLProcessor(ProcessorMixin):
|
||||
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, self.ctx_video_token)
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
assert self.ctx_video_token_id is not None
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -523,7 +528,13 @@ class InternVLProcessor(ProcessorMixin):
|
||||
while image_token in new_prompt:
|
||||
new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
|
||||
image_repl = self.get_image_repl(image_num_patches[image_index])
|
||||
replace_strings.append(image_repl.full)
|
||||
# Convert the token IDs back to text for the
|
||||
# string-based replacement below
|
||||
replace_strings.append(
|
||||
self.tokenizer.decode(
|
||||
image_repl.full, skip_special_tokens=False
|
||||
)
|
||||
)
|
||||
image_index += 1
|
||||
|
||||
while "<placeholder>" in new_prompt:
|
||||
@@ -548,7 +559,13 @@ class InternVLProcessor(ProcessorMixin):
|
||||
while video_token in new_prompt:
|
||||
new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
|
||||
video_repl = self.get_video_repl(video_num_patches[video_index])
|
||||
replace_strings.append(video_repl.full)
|
||||
# Convert the token IDs back to text for the
|
||||
# string-based replacement below
|
||||
replace_strings.append(
|
||||
self.tokenizer.decode(
|
||||
video_repl.full, skip_special_tokens=False
|
||||
)
|
||||
)
|
||||
video_index += 1
|
||||
|
||||
while "<placeholder>" in new_prompt:
|
||||
|
||||
@@ -24,7 +24,7 @@ from transformers import BatchFeature, PretrainedConfig, TensorType
|
||||
|
||||
from vllm.model_executor.models.parakeet import ParakeetExtractor
|
||||
from vllm.multimodal.inputs import AudioItem
|
||||
from vllm.multimodal.processing.processor import PromptUpdateDetails
|
||||
from vllm.multimodal.processing.processor import PromptUpdateDetails, cached_encode
|
||||
from vllm.multimodal.video_prune.evs import compute_retained_tokens_count
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
@@ -633,7 +633,7 @@ class BaseNanoNemotronVLProcessor(ABC):
|
||||
self,
|
||||
feature_size: int,
|
||||
num_patches: int | None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_num_image_tokens(
|
||||
@@ -730,7 +730,12 @@ class BaseNanoNemotronVLProcessor(ABC):
|
||||
zip(num_tokens_per_image, image_num_patches, strict=True)
|
||||
):
|
||||
image_repl = self.get_image_repl(feature_size, num_patches)
|
||||
parts[i] = parts[i].replace("<image>", image_repl.full)
|
||||
# image_repl.full is a list of token IDs
|
||||
# Convert token IDs back to text for the HF processor flow
|
||||
image_repl_text = self.tokenizer.decode(
|
||||
image_repl.full, skip_special_tokens=False
|
||||
)
|
||||
parts[i] = parts[i].replace("<image>", image_repl_text)
|
||||
text = ["".join(parts)]
|
||||
|
||||
return text, image_inputs
|
||||
@@ -1001,7 +1006,11 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
for idx, part in enumerate(parts):
|
||||
if part == AUDIO_CONTEXT:
|
||||
audio_repl = self.get_audio_repl(audios[audio_index])
|
||||
parts[idx] = audio_repl.full
|
||||
# audio_repl.full is a list of token IDs
|
||||
# Convert token IDs back to text for the HF processor flow
|
||||
parts[idx] = self.tokenizer.decode(
|
||||
audio_repl.full, skip_special_tokens=False
|
||||
)
|
||||
audio_index += 1
|
||||
text = ["".join(parts)]
|
||||
audio_inputs = extractor(audios)
|
||||
@@ -1078,20 +1087,30 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
self,
|
||||
feature_size: int,
|
||||
num_patches: int | None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
repl_features = IMG_CONTEXT * feature_size
|
||||
repl_full = IMG_START + repl_features + IMG_END
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
full_ids, self._img_context_token_ids
|
||||
)
|
||||
|
||||
def get_audio_repl(
|
||||
self,
|
||||
audio: npt.NDArray,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
assert self.audio_extractor is not None
|
||||
num_tokens = self.audio_extractor.audio_token_count(len(audio))
|
||||
repl_full = f"{AUDIO_START}{AUDIO_CONTEXT * num_tokens}{AUDIO_END}"
|
||||
return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
|
||||
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
ctx_token_ids = cached_encode(
|
||||
self.tokenizer, AUDIO_CONTEXT, add_special_tokens=False
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_token_ids(full_ids, ctx_token_ids)
|
||||
|
||||
@classmethod
|
||||
def get_video_repl(
|
||||
@@ -1105,7 +1124,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
img_end_token_ids: list[int],
|
||||
img_context_token_ids: list[int],
|
||||
video_temporal_patch_size: int = 1,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Build prompt replacement for a video.
|
||||
The replacement returned is not actually used to replace the placeholder
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
# Copyright (c) 2024 NVIDIA
|
||||
# Licensed under Apache 2.0 License [see LICENSE for details]
|
||||
# --------------------------------------------------------
|
||||
from vllm.multimodal.processing import PromptUpdateDetails
|
||||
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
from .internvl import InternVLImageProcessor, InternVLProcessor
|
||||
@@ -37,7 +37,7 @@ class NVLMProcessor(InternVLProcessor):
|
||||
self,
|
||||
num_patches: int | None,
|
||||
num_features: int | None = None,
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
if num_patches is None:
|
||||
raise NotImplementedError("Embedding inputs are not supported")
|
||||
|
||||
@@ -58,4 +58,6 @@ class NVLMProcessor(InternVLProcessor):
|
||||
# when trying to find "<tile" as a subsequence of "<Image><tile"
|
||||
repl = self.start_image_token + features + self.end_image_token
|
||||
|
||||
return PromptUpdateDetails.select_text(repl, self.ctx_image_token)
|
||||
full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
|
||||
|
||||
Reference in New Issue
Block a user