[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
Cyrus Leung
2026-08-23 14:50:32 +08:00
committed by GitHub
co-authored by Qwen Code
parent a3561ef8e4
commit ebcd606467
53 changed files with 831 additions and 855 deletions
+24 -7
View File
@@ -21,7 +21,7 @@ from transformers import (
from transformers.processing_utils import ProcessorMixin
from vllm.multimodal.image import convert_image_mode
from vllm.multimodal.processing import PromptUpdateDetails
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
from vllm.tokenizers.hf import HfTokenizer
IMAGENET_MEAN = (0.485, 0.456, 0.406)
@@ -444,7 +444,7 @@ class InternVLProcessor(ProcessorMixin):
self,
num_patches: int | None,
num_features: int | None = None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
if num_patches is None:
assert num_features is not None
else:
@@ -453,9 +453,11 @@ class InternVLProcessor(ProcessorMixin):
repl_features = self.ctx_image_token * num_features
repl_full = self.start_image_token + repl_features + self.end_image_token
return PromptUpdateDetails.select_text(repl_full, self.ctx_image_token)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails[str]:
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails:
assert self.ctx_video_token is not None
repl_features = self.ctx_video_token * self.image_seq_length
@@ -467,7 +469,10 @@ class InternVLProcessor(ProcessorMixin):
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
)
return PromptUpdateDetails.select_text(repl_full, self.ctx_video_token)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
assert self.ctx_video_token_id is not None
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id)
def __call__(
self,
@@ -523,7 +528,13 @@ class InternVLProcessor(ProcessorMixin):
while image_token in new_prompt:
new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
image_repl = self.get_image_repl(image_num_patches[image_index])
replace_strings.append(image_repl.full)
# Convert the token IDs back to text for the
# string-based replacement below
replace_strings.append(
self.tokenizer.decode(
image_repl.full, skip_special_tokens=False
)
)
image_index += 1
while "<placeholder>" in new_prompt:
@@ -548,7 +559,13 @@ class InternVLProcessor(ProcessorMixin):
while video_token in new_prompt:
new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
video_repl = self.get_video_repl(video_num_patches[video_index])
replace_strings.append(video_repl.full)
# Convert the token IDs back to text for the
# string-based replacement below
replace_strings.append(
self.tokenizer.decode(
video_repl.full, skip_special_tokens=False
)
)
video_index += 1
while "<placeholder>" in new_prompt:
@@ -24,7 +24,7 @@ from transformers import BatchFeature, PretrainedConfig, TensorType
from vllm.model_executor.models.parakeet import ParakeetExtractor
from vllm.multimodal.inputs import AudioItem
from vllm.multimodal.processing.processor import PromptUpdateDetails
from vllm.multimodal.processing.processor import PromptUpdateDetails, cached_encode
from vllm.multimodal.video_prune.evs import compute_retained_tokens_count
from vllm.platforms import current_platform
from vllm.tokenizers.hf import HfTokenizer
@@ -633,7 +633,7 @@ class BaseNanoNemotronVLProcessor(ABC):
self,
feature_size: int,
num_patches: int | None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
raise NotImplementedError
def get_num_image_tokens(
@@ -730,7 +730,12 @@ class BaseNanoNemotronVLProcessor(ABC):
zip(num_tokens_per_image, image_num_patches, strict=True)
):
image_repl = self.get_image_repl(feature_size, num_patches)
parts[i] = parts[i].replace("<image>", image_repl.full)
# image_repl.full is a list of token IDs
# Convert token IDs back to text for the HF processor flow
image_repl_text = self.tokenizer.decode(
image_repl.full, skip_special_tokens=False
)
parts[i] = parts[i].replace("<image>", image_repl_text)
text = ["".join(parts)]
return text, image_inputs
@@ -1001,7 +1006,11 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
for idx, part in enumerate(parts):
if part == AUDIO_CONTEXT:
audio_repl = self.get_audio_repl(audios[audio_index])
parts[idx] = audio_repl.full
# audio_repl.full is a list of token IDs
# Convert token IDs back to text for the HF processor flow
parts[idx] = self.tokenizer.decode(
audio_repl.full, skip_special_tokens=False
)
audio_index += 1
text = ["".join(parts)]
audio_inputs = extractor(audios)
@@ -1078,20 +1087,30 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
self,
feature_size: int,
num_patches: int | None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
repl_features = IMG_CONTEXT * feature_size
repl_full = IMG_START + repl_features + IMG_END
return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
return PromptUpdateDetails.select_token_ids(
full_ids, self._img_context_token_ids
)
def get_audio_repl(
self,
audio: npt.NDArray,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
assert self.audio_extractor is not None
num_tokens = self.audio_extractor.audio_token_count(len(audio))
repl_full = f"{AUDIO_START}{AUDIO_CONTEXT * num_tokens}{AUDIO_END}"
return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
ctx_token_ids = cached_encode(
self.tokenizer, AUDIO_CONTEXT, add_special_tokens=False
)
return PromptUpdateDetails.select_token_ids(full_ids, ctx_token_ids)
@classmethod
def get_video_repl(
@@ -1105,7 +1124,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
img_end_token_ids: list[int],
img_context_token_ids: list[int],
video_temporal_patch_size: int = 1,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""
Build prompt replacement for a video.
The replacement returned is not actually used to replace the placeholder
+5 -3
View File
@@ -7,7 +7,7 @@
# Copyright (c) 2024 NVIDIA
# Licensed under Apache 2.0 License [see LICENSE for details]
# --------------------------------------------------------
from vllm.multimodal.processing import PromptUpdateDetails
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
from vllm.tokenizers.hf import HfTokenizer
from .internvl import InternVLImageProcessor, InternVLProcessor
@@ -37,7 +37,7 @@ class NVLMProcessor(InternVLProcessor):
self,
num_patches: int | None,
num_features: int | None = None,
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
if num_patches is None:
raise NotImplementedError("Embedding inputs are not supported")
@@ -58,4 +58,6 @@ class NVLMProcessor(InternVLProcessor):
# when trying to find "<tile" as a subsequence of "<Image><tile"
repl = self.start_image_token + features + self.end_image_token
return PromptUpdateDetails.select_text(repl, self.ctx_image_token)
full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)