[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
Cyrus Leung
2026-08-23 14:50:32 +08:00
committed by GitHub
co-authored by Qwen Code
parent a3561ef8e4
commit ebcd606467
53 changed files with 831 additions and 855 deletions
+1 -1
View File
@@ -483,7 +483,7 @@ class AudioFlamingo3MultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_audioflamingo3,
)
]
+6 -2
View File
@@ -40,6 +40,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -277,10 +278,12 @@ class Cohere2VisionMultiModalProcessor(
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
img_tokens_per_tile = int(hf_processor.patch_size**2)
img_line_break_token = hf_processor.img_line_break_token
boi_token = hf_processor.boi_token
eoi_token = hf_processor.eoi_token
tokenizer = self.info.get_tokenizer()
def get_replacement(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -295,12 +298,13 @@ class Cohere2VisionMultiModalProcessor(
patch_tokens = image_token * img_tokens_per_tile + img_line_break_token
repl = f"{boi_token}{patch_tokens * num_patches}{eoi_token}"
return PromptUpdateDetails.select_text(repl, image_token)
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(repl_ids, image_token_id)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement,
)
]
+9 -2
View File
@@ -69,6 +69,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -1197,6 +1198,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
before_placeholder = {
"image": "<|image@placeholder|>",
@@ -1223,12 +1225,17 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
)
else:
num_tokens = int(grid_thw.prod()) // merge_length
return after_placeholder[modality] * num_tokens
placeholder_ids = cached_encode(
tokenizer, after_placeholder[modality], add_special_tokens=False
)
return placeholder_ids * num_tokens
return [
PromptReplacement(
modality=modality,
target=before_placeholder[modality],
target=cached_encode(
tokenizer, before_placeholder[modality], add_special_tokens=False
),
replacement=partial(get_replacement_ernie45vl, modality=modality),
)
for modality in ("image", "video")
+1 -1
View File
@@ -753,7 +753,7 @@ class FunAudioChatMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_funaudiochat,
)
]
+7 -4
View File
@@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
replace_token_matches,
)
from vllm.sequence import IntermediateTensors
@@ -165,7 +166,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
image_height: int,
processor: Gemma3Processor,
mm_kwargs: Mapping[str, object],
) -> PromptUpdateDetails[str]:
) -> PromptUpdateDetails:
boi_token = processor.boi_token
num_crops = self.get_num_crops(
@@ -189,8 +190,9 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
tokenizer = processor.tokenizer
vocab = tokenizer.get_vocab()
image_token_id = vocab[tokenizer.image_token]
repl_full_ids = cached_encode(tokenizer, repl_full, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(repl_full, image_token_id)
return PromptUpdateDetails.select_token_id(repl_full_ids, image_token_id)
def get_num_image_tokens(
self,
@@ -313,7 +315,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.boi_token
tokenizer = self.info.get_tokenizer()
boi_token_id = tokenizer.get_vocab()[hf_processor.boi_token]
def get_replacement_gemma3(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -329,7 +332,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=image_token,
target=[boi_token_id],
replacement=get_replacement_gemma3,
)
]
+17 -10
View File
@@ -51,6 +51,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
replace_token_matches,
)
from vllm.sequence import IntermediateTensors
@@ -183,31 +184,37 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
image_width: int,
image_height: int,
processor: Gemma3nProcessor,
) -> str:
) -> PromptUpdateDetails:
"""
Get the replacement text for image tokens.
Get the replacement metadata for image tokens.
For Gemma3n, this should return the full_image_sequence which includes
BOI token, repeated image tokens, and EOI token.
"""
full_token_ids = cached_encode(
processor.tokenizer, processor.full_image_sequence, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
processor.full_image_sequence, processor.image_token_id
full_token_ids, processor.image_token_id
)
def get_audio_repl(
self,
*,
processor: Gemma3nProcessor,
) -> str:
) -> PromptUpdateDetails:
"""
Get the replacement text for audio tokens.
Get the replacement metadata for audio tokens.
For Gemma3n, this should return the full_audio_sequence which includes
BOA token, repeated audio tokens, and EOA token.
"""
# Return the full audio sequence as defined by the processor
full_token_ids = cached_encode(
processor.tokenizer, processor.full_audio_sequence, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
processor.full_audio_sequence, processor.audio_token_id
full_token_ids, processor.audio_token_id
)
@@ -345,7 +352,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
# Handle image tokens
if "image" in mm_items:
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
def get_replacement_image(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -359,14 +366,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt_updates.append(
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement_image,
)
)
# Handle audio tokens
if "audio" in mm_items:
audio_token = hf_processor.audio_token
audio_token_id = hf_processor.audio_token_id
def get_replacement_audio(item_idx: int):
return self.info.get_audio_repl(
@@ -376,7 +383,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
prompt_updates.append(
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_audio,
)
)
+9 -9
View File
@@ -332,7 +332,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
image_height: int,
processor: Gemma4Processor | None,
max_soft_tokens: int | None = None,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Return the dynamic image token sequence for this image.
Computes the exact number of soft tokens the vision tower will
@@ -386,7 +386,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
*,
audio_len: int,
processor: Gemma4Processor | None,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Return the dynamic audio token sequence for this audio.
Computes the number of soft tokens from the audio waveform
@@ -414,7 +414,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
timestamps: list[float],
num_soft_tokens_per_frame: list[int],
processor: Gemma4Processor,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Build the full token replacement for one video.
Produces the same interleaved sequence as the HF Gemma4Processor:
@@ -818,7 +818,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# one image_token exists per image in the token stream.
# The replacement expands it to the full image sequence
# (boi + N×image_token + eoi, where N = max_soft_tokens).
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
def get_replacement_image(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -848,13 +848,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt_updates.append(
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement_image,
)
)
if "video" in mm_items:
video_token = hf_processor.video_token
video_token_id = hf_processor.video_token_id
def get_replacement_video(item_idx: int):
out_item = out_mm_kwargs["video"][item_idx]
@@ -869,13 +869,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt_updates.append(
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=get_replacement_video,
)
)
if "audio" in mm_items:
audio_token = hf_processor.audio_token
audio_token_id = hf_processor.audio_token_id
def get_replacement_audio(item_idx: int):
audios = mm_items.get_items("audio", AudioProcessorItems)
@@ -888,7 +888,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
prompt_updates.append(
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_audio,
)
)
+11 -2
View File
@@ -94,6 +94,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import get_processor_cls_name_from_config
@@ -1694,6 +1695,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
merge_length = image_processor.merge_size**2
@@ -1722,12 +1725,18 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=hf_processor.image_token,
target=cached_encode(
tokenizer, hf_processor.image_token, add_special_tokens=False
),
replacement=get_image_replacement,
),
PromptReplacement(
modality="video",
target="<|begin_of_video|><|video|><|end_of_video|>",
target=[
vocab["<|begin_of_video|>"],
vocab["<|video|>"],
vocab["<|end_of_video|>"],
],
replacement=get_video_replacement,
),
]
+1 -1
View File
@@ -896,7 +896,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_glmasr,
)
]
+4 -1
View File
@@ -25,6 +25,7 @@ from vllm.multimodal.processing.processor import (
ProcessorInputs,
PromptReplacement,
TimingContext,
cached_encode,
)
from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
@@ -90,6 +91,8 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
hf_processor: H2OVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -127,7 +130,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
return PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement_internvl,
)
+10 -5
View File
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -315,7 +316,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token, _, _ = self.info._get_image_token(hf_processor)
image_token_id = hf_processor.image_token_id
tokenizer = self.info.get_tokenizer()
def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:
images = mm_items.get_items("image", ImageProcessorItems)
@@ -329,15 +331,18 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
mm_kwargs=hf_processor_mm_kwargs,
)
return PromptUpdateDetails.select_text(
image_repl,
embed_text=image_token,
image_repl_ids = cached_encode(
tokenizer, image_repl, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
image_repl_ids,
image_token_id,
)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_replacement_idefics3,
)
]
+22 -9
View File
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_video_processor_from_config
@@ -355,11 +356,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = hf_processor.tokenizer
video_token_id = tokenizer.encode(
hf_processor.video_token, add_special_tokens=False
)
assert len(video_token_id) == 1
video_token_id = video_token_id[0]
vocab = tokenizer.get_vocab()
video_token_id = vocab[hf_processor.video_token]
prompt_text = re.sub(
hf_processor.image_token, "<image_placeholder>", prompt_text
@@ -469,10 +468,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
img_context_token = hf_processor.image_token
img_context_token_id = hf_processor.image_token_id
start_image_token = hf_processor.start_image_token
end_image_token = hf_processor.end_image_token
video_token = hf_processor.video_token
tokenizer = self.info.get_tokenizer()
video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False)
assert len(video_token_id) == 1
video_token_id = video_token_id[0]
out_mm_data = out_mm_kwargs.get_data()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
@@ -501,7 +506,12 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
repl_features = img_context_token * feature_size
repl_full = start_image_token + repl_features + end_image_token
return PromptUpdateDetails.select_text(repl_full, img_context_token)
repl_full_ids = cached_encode(
tokenizer, repl_full, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
repl_full_ids, img_context_token_id
)
def get_replacement_interns1_video(item_idx: int):
num_patches = video_num_patches[item_idx]
@@ -512,17 +522,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
)
return PromptUpdateDetails.select_text(repl_full, video_token)
repl_full_ids = cached_encode(
tokenizer, repl_full, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(repl_full_ids, video_token_id)
return [
PromptReplacement(
modality="image",
target=img_context_token,
target=[img_context_token_id],
replacement=get_replacement_interns1_image,
),
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=get_replacement_interns1_video,
),
]
+7 -2
View File
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processors.internvl import (
@@ -264,6 +265,8 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
hf_processor: InternVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -298,7 +301,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
return PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement_internvl,
)
@@ -498,6 +501,8 @@ class InternVLMultiModalProcessor(
hf_processor: InternVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor)
@@ -514,7 +519,7 @@ class InternVLMultiModalProcessor(
return PromptReplacement(
modality="video",
target="<video>",
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
replacement=get_video_replacement_internvl,
)
+8 -3
View File
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
@@ -450,6 +451,10 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
image_pad_token_ids = cached_encode(
tokenizer, "<|image_pad|>", add_special_tokens=False
)
pixel_shuffle_scale = getattr(image_processor, "pixel_shuffle_scale", 2)
merge_length = pixel_shuffle_scale**2
@@ -460,13 +465,13 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
assert isinstance(grid_thw, torch.Tensor)
feature_size = int(grid_thw.prod()) // merge_length
repl_full = "<|image_pad|>" * feature_size
return PromptUpdateDetails.select_text(repl_full, "<|image_pad|>")
repl_full = image_pad_token_ids * feature_size
return PromptUpdateDetails.select_token_ids(repl_full, image_pad_token_ids)
return [
PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement_isaac,
)
]
+4 -1
View File
@@ -33,6 +33,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -541,6 +542,8 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
def get_replacement(idx: int) -> Sequence[int]:
out_item = out_mm_kwargs["image"][idx]
image_token_thw = out_item["image_token_thw"].data
@@ -552,7 +555,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
return [
PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement,
),
]
+10 -5
View File
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdateDetails,
cached_encode,
)
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
@@ -448,7 +449,8 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptReplacement]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.image_token
image_token_id = hf_processor.image_token_id
tokenizer = self.info.get_tokenizer()
def get_image_replacement_lfm2vl(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
@@ -463,15 +465,18 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
processor=hf_processor,
mm_kwargs=hf_processor_mm_kwargs,
)
return PromptUpdateDetails.select_text(
image_repl,
embed_text=image_token,
image_repl_ids = cached_encode(
tokenizer, image_repl, add_special_tokens=False
)
return PromptUpdateDetails.select_token_id(
image_repl_ids,
image_token_id,
)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=get_image_replacement_lfm2vl,
)
]
+1 -1
View File
@@ -662,7 +662,7 @@ class MiDashengLMMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_midashenglm,
)
]
+1 -1
View File
@@ -1112,7 +1112,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
embed_t = torch.tensor(is_embed_mask)
return PromptUpdateDetails(
full=full,
is_embed=lambda _tok, _seq: embed_t,
is_embed=lambda _seq: embed_t,
)
def get_audio_replacement(item_idx: int) -> PromptUpdateDetails:
+15 -4
View File
@@ -58,6 +58,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -502,7 +503,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
out_mm_kwargs=out_mm_kwargs,
)
audio_placeholder = self.info.audio_pattern
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
audio_placeholder = cached_encode(
tokenizer, self.info.audio_pattern, add_special_tokens=False
)
unk_token_ids = [vocab["<unk>"]]
def get_audio_replacement(item_idx: int):
audios = mm_items.get_items(
@@ -517,9 +524,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
else:
audio_len = audios.get_audio_length(item_idx)
return PromptUpdateDetails.select_text(
self.get_audio_prompt_texts(audio_len),
"<unk>",
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_audio_prompt_texts(audio_len),
add_special_tokens=False,
),
unk_token_ids,
)
return [
+40 -29
View File
@@ -88,7 +88,7 @@ from vllm.multimodal.processing.processor import (
PromptUpdate,
PromptUpdateDetails,
ResolvedPromptUpdate,
_seq2text,
cached_encode,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -938,30 +938,24 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
mm_prompt_updates: MultiModalPromptUpdates,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
"""Apply multi-modal prompt updates to token IDs."""
tokenizer = self.info.get_tokenizer()
new_token_ids, match_result = self._apply_token_matches(
token_ids,
mm_prompt_updates,
)
# If the search text does not represent a special token,
# it may have different token IDs in the prompt, because
# the tokens may go across the boundaries of the search text.
# ----
# e.g. when searching for "foo" in "food", if "food" itself makes
# up a token, then the token ID of "foo" will not appear at all
# ----
# Since it is inefficient to search for all possible tokenizations
# of the search text in the prompt, we instead perform string-based
# updates on the decoded token IDs, then encode them back.
# If the target does not consist of special tokens, it may be
# tokenized differently inside the prompt, so fall back to
# performing the updates on the decoded text, then encoding the
# result back. The segments are encoded separately to avoid BPE
# merges across segment boundaries.
if not all(
all(update_idx is not None for update_idx in update_idxs)
for update_idxs in match_result.values()
):
new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
_seq2text(tokenizer, token_ids, use_cache=False),
new_token_ids, match_result = self._apply_prompt_updates_via_text(
token_ids,
mm_prompt_updates,
encode_segments_separately=True,
)
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
@@ -1062,6 +1056,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
additional_placeholders.append((modality, sub_pattern))
placeholders += additional_placeholders
vocab = tokenizer.get_vocab()
unk_token_ids = [vocab["<unk>"]]
def get_image_replacement(item_idx: int):
images = mm_items.get_items(
"image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
@@ -1069,9 +1066,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
image_size = images.get_image_size(item_idx)
return PromptUpdateDetails.select_text(
self.get_image_prompt_texts(image_size, item_idx),
"<unk>",
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_image_prompt_texts(image_size, item_idx),
add_special_tokens=False,
),
unk_token_ids,
)
def get_video_replacement(item_idx: int):
@@ -1082,9 +1083,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
frame_size = videos.get_frame_size(item_idx)
num_frames = videos.get_num_frames(item_idx)
return PromptUpdateDetails.select_text(
self.get_video_prompt_texts(frame_size, num_frames),
"<unk>",
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_video_prompt_texts(frame_size, num_frames),
add_special_tokens=False,
),
unk_token_ids,
)
get_replacement = {
@@ -1094,7 +1099,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
return [
PromptReplacement(
modality=modality, target=pattern, replacement=get_replacement[modality]
modality=modality,
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
replacement=get_replacement[modality],
)
for modality, pattern in placeholders
]
@@ -1114,7 +1121,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
image_processor = self.info.get_image_processor()
version = self.info.get_model_version()
text = _seq2text(tokenizer, cached_update.content.full)
text = tokenizer.decode(cached_update.content.full)
prev_item_idx = cached_update.item_idx
if version == (2, 0) or version == (2, 5):
@@ -1130,13 +1137,17 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
embed_text = getattr(tokenizer, "image_token", "<unk>")
new_update = new_update.with_content(
PromptUpdateDetails.select_text(
text.replace(
f"{im_start}{prev_item_idx}{im_end}",
f"{im_start}{new_item_idx}{im_end}",
1,
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(
f"{im_start}{prev_item_idx}{im_end}",
f"{im_start}{new_item_idx}{im_end}",
1,
),
add_special_tokens=False,
),
embed_text,
cached_encode(tokenizer, embed_text, add_special_tokens=False),
)
)
+48 -29
View File
@@ -36,7 +36,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdateDetails,
ResolvedPromptUpdate,
_seq2text,
cached_encode,
)
from vllm.sequence import IntermediateTensors
@@ -338,6 +338,9 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
# than `<unk>`, so use those tokens as the embedding selector.
image_embed_text = getattr(tokenizer, "image_token", "<|image_pad|>")
video_embed_text = getattr(tokenizer, "video_token", "<|video_pad|>")
vocab = tokenizer.get_vocab()
image_embed_ids = [vocab[image_embed_text]]
video_embed_ids = [vocab[video_embed_text]]
def get_image_replacement(item_idx: int):
images = mm_items.get_items(
@@ -345,13 +348,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
(MiniCPMVImageEmbeddingItems, ImageProcessorItems),
)
image_size = images.get_image_size(item_idx)
return PromptUpdateDetails.select_text(
self.get_image_prompt_texts(
image_size,
item_idx,
downsample_mode=ds_mode,
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_image_prompt_texts(
image_size,
item_idx,
downsample_mode=ds_mode,
),
add_special_tokens=False,
),
image_embed_text,
image_embed_ids,
)
def get_video_replacement(item_idx: int):
@@ -370,14 +377,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
width=int(image_sizes[0, 0].item()),
height=int(image_sizes[0, 1].item()),
)
return PromptUpdateDetails.select_text(
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
),
video_embed_text,
video_embed_ids,
)
videos = mm_items.get_items(
@@ -386,14 +397,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
)
frame_size = videos.get_frame_size(item_idx)
num_frames = videos.get_num_frames(item_idx)
return PromptUpdateDetails.select_text(
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
return PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
),
video_embed_text,
video_embed_ids,
)
get_replacement = {
@@ -404,7 +419,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
return [
PromptReplacement(
modality=modality,
target=pattern,
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
replacement=get_replacement[modality],
)
for modality, pattern in placeholders
@@ -424,17 +439,21 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
text = _seq2text(tokenizer, cached_update.content.full)
text = tokenizer.decode(cached_update.content.full)
prev_item_idx = cached_update.item_idx
new_update = new_update.with_content(
PromptUpdateDetails.select_text(
text.replace(
f"{id_start}{prev_item_idx}{id_end}",
f"{id_start}{new_item_idx}{id_end}",
1,
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(
f"{id_start}{prev_item_idx}{id_end}",
f"{id_start}{new_item_idx}{id_end}",
1,
),
add_special_tokens=False,
),
video_token,
cached_encode(tokenizer, video_token, add_special_tokens=False),
)
)
return new_update
+9 -3
View File
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -666,9 +667,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
num_patches_per_chunk = self.info.get_patch_per_chunk(vision_config)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.image_token
img_patch_token = hf_processor.img_patch_token
tokenizer = self.info.get_tokenizer()
img_patch_token_ids = cached_encode(
tokenizer, img_patch_token, add_special_tokens=False
)
def get_replacement(item_idx: int):
out_item = out_mm_kwargs["image"][item_idx]
aspect_ratio = out_item["aspect_ratios"].data
@@ -678,12 +683,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
num_patches_per_chunk=num_patches_per_chunk,
)
return PromptUpdateDetails.select_text(repl, img_patch_token)
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
return PromptUpdateDetails.select_token_ids(repl_ids, img_patch_token_ids)
return [
PromptReplacement(
modality="image",
target=image_token,
target=[hf_processor.image_token_id],
replacement=get_replacement,
)
]
+4 -1
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
PromptInsertion,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -1263,7 +1264,9 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
return [
PromptInsertion(
modality="image",
target=PromptIndexTargets.prefix("<|endoftext|>"),
target=PromptIndexTargets.prefix(
cached_encode(tokenizer, "<|endoftext|>", add_special_tokens=False),
),
insertion=get_insertion_molmo,
)
]
+2 -2
View File
@@ -1393,7 +1393,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
def get_replacement(
item_idx: int,
suffix_token_ids: list[int] | None = None,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
num_tokens = audio_token_lens[item_idx]
if num_tokens == 0:
raise ValueError("The audio is too short to be represented.")
@@ -1410,7 +1410,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
processor.audio_end_id,
*suffix_token_ids,
],
is_embed=lambda _tokenizer, _seq: torch.cat(
is_embed=lambda _seq: torch.cat(
[
torch.tensor([False]),
is_embed,
@@ -64,6 +64,7 @@ from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
cached_encode,
)
from vllm.multimodal.processing.processor import ProcessorInputs
from vllm.sequence import IntermediateTensors
@@ -535,7 +536,7 @@ class MossTranscribeDiarizeMultiModalProcessor(
raise ValueError("Audio input is too short to produce any tokens.")
return num_tokens
def get_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def get_replacement(item_idx: int) -> PromptUpdateDetails:
num_tokens = get_num_tokens(item_idx)
audio_tokens = processor._audio_span_ids(num_tokens)
return PromptUpdateDetails.select_token_id(
@@ -546,7 +547,9 @@ class MossTranscribeDiarizeMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=AUDIO_PLACEHOLDER,
target=cached_encode(
tokenizer, AUDIO_PLACEHOLDER, add_special_tokens=False
),
replacement=get_replacement,
),
]
+2 -2
View File
@@ -407,7 +407,7 @@ class MuseGlimmerMultiModalProcessor(
video_end_id = vocab["<|vid_end|>"]
video_separator_id = vocab["<|vid_frame_separator|>"]
def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def image_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["image"][item_idx]
num_tokens = int(out_item["image_feature_sizes"].data)
replacement = (
@@ -417,7 +417,7 @@ class MuseGlimmerMultiModalProcessor(
replacement, config.image_token_id
)
def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def video_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["video"][item_idx]
pixel_values = out_item["video_pixel_values"].data
num_groups = len(pixel_values)
+11 -3
View File
@@ -70,6 +70,7 @@ from vllm.multimodal.processing.processor import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.multimodal.video_prune.evs import (
compute_retained_tokens_count,
@@ -429,6 +430,9 @@ class NanoNemotronVLMultiModalProcessor(
hf_processor: NanoNemotronVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -470,7 +474,7 @@ class NanoNemotronVLMultiModalProcessor(
return PromptReplacement(
modality="image",
target="<image>",
target=[vocab["<image>"]],
replacement=get_image_replacement,
)
@@ -480,6 +484,8 @@ class NanoNemotronVLMultiModalProcessor(
hf_processor: NanoNemotronVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor)
@@ -546,7 +552,7 @@ class NanoNemotronVLMultiModalProcessor(
return PromptReplacement(
modality="video",
target="<video>",
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
replacement=get_video_replacement,
)
@@ -556,13 +562,15 @@ class NanoNemotronVLMultiModalProcessor(
hf_processor: NanoNemotronVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
def get_audio_replacement(item_idx: int):
audios = mm_items.get_items("audio", AudioProcessorItems)
return hf_processor.get_audio_repl(audios.get(item_idx))
return PromptReplacement(
modality="audio",
target=AUDIO_CONTEXT,
target=cached_encode(tokenizer, AUDIO_CONTEXT, add_special_tokens=False),
replacement=get_audio_replacement,
)
+10 -3
View File
@@ -26,6 +26,7 @@ from vllm.multimodal.parse import (
from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdateDetails,
cached_encode,
)
from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor
@@ -106,6 +107,8 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
hf_processor: NVLMProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -138,14 +141,18 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
repl = hf_processor.get_image_repl(num_patches, num_features=feature_size)
return PromptUpdateDetails.select_text(
repl.full + "\n", hf_processor.ctx_image_token
# `repl.full` ends in the special `</Image>` token, so there is
# no BPE merge across the boundary with the newline
newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
return PromptUpdateDetails.select_token_id(
repl.full + newline_ids, hf_processor.ctx_image_token_id
)
# See note in dummy data regarding why we have the extra newline
return PromptReplacement(
modality="image",
target="<image>\n",
target=cached_encode(tokenizer, "<image>\n", add_special_tokens=False),
replacement=get_replacement_nvlm,
)
+1 -1
View File
@@ -339,7 +339,7 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
tokenizer = self.info.get_tokenizer()
bos_token_id = tokenizer.bos_token_id
def get_insertion(item_idx: int) -> PromptUpdateDetails[list[int]]:
def get_insertion(item_idx: int) -> PromptUpdateDetails:
images = mm_items.get_items(
"image", (ImageEmbeddingItems, ImageProcessorItems)
)
+4 -1
View File
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
cached_encode,
)
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
@@ -383,6 +384,8 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> list[PromptReplacement]:
tokenizer = self.info.get_tokenizer()
def get_replacement_ovis(item_idx: int):
out_item = out_mm_kwargs["image"][item_idx]
grid = out_item["grids"].data
@@ -393,7 +396,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=IMAGE_TOKEN,
target=cached_encode(tokenizer, IMAGE_TOKEN, add_special_tokens=False),
replacement=get_replacement_ovis,
),
]
+14 -2
View File
@@ -54,6 +54,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
ResolvedPromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -436,8 +437,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
def get_image_token_ids(item_idx: int) -> list[int]:
return cached_encode(
tokenizer, image_tokens[item_idx], add_special_tokens=False
)
def get_replacement_phi3v(item_idx: int):
images = mm_items.get_items(
"image", (ImageEmbeddingItems, ImageProcessorItems)
@@ -458,7 +465,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=image_tokens.__getitem__,
target=get_image_token_ids,
replacement=get_replacement_phi3v,
)
]
@@ -475,8 +482,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
if cached_update.modality == "image":
hf_processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
new_update = new_update.with_target(image_tokens[new_item_idx])
new_update = new_update.with_target(
cached_encode(
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
)
)
return new_update
+27 -4
View File
@@ -47,6 +47,7 @@ from vllm.multimodal.processing.processor import (
PromptReplacement,
PromptUpdate,
ResolvedPromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -934,8 +935,20 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
image_tokens: list[str] = self.info.image_tokens # type: ignore
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
def get_image_token_ids(item_idx: int) -> list[int]:
return cached_encode(
tokenizer, image_tokens[item_idx], add_special_tokens=False
)
def get_audio_token_ids(item_idx: int) -> list[int]:
return cached_encode(
tokenizer, audio_tokens[item_idx], add_special_tokens=False
)
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
@@ -970,12 +983,12 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
return [
PromptReplacement(
modality="image",
target=image_tokens.__getitem__,
target=get_image_token_ids,
replacement=get_image_replacement_phi4mm,
),
PromptReplacement(
modality="audio",
target=audio_tokens.__getitem__,
target=get_audio_token_ids,
replacement=get_audio_replacement_phi4mm,
),
]
@@ -990,12 +1003,22 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
new_item_idx,
)
tokenizer = self.info.get_tokenizer()
if cached_update.modality == "image":
image_tokens: list[str] = self.info.image_tokens # type: ignore
new_update = new_update.with_target(image_tokens[new_item_idx])
new_update = new_update.with_target(
cached_encode(
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
)
)
elif cached_update.modality == "audio":
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
new_update = new_update.with_target(audio_tokens[new_item_idx])
new_update = new_update.with_target(
cached_encode(
tokenizer, audio_tokens[new_item_idx], add_special_tokens=False
)
)
return new_update
+6 -1
View File
@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
BaseProcessingInfo,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -193,6 +194,8 @@ class Phi4SiglipMultiModalProcessor(
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
def get_replacement(item_idx: int):
# Read the actual patch grid from the NaFlex processor's
# spatial_shapes output (same pattern as LFM2-VL). This avoids
@@ -207,7 +210,9 @@ class Phi4SiglipMultiModalProcessor(
return [
PromptReplacement(
modality="image",
target=DEFAULT_IMAGE_TOKEN,
target=cached_encode(
tokenizer, DEFAULT_IMAGE_TOKEN, add_special_tokens=False
),
replacement=get_replacement,
),
]
+1 -1
View File
@@ -301,7 +301,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
return [
PromptReplacement(
modality="image",
target="", # Never match the prompt (see below note)
target=[], # Never match the prompt (see below note)
replacement=get_replacement,
),
]
@@ -779,17 +779,17 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_qwen2_audio,
),
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=partial(get_replacement_qwen2_vision, modality="image"),
),
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=video_replacement_fn,
),
]
+1 -1
View File
@@ -328,7 +328,7 @@ class Qwen3ASRMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_qwen2_audio,
),
]
@@ -1493,17 +1493,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
return [
PromptReplacement(
modality="audio",
target=audio_token,
target=[audio_token_id],
replacement=get_replacement_qwen2_audio,
),
PromptReplacement(
modality="image",
target=image_token,
target=[image_token_id],
replacement=partial(get_replacement_qwen2_vision, modality="image"),
),
PromptReplacement(
modality="video",
target=video_token,
target=[video_token_id],
replacement=video_replacement_fn,
),
]
+8 -7
View File
@@ -1555,17 +1555,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
if self._expands_only_video_token(hf_processor):
# transformers>=5.10 expands only the bare video_token
video_target = hf_processor.video_token
video_target = [video_token_id]
else:
# Old-style processors expand the full placeholder
# NOTE: We match string on purpose since searching sequence of
# token ids takes more time.
video_target = "<|vision_start|><|video_pad|><|vision_end|>"
video_target = [
vision_start_token_id,
video_token_id,
vision_end_token_id,
]
return [
PromptReplacement(
modality="image",
target=hf_processor.image_token,
target=[hf_processor.image_token_id],
replacement=get_image_replacement_qwen3vl,
),
PromptReplacement(
@@ -1585,7 +1587,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
vision_end_token_id: int,
video_token_id: int,
select_token_id: bool = False,
) -> PromptUpdateDetails[list[int]]:
) -> PromptUpdateDetails:
"""Build prompt replacement for a video in Qwen3VL format.
The replacement structure for each frame is:
@@ -1609,7 +1611,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# Tokenize timestamp strings independently to avoid tokenizer merging
# tokens across boundaries.
# TODO: switch to `_seq2tokens` which has some caching.
timestamp_token_ids = [
tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)
for timestamp in timestamps
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
TimingContext,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -700,6 +701,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"""Replace each modality's placeholder token with the token ids that item's
replacement text encodes to, marking which of them hold embeddings."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
if isinstance(token, str):
return cached_encode(tokenizer, token, add_special_tokens=False)
if isinstance(token, int):
return [token]
return list(token)
updates = []
for modality, items in out_mm_kwargs.items():
# Popped so they are neither cached nor sent to the model; the updates
@@ -711,10 +721,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
for item in items
]
token = getattr(hf_processor, f"{modality}_token")
updates.append(
PromptReplacement(
modality=modality,
target=getattr(hf_processor, f"{modality}_token"),
target=get_target_token_ids(token),
replacement=replacements.__getitem__,
)
)
+11 -1
View File
@@ -39,6 +39,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
@@ -298,6 +299,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
replacement_id = hf_processor.audio_replacement_token_id # type: ignore
@@ -322,7 +324,15 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
return [
PromptReplacement(
modality="audio",
target="<|audio|>",
# `<|audio|>` is paired with `replacement_id` in
# `UltravoxProcessingInfo.get_hf_processor`, but it is not
# guaranteed to be a single vocab entry, so match the encoded
# placeholder text instead of `replacement_id`
target=cached_encode(
tokenizer,
hf_processor.audio_token_replacement, # type: ignore
add_special_tokens=False,
),
replacement=get_replacement_ultravox,
)
]
+1 -1
View File
@@ -310,7 +310,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
return [
PromptReplacement(
modality="audio",
target="", # Never match the prompt (see below note)
target=[], # Never match the prompt (see below note)
replacement=get_replacement,
),
]