[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
@@ -483,7 +483,7 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_audioflamingo3,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -277,10 +278,12 @@ class Cohere2VisionMultiModalProcessor(
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
img_tokens_per_tile = int(hf_processor.patch_size**2)
|
||||
img_line_break_token = hf_processor.img_line_break_token
|
||||
boi_token = hf_processor.boi_token
|
||||
eoi_token = hf_processor.eoi_token
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -295,12 +298,13 @@ class Cohere2VisionMultiModalProcessor(
|
||||
patch_tokens = image_token * img_tokens_per_tile + img_line_break_token
|
||||
repl = f"{boi_token}{patch_tokens * num_patches}{eoi_token}"
|
||||
|
||||
return PromptUpdateDetails.select_text(repl, image_token)
|
||||
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
|
||||
return PromptUpdateDetails.select_token_id(repl_ids, image_token_id)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -69,6 +69,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -1197,6 +1198,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
before_placeholder = {
|
||||
"image": "<|image@placeholder|>",
|
||||
@@ -1223,12 +1225,17 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
)
|
||||
else:
|
||||
num_tokens = int(grid_thw.prod()) // merge_length
|
||||
return after_placeholder[modality] * num_tokens
|
||||
placeholder_ids = cached_encode(
|
||||
tokenizer, after_placeholder[modality], add_special_tokens=False
|
||||
)
|
||||
return placeholder_ids * num_tokens
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=before_placeholder[modality],
|
||||
target=cached_encode(
|
||||
tokenizer, before_placeholder[modality], add_special_tokens=False
|
||||
),
|
||||
replacement=partial(get_replacement_ernie45vl, modality=modality),
|
||||
)
|
||||
for modality in ("image", "video")
|
||||
|
||||
@@ -753,7 +753,7 @@ class FunAudioChatMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_funaudiochat,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
replace_token_matches,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -165,7 +166,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
|
||||
image_height: int,
|
||||
processor: Gemma3Processor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> PromptUpdateDetails[str]:
|
||||
) -> PromptUpdateDetails:
|
||||
boi_token = processor.boi_token
|
||||
|
||||
num_crops = self.get_num_crops(
|
||||
@@ -189,8 +190,9 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
|
||||
tokenizer = processor.tokenizer
|
||||
vocab = tokenizer.get_vocab()
|
||||
image_token_id = vocab[tokenizer.image_token]
|
||||
repl_full_ids = cached_encode(tokenizer, repl_full, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(repl_full, image_token_id)
|
||||
return PromptUpdateDetails.select_token_id(repl_full_ids, image_token_id)
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
@@ -313,7 +315,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.boi_token
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
boi_token_id = tokenizer.get_vocab()[hf_processor.boi_token]
|
||||
|
||||
def get_replacement_gemma3(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -329,7 +332,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[boi_token_id],
|
||||
replacement=get_replacement_gemma3,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -51,6 +51,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
replace_token_matches,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -183,31 +184,37 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
processor: Gemma3nProcessor,
|
||||
) -> str:
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Get the replacement text for image tokens.
|
||||
Get the replacement metadata for image tokens.
|
||||
|
||||
For Gemma3n, this should return the full_image_sequence which includes
|
||||
BOI token, repeated image tokens, and EOI token.
|
||||
"""
|
||||
full_token_ids = cached_encode(
|
||||
processor.tokenizer, processor.full_image_sequence, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
processor.full_image_sequence, processor.image_token_id
|
||||
full_token_ids, processor.image_token_id
|
||||
)
|
||||
|
||||
def get_audio_repl(
|
||||
self,
|
||||
*,
|
||||
processor: Gemma3nProcessor,
|
||||
) -> str:
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Get the replacement text for audio tokens.
|
||||
Get the replacement metadata for audio tokens.
|
||||
|
||||
For Gemma3n, this should return the full_audio_sequence which includes
|
||||
BOA token, repeated audio tokens, and EOA token.
|
||||
"""
|
||||
# Return the full audio sequence as defined by the processor
|
||||
full_token_ids = cached_encode(
|
||||
processor.tokenizer, processor.full_audio_sequence, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
processor.full_audio_sequence, processor.audio_token_id
|
||||
full_token_ids, processor.audio_token_id
|
||||
)
|
||||
|
||||
|
||||
@@ -345,7 +352,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
|
||||
# Handle image tokens
|
||||
if "image" in mm_items:
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
|
||||
def get_replacement_image(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -359,14 +366,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_image,
|
||||
)
|
||||
)
|
||||
|
||||
# Handle audio tokens
|
||||
if "audio" in mm_items:
|
||||
audio_token = hf_processor.audio_token
|
||||
audio_token_id = hf_processor.audio_token_id
|
||||
|
||||
def get_replacement_audio(item_idx: int):
|
||||
return self.info.get_audio_repl(
|
||||
@@ -376,7 +383,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_audio,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -332,7 +332,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
image_height: int,
|
||||
processor: Gemma4Processor | None,
|
||||
max_soft_tokens: int | None = None,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Return the dynamic image token sequence for this image.
|
||||
|
||||
Computes the exact number of soft tokens the vision tower will
|
||||
@@ -386,7 +386,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
audio_len: int,
|
||||
processor: Gemma4Processor | None,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Return the dynamic audio token sequence for this audio.
|
||||
|
||||
Computes the number of soft tokens from the audio waveform
|
||||
@@ -414,7 +414,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
timestamps: list[float],
|
||||
num_soft_tokens_per_frame: list[int],
|
||||
processor: Gemma4Processor,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Build the full token replacement for one video.
|
||||
|
||||
Produces the same interleaved sequence as the HF Gemma4Processor:
|
||||
@@ -818,7 +818,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
# one image_token exists per image in the token stream.
|
||||
# The replacement expands it to the full image sequence
|
||||
# (boi + N×image_token + eoi, where N = max_soft_tokens).
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
|
||||
def get_replacement_image(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -848,13 +848,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_image,
|
||||
)
|
||||
)
|
||||
|
||||
if "video" in mm_items:
|
||||
video_token = hf_processor.video_token
|
||||
video_token_id = hf_processor.video_token_id
|
||||
|
||||
def get_replacement_video(item_idx: int):
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
@@ -869,13 +869,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=get_replacement_video,
|
||||
)
|
||||
)
|
||||
|
||||
if "audio" in mm_items:
|
||||
audio_token = hf_processor.audio_token
|
||||
audio_token_id = hf_processor.audio_token_id
|
||||
|
||||
def get_replacement_audio(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
@@ -888,7 +888,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
prompt_updates.append(
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_audio,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -94,6 +94,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import get_processor_cls_name_from_config
|
||||
@@ -1694,6 +1695,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
merge_length = image_processor.merge_size**2
|
||||
|
||||
@@ -1722,12 +1725,18 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=hf_processor.image_token,
|
||||
target=cached_encode(
|
||||
tokenizer, hf_processor.image_token, add_special_tokens=False
|
||||
),
|
||||
replacement=get_image_replacement,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<|begin_of_video|><|video|><|end_of_video|>",
|
||||
target=[
|
||||
vocab["<|begin_of_video|>"],
|
||||
vocab["<|video|>"],
|
||||
vocab["<|end_of_video|>"],
|
||||
],
|
||||
replacement=get_video_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -896,7 +896,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_glmasr,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.multimodal.processing.processor import (
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
TimingContext,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
|
||||
|
||||
@@ -90,6 +91,8 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
||||
hf_processor: H2OVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -127,7 +130,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
||||
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement_internvl,
|
||||
)
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -315,7 +316,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token, _, _ = self.info._get_image_token(hf_processor)
|
||||
image_token_id = hf_processor.image_token_id
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -329,15 +331,18 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
image_repl,
|
||||
embed_text=image_token,
|
||||
image_repl_ids = cached_encode(
|
||||
tokenizer, image_repl, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
image_repl_ids,
|
||||
image_token_id,
|
||||
)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_idefics3,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_video_processor_from_config
|
||||
@@ -355,11 +356,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = hf_processor.tokenizer
|
||||
video_token_id = tokenizer.encode(
|
||||
hf_processor.video_token, add_special_tokens=False
|
||||
)
|
||||
assert len(video_token_id) == 1
|
||||
video_token_id = video_token_id[0]
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
video_token_id = vocab[hf_processor.video_token]
|
||||
|
||||
prompt_text = re.sub(
|
||||
hf_processor.image_token, "<image_placeholder>", prompt_text
|
||||
@@ -469,10 +468,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
img_context_token = hf_processor.image_token
|
||||
img_context_token_id = hf_processor.image_token_id
|
||||
start_image_token = hf_processor.start_image_token
|
||||
end_image_token = hf_processor.end_image_token
|
||||
video_token = hf_processor.video_token
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False)
|
||||
assert len(video_token_id) == 1
|
||||
video_token_id = video_token_id[0]
|
||||
|
||||
out_mm_data = out_mm_kwargs.get_data()
|
||||
if "video_num_patches" in out_mm_data:
|
||||
video_num_patches = out_mm_data["video_num_patches"]
|
||||
@@ -501,7 +506,12 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
|
||||
repl_features = img_context_token * feature_size
|
||||
repl_full = start_image_token + repl_features + end_image_token
|
||||
return PromptUpdateDetails.select_text(repl_full, img_context_token)
|
||||
repl_full_ids = cached_encode(
|
||||
tokenizer, repl_full, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
repl_full_ids, img_context_token_id
|
||||
)
|
||||
|
||||
def get_replacement_interns1_video(item_idx: int):
|
||||
num_patches = video_num_patches[item_idx]
|
||||
@@ -512,17 +522,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl_full, video_token)
|
||||
repl_full_ids = cached_encode(
|
||||
tokenizer, repl_full, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(repl_full_ids, video_token_id)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=img_context_token,
|
||||
target=[img_context_token_id],
|
||||
replacement=get_replacement_interns1_image,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=get_replacement_interns1_video,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
@@ -264,6 +265,8 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
hf_processor: InternVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -298,7 +301,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement_internvl,
|
||||
)
|
||||
|
||||
@@ -498,6 +501,8 @@ class InternVLMultiModalProcessor(
|
||||
hf_processor: InternVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "video_num_patches" in out_mm_data:
|
||||
video_num_patches = out_mm_data["video_num_patches"]
|
||||
assert isinstance(video_num_patches, torch.Tensor)
|
||||
@@ -514,7 +519,7 @@ class InternVLMultiModalProcessor(
|
||||
|
||||
return PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
|
||||
replacement=get_video_replacement_internvl,
|
||||
)
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
@@ -450,6 +451,10 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_pad_token_ids = cached_encode(
|
||||
tokenizer, "<|image_pad|>", add_special_tokens=False
|
||||
)
|
||||
|
||||
pixel_shuffle_scale = getattr(image_processor, "pixel_shuffle_scale", 2)
|
||||
merge_length = pixel_shuffle_scale**2
|
||||
@@ -460,13 +465,13 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
|
||||
feature_size = int(grid_thw.prod()) // merge_length
|
||||
repl_full = "<|image_pad|>" * feature_size
|
||||
return PromptUpdateDetails.select_text(repl_full, "<|image_pad|>")
|
||||
repl_full = image_pad_token_ids * feature_size
|
||||
return PromptUpdateDetails.select_token_ids(repl_full, image_pad_token_ids)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement_isaac,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
@@ -541,6 +542,8 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement(idx: int) -> Sequence[int]:
|
||||
out_item = out_mm_kwargs["image"][idx]
|
||||
image_token_thw = out_item["image_token_thw"].data
|
||||
@@ -552,7 +555,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -448,7 +449,8 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptReplacement]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.image_token
|
||||
image_token_id = hf_processor.image_token_id
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_image_replacement_lfm2vl(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
@@ -463,15 +465,18 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
|
||||
processor=hf_processor,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
return PromptUpdateDetails.select_text(
|
||||
image_repl,
|
||||
embed_text=image_token,
|
||||
image_repl_ids = cached_encode(
|
||||
tokenizer, image_repl, add_special_tokens=False
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
image_repl_ids,
|
||||
image_token_id,
|
||||
)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=get_image_replacement_lfm2vl,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -662,7 +662,7 @@ class MiDashengLMMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_midashenglm,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -1112,7 +1112,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
embed_t = torch.tensor(is_embed_mask)
|
||||
return PromptUpdateDetails(
|
||||
full=full,
|
||||
is_embed=lambda _tok, _seq: embed_t,
|
||||
is_embed=lambda _seq: embed_t,
|
||||
)
|
||||
|
||||
def get_audio_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -502,7 +503,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
out_mm_kwargs=out_mm_kwargs,
|
||||
)
|
||||
|
||||
audio_placeholder = self.info.audio_pattern
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
audio_placeholder = cached_encode(
|
||||
tokenizer, self.info.audio_pattern, add_special_tokens=False
|
||||
)
|
||||
unk_token_ids = [vocab["<unk>"]]
|
||||
|
||||
def get_audio_replacement(item_idx: int):
|
||||
audios = mm_items.get_items(
|
||||
@@ -517,9 +524,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
else:
|
||||
audio_len = audios.get_audio_length(item_idx)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_audio_prompt_texts(audio_len),
|
||||
"<unk>",
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_audio_prompt_texts(audio_len),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
unk_token_ids,
|
||||
)
|
||||
|
||||
return [
|
||||
|
||||
@@ -88,7 +88,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
ResolvedPromptUpdate,
|
||||
_seq2text,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -938,30 +938,24 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
"""Apply multi-modal prompt updates to token IDs."""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
new_token_ids, match_result = self._apply_token_matches(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
|
||||
# If the search text does not represent a special token,
|
||||
# it may have different token IDs in the prompt, because
|
||||
# the tokens may go across the boundaries of the search text.
|
||||
# ----
|
||||
# e.g. when searching for "foo" in "food", if "food" itself makes
|
||||
# up a token, then the token ID of "foo" will not appear at all
|
||||
# ----
|
||||
# Since it is inefficient to search for all possible tokenizations
|
||||
# of the search text in the prompt, we instead perform string-based
|
||||
# updates on the decoded token IDs, then encode them back.
|
||||
# If the target does not consist of special tokens, it may be
|
||||
# tokenized differently inside the prompt, so fall back to
|
||||
# performing the updates on the decoded text, then encoding the
|
||||
# result back. The segments are encoded separately to avoid BPE
|
||||
# merges across segment boundaries.
|
||||
if not all(
|
||||
all(update_idx is not None for update_idx in update_idxs)
|
||||
for update_idxs in match_result.values()
|
||||
):
|
||||
new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
|
||||
_seq2text(tokenizer, token_ids, use_cache=False),
|
||||
new_token_ids, match_result = self._apply_prompt_updates_via_text(
|
||||
token_ids,
|
||||
mm_prompt_updates,
|
||||
encode_segments_separately=True,
|
||||
)
|
||||
|
||||
matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
|
||||
@@ -1062,6 +1056,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
additional_placeholders.append((modality, sub_pattern))
|
||||
placeholders += additional_placeholders
|
||||
|
||||
vocab = tokenizer.get_vocab()
|
||||
unk_token_ids = [vocab["<unk>"]]
|
||||
|
||||
def get_image_replacement(item_idx: int):
|
||||
images = mm_items.get_items(
|
||||
"image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
|
||||
@@ -1069,9 +1066,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
image_size = images.get_image_size(item_idx)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_image_prompt_texts(image_size, item_idx),
|
||||
"<unk>",
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_image_prompt_texts(image_size, item_idx),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
unk_token_ids,
|
||||
)
|
||||
|
||||
def get_video_replacement(item_idx: int):
|
||||
@@ -1082,9 +1083,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
frame_size = videos.get_frame_size(item_idx)
|
||||
num_frames = videos.get_num_frames(item_idx)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_video_prompt_texts(frame_size, num_frames),
|
||||
"<unk>",
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_video_prompt_texts(frame_size, num_frames),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
unk_token_ids,
|
||||
)
|
||||
|
||||
get_replacement = {
|
||||
@@ -1094,7 +1099,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality, target=pattern, replacement=get_replacement[modality]
|
||||
modality=modality,
|
||||
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
|
||||
replacement=get_replacement[modality],
|
||||
)
|
||||
for modality, pattern in placeholders
|
||||
]
|
||||
@@ -1114,7 +1121,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
image_processor = self.info.get_image_processor()
|
||||
version = self.info.get_model_version()
|
||||
|
||||
text = _seq2text(tokenizer, cached_update.content.full)
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
|
||||
if version == (2, 0) or version == (2, 5):
|
||||
@@ -1130,13 +1137,17 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
embed_text = getattr(tokenizer, "image_token", "<unk>")
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_text(
|
||||
text.replace(
|
||||
f"{im_start}{prev_item_idx}{im_end}",
|
||||
f"{im_start}{new_item_idx}{im_end}",
|
||||
1,
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{im_start}{prev_item_idx}{im_end}",
|
||||
f"{im_start}{new_item_idx}{im_end}",
|
||||
1,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
embed_text,
|
||||
cached_encode(tokenizer, embed_text, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
ResolvedPromptUpdate,
|
||||
_seq2text,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -338,6 +338,9 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
# than `<unk>`, so use those tokens as the embedding selector.
|
||||
image_embed_text = getattr(tokenizer, "image_token", "<|image_pad|>")
|
||||
video_embed_text = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
vocab = tokenizer.get_vocab()
|
||||
image_embed_ids = [vocab[image_embed_text]]
|
||||
video_embed_ids = [vocab[video_embed_text]]
|
||||
|
||||
def get_image_replacement(item_idx: int):
|
||||
images = mm_items.get_items(
|
||||
@@ -345,13 +348,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
(MiniCPMVImageEmbeddingItems, ImageProcessorItems),
|
||||
)
|
||||
image_size = images.get_image_size(item_idx)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
image_embed_text,
|
||||
image_embed_ids,
|
||||
)
|
||||
|
||||
def get_video_replacement(item_idx: int):
|
||||
@@ -370,14 +377,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
width=int(image_sizes[0, 0].item()),
|
||||
height=int(image_sizes[0, 1].item()),
|
||||
)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
video_embed_text,
|
||||
video_embed_ids,
|
||||
)
|
||||
|
||||
videos = mm_items.get_items(
|
||||
@@ -386,14 +397,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
)
|
||||
frame_size = videos.get_frame_size(item_idx)
|
||||
num_frames = videos.get_num_frames(item_idx)
|
||||
return PromptUpdateDetails.select_text(
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
return PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
video_embed_text,
|
||||
video_embed_ids,
|
||||
)
|
||||
|
||||
get_replacement = {
|
||||
@@ -404,7 +419,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=pattern,
|
||||
target=cached_encode(tokenizer, pattern, add_special_tokens=False),
|
||||
replacement=get_replacement[modality],
|
||||
)
|
||||
for modality, pattern in placeholders
|
||||
@@ -424,17 +439,21 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = _seq2text(tokenizer, cached_update.content.full)
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_text(
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
video_token,
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -666,9 +667,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
|
||||
|
||||
num_patches_per_chunk = self.info.get_patch_per_chunk(vision_config)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.image_token
|
||||
img_patch_token = hf_processor.img_patch_token
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
img_patch_token_ids = cached_encode(
|
||||
tokenizer, img_patch_token, add_special_tokens=False
|
||||
)
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
aspect_ratio = out_item["aspect_ratios"].data
|
||||
@@ -678,12 +683,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
|
||||
num_patches_per_chunk=num_patches_per_chunk,
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl, img_patch_token)
|
||||
repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
|
||||
return PromptUpdateDetails.select_token_ids(repl_ids, img_patch_token_ids)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[hf_processor.image_token_id],
|
||||
replacement=get_replacement,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
|
||||
PromptInsertion,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -1263,7 +1264,9 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
|
||||
return [
|
||||
PromptInsertion(
|
||||
modality="image",
|
||||
target=PromptIndexTargets.prefix("<|endoftext|>"),
|
||||
target=PromptIndexTargets.prefix(
|
||||
cached_encode(tokenizer, "<|endoftext|>", add_special_tokens=False),
|
||||
),
|
||||
insertion=get_insertion_molmo,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -1393,7 +1393,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
||||
def get_replacement(
|
||||
item_idx: int,
|
||||
suffix_token_ids: list[int] | None = None,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
num_tokens = audio_token_lens[item_idx]
|
||||
if num_tokens == 0:
|
||||
raise ValueError("The audio is too short to be represented.")
|
||||
@@ -1410,7 +1410,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
|
||||
processor.audio_end_id,
|
||||
*suffix_token_ids,
|
||||
],
|
||||
is_embed=lambda _tokenizer, _seq: torch.cat(
|
||||
is_embed=lambda _seq: torch.cat(
|
||||
[
|
||||
torch.tensor([False]),
|
||||
is_embed,
|
||||
|
||||
@@ -64,6 +64,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import ProcessorInputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -535,7 +536,7 @@ class MossTranscribeDiarizeMultiModalProcessor(
|
||||
raise ValueError("Audio input is too short to produce any tokens.")
|
||||
return num_tokens
|
||||
|
||||
def get_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def get_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
num_tokens = get_num_tokens(item_idx)
|
||||
audio_tokens = processor._audio_span_ids(num_tokens)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
@@ -546,7 +547,9 @@ class MossTranscribeDiarizeMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=AUDIO_PLACEHOLDER,
|
||||
target=cached_encode(
|
||||
tokenizer, AUDIO_PLACEHOLDER, add_special_tokens=False
|
||||
),
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -407,7 +407,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
video_end_id = vocab["<|vid_end|>"]
|
||||
video_separator_id = vocab["<|vid_frame_separator|>"]
|
||||
|
||||
def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def image_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
num_tokens = int(out_item["image_feature_sizes"].data)
|
||||
replacement = (
|
||||
@@ -417,7 +417,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
replacement, config.image_token_id
|
||||
)
|
||||
|
||||
def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def video_replacement(item_idx: int) -> PromptUpdateDetails:
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
pixel_values = out_item["video_pixel_values"].data
|
||||
num_groups = len(pixel_values)
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.multimodal.processing.processor import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.multimodal.video_prune.evs import (
|
||||
compute_retained_tokens_count,
|
||||
@@ -429,6 +430,9 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
hf_processor: NanoNemotronVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -470,7 +474,7 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=[vocab["<image>"]],
|
||||
replacement=get_image_replacement,
|
||||
)
|
||||
|
||||
@@ -480,6 +484,8 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
hf_processor: NanoNemotronVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "video_num_patches" in out_mm_data:
|
||||
video_num_patches = out_mm_data["video_num_patches"]
|
||||
assert isinstance(video_num_patches, torch.Tensor)
|
||||
@@ -546,7 +552,7 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
|
||||
return PromptReplacement(
|
||||
modality="video",
|
||||
target="<video>",
|
||||
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
|
||||
replacement=get_video_replacement,
|
||||
)
|
||||
|
||||
@@ -556,13 +562,15 @@ class NanoNemotronVLMultiModalProcessor(
|
||||
hf_processor: NanoNemotronVLProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_audio_replacement(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
return hf_processor.get_audio_repl(audios.get(item_idx))
|
||||
|
||||
return PromptReplacement(
|
||||
modality="audio",
|
||||
target=AUDIO_CONTEXT,
|
||||
target=cached_encode(tokenizer, AUDIO_CONTEXT, add_special_tokens=False),
|
||||
replacement=get_audio_replacement,
|
||||
)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.multimodal.parse import (
|
||||
from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
|
||||
from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor
|
||||
@@ -106,6 +107,8 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
|
||||
hf_processor: NVLMProcessor,
|
||||
out_mm_data: BatchedTensorInputs,
|
||||
):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if "image_num_patches" in out_mm_data:
|
||||
image_num_patches = out_mm_data["image_num_patches"]
|
||||
assert isinstance(image_num_patches, torch.Tensor)
|
||||
@@ -138,14 +141,18 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
|
||||
|
||||
repl = hf_processor.get_image_repl(num_patches, num_features=feature_size)
|
||||
|
||||
return PromptUpdateDetails.select_text(
|
||||
repl.full + "\n", hf_processor.ctx_image_token
|
||||
# `repl.full` ends in the special `</Image>` token, so there is
|
||||
# no BPE merge across the boundary with the newline
|
||||
newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
repl.full + newline_ids, hf_processor.ctx_image_token_id
|
||||
)
|
||||
|
||||
# See note in dummy data regarding why we have the extra newline
|
||||
return PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>\n",
|
||||
target=cached_encode(tokenizer, "<image>\n", add_special_tokens=False),
|
||||
replacement=get_replacement_nvlm,
|
||||
)
|
||||
|
||||
|
||||
@@ -339,7 +339,7 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
bos_token_id = tokenizer.bos_token_id
|
||||
|
||||
def get_insertion(item_idx: int) -> PromptUpdateDetails[list[int]]:
|
||||
def get_insertion(item_idx: int) -> PromptUpdateDetails:
|
||||
images = mm_items.get_items(
|
||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||
)
|
||||
|
||||
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -383,6 +384,8 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> list[PromptReplacement]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement_ovis(item_idx: int):
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
grid = out_item["grids"].data
|
||||
@@ -393,7 +396,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=IMAGE_TOKEN,
|
||||
target=cached_encode(tokenizer, IMAGE_TOKEN, add_special_tokens=False),
|
||||
replacement=get_replacement_ovis,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
ResolvedPromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -436,8 +437,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
|
||||
|
||||
def get_image_token_ids(item_idx: int) -> list[int]:
|
||||
return cached_encode(
|
||||
tokenizer, image_tokens[item_idx], add_special_tokens=False
|
||||
)
|
||||
|
||||
def get_replacement_phi3v(item_idx: int):
|
||||
images = mm_items.get_items(
|
||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||
@@ -458,7 +465,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_tokens.__getitem__,
|
||||
target=get_image_token_ids,
|
||||
replacement=get_replacement_phi3v,
|
||||
)
|
||||
]
|
||||
@@ -475,8 +482,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
|
||||
if cached_update.modality == "image":
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
|
||||
new_update = new_update.with_target(image_tokens[new_item_idx])
|
||||
new_update = new_update.with_target(
|
||||
cached_encode(
|
||||
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
|
||||
)
|
||||
)
|
||||
|
||||
return new_update
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
ResolvedPromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -934,8 +935,20 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_tokens: list[str] = self.info.image_tokens # type: ignore
|
||||
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
|
||||
|
||||
def get_image_token_ids(item_idx: int) -> list[int]:
|
||||
return cached_encode(
|
||||
tokenizer, image_tokens[item_idx], add_special_tokens=False
|
||||
)
|
||||
|
||||
def get_audio_token_ids(item_idx: int) -> list[int]:
|
||||
return cached_encode(
|
||||
tokenizer, audio_tokens[item_idx], add_special_tokens=False
|
||||
)
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
@@ -970,12 +983,12 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_tokens.__getitem__,
|
||||
target=get_image_token_ids,
|
||||
replacement=get_image_replacement_phi4mm,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_tokens.__getitem__,
|
||||
target=get_audio_token_ids,
|
||||
replacement=get_audio_replacement_phi4mm,
|
||||
),
|
||||
]
|
||||
@@ -990,12 +1003,22 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
new_item_idx,
|
||||
)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
if cached_update.modality == "image":
|
||||
image_tokens: list[str] = self.info.image_tokens # type: ignore
|
||||
new_update = new_update.with_target(image_tokens[new_item_idx])
|
||||
new_update = new_update.with_target(
|
||||
cached_encode(
|
||||
tokenizer, image_tokens[new_item_idx], add_special_tokens=False
|
||||
)
|
||||
)
|
||||
elif cached_update.modality == "audio":
|
||||
audio_tokens: list[str] = self.info.audio_tokens # type: ignore
|
||||
new_update = new_update.with_target(audio_tokens[new_item_idx])
|
||||
new_update = new_update.with_target(
|
||||
cached_encode(
|
||||
tokenizer, audio_tokens[new_item_idx], add_special_tokens=False
|
||||
)
|
||||
)
|
||||
|
||||
return new_update
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -193,6 +194,8 @@ class Phi4SiglipMultiModalProcessor(
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
# Read the actual patch grid from the NaFlex processor's
|
||||
# spatial_shapes output (same pattern as LFM2-VL). This avoids
|
||||
@@ -207,7 +210,9 @@ class Phi4SiglipMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=DEFAULT_IMAGE_TOKEN,
|
||||
target=cached_encode(
|
||||
tokenizer, DEFAULT_IMAGE_TOKEN, add_special_tokens=False
|
||||
),
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -301,7 +301,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="", # Never match the prompt (see below note)
|
||||
target=[], # Never match the prompt (see below note)
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -779,17 +779,17 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_qwen2_audio,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=partial(get_replacement_qwen2_vision, modality="image"),
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=video_replacement_fn,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -328,7 +328,7 @@ class Qwen3ASRMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_qwen2_audio,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -1493,17 +1493,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=[audio_token_id],
|
||||
replacement=get_replacement_qwen2_audio,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=[image_token_id],
|
||||
replacement=partial(get_replacement_qwen2_vision, modality="image"),
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target=video_token,
|
||||
target=[video_token_id],
|
||||
replacement=video_replacement_fn,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -1555,17 +1555,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
|
||||
if self._expands_only_video_token(hf_processor):
|
||||
# transformers>=5.10 expands only the bare video_token
|
||||
video_target = hf_processor.video_token
|
||||
video_target = [video_token_id]
|
||||
else:
|
||||
# Old-style processors expand the full placeholder
|
||||
# NOTE: We match string on purpose since searching sequence of
|
||||
# token ids takes more time.
|
||||
video_target = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
video_target = [
|
||||
vision_start_token_id,
|
||||
video_token_id,
|
||||
vision_end_token_id,
|
||||
]
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=hf_processor.image_token,
|
||||
target=[hf_processor.image_token_id],
|
||||
replacement=get_image_replacement_qwen3vl,
|
||||
),
|
||||
PromptReplacement(
|
||||
@@ -1585,7 +1587,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
vision_end_token_id: int,
|
||||
video_token_id: int,
|
||||
select_token_id: bool = False,
|
||||
) -> PromptUpdateDetails[list[int]]:
|
||||
) -> PromptUpdateDetails:
|
||||
"""Build prompt replacement for a video in Qwen3VL format.
|
||||
|
||||
The replacement structure for each frame is:
|
||||
@@ -1609,7 +1611,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
|
||||
# Tokenize timestamp strings independently to avoid tokenizer merging
|
||||
# tokens across boundaries.
|
||||
# TODO: switch to `_seq2tokens` which has some caching.
|
||||
timestamp_token_ids = [
|
||||
tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)
|
||||
for timestamp in timestamps
|
||||
|
||||
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
TimingContext,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -700,6 +701,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Replace each modality's placeholder token with the token ids that item's
|
||||
replacement text encodes to, marking which of them hold embeddings."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
|
||||
if isinstance(token, str):
|
||||
return cached_encode(tokenizer, token, add_special_tokens=False)
|
||||
if isinstance(token, int):
|
||||
return [token]
|
||||
return list(token)
|
||||
|
||||
updates = []
|
||||
for modality, items in out_mm_kwargs.items():
|
||||
# Popped so they are neither cached nor sent to the model; the updates
|
||||
@@ -711,10 +721,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
token = getattr(hf_processor, f"{modality}_token")
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=getattr(hf_processor, f"{modality}_token"),
|
||||
target=get_target_token_ids(token),
|
||||
replacement=replacements.__getitem__,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.multimodal.processing import (
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -298,6 +299,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
replacement_id = hf_processor.audio_replacement_token_id # type: ignore
|
||||
|
||||
@@ -322,7 +324,15 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target="<|audio|>",
|
||||
# `<|audio|>` is paired with `replacement_id` in
|
||||
# `UltravoxProcessingInfo.get_hf_processor`, but it is not
|
||||
# guaranteed to be a single vocab entry, so match the encoded
|
||||
# placeholder text instead of `replacement_id`
|
||||
target=cached_encode(
|
||||
tokenizer,
|
||||
hf_processor.audio_token_replacement, # type: ignore
|
||||
add_special_tokens=False,
|
||||
),
|
||||
replacement=get_replacement_ultravox,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -310,7 +310,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target="", # Never match the prompt (see below note)
|
||||
target=[], # Never match the prompt (see below note)
|
||||
replacement=get_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user