[Refactor] Dynamic target and content for prompt updates (#23411)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -22,10 +22,12 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
|
||||
MultiModalDataItems)
|
||||
# yapf: disable
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, BoundPromptUpdate,
|
||||
BaseProcessingInfo,
|
||||
MultiModalPromptUpdates,
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement, PromptTargetMatch,
|
||||
PromptUpdate, PromptUpdateDetails,
|
||||
PromptReplacement, PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
find_mm_placeholders,
|
||||
replace_token_matches)
|
||||
# yapf: enable
|
||||
@@ -337,14 +339,10 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
def _apply_token_matches(
|
||||
self,
|
||||
prompt: list[int],
|
||||
mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
|
||||
mm_item_counts: Mapping[str, int],
|
||||
) -> list[int]:
|
||||
token_ids = super()._apply_token_matches(
|
||||
prompt,
|
||||
mm_matches,
|
||||
mm_item_counts,
|
||||
)
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||
token_ids, res = super()._apply_token_matches(prompt,
|
||||
mm_prompt_updates)
|
||||
|
||||
# "\n\n\n" and "\n\n\n\n" are single tokens
|
||||
# Since our replacement can insert "\n\n" next to "\n"
|
||||
@@ -373,13 +371,12 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
[newline_4],
|
||||
)
|
||||
|
||||
return token_ids
|
||||
return token_ids, res
|
||||
|
||||
def _find_mm_placeholders(
|
||||
self,
|
||||
mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
|
||||
new_token_ids: list[int],
|
||||
mm_item_counts: Mapping[str, int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
# We need to detect "\n\n" inside "\n\n\n" and "\n\n\n\n"
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -404,8 +401,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
|
||||
repl_token_ids.extend(repl_toks)
|
||||
repl_orig_idxs.extend(orig_idx for _ in range(len(repl_toks)))
|
||||
|
||||
repls = find_mm_placeholders(mm_prompt_updates, repl_token_ids,
|
||||
mm_item_counts)
|
||||
repls = find_mm_placeholders(repl_token_ids, mm_prompt_updates)
|
||||
|
||||
return {
|
||||
modality: [
|
||||
|
||||
@@ -29,10 +29,12 @@ from vllm.multimodal.parse import (ImageProcessorItems, MultiModalDataItems,
|
||||
MultiModalDataParser)
|
||||
# yapf: disable
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, BoundPromptUpdate,
|
||||
BaseProcessingInfo,
|
||||
MultiModalPromptUpdates,
|
||||
MultiModalPromptUpdatesApplyResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement, PromptTargetMatch,
|
||||
PromptUpdate, PromptUpdateDetails,
|
||||
PromptReplacement, PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
find_mm_placeholders,
|
||||
replace_token_matches)
|
||||
# yapf: enable
|
||||
@@ -254,14 +256,10 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]
|
||||
def _apply_token_matches(
|
||||
self,
|
||||
prompt: list[int],
|
||||
mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
|
||||
mm_item_counts: Mapping[str, int],
|
||||
) -> list[int]:
|
||||
token_ids = super()._apply_token_matches(
|
||||
prompt,
|
||||
mm_matches,
|
||||
mm_item_counts,
|
||||
)
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
|
||||
token_ids, res = super()._apply_token_matches(prompt,
|
||||
mm_prompt_updates)
|
||||
|
||||
# "\n\n\n" and "\n\n\n\n" are single tokens
|
||||
# Since our replacement can insert "\n\n" next to "\n"
|
||||
@@ -290,13 +288,12 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]
|
||||
[newline_4],
|
||||
)
|
||||
|
||||
return token_ids
|
||||
return token_ids, res
|
||||
|
||||
def _find_mm_placeholders(
|
||||
self,
|
||||
mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
|
||||
new_token_ids: list[int],
|
||||
mm_item_counts: Mapping[str, int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
# We need to detect "\n\n" inside "\n\n\n" and "\n\n\n\n"
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -321,8 +318,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]
|
||||
repl_token_ids.extend(repl_toks)
|
||||
repl_orig_idxs.extend(orig_idx for _ in range(len(repl_toks)))
|
||||
|
||||
repls = find_mm_placeholders(mm_prompt_updates, repl_token_ids,
|
||||
mm_item_counts)
|
||||
repls = find_mm_placeholders(repl_token_ids, mm_prompt_updates)
|
||||
|
||||
return {
|
||||
modality: [
|
||||
|
||||
@@ -828,26 +828,19 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
|
||||
target=[image_token_id] * num_image_tokens,
|
||||
replacement=get_replacement_mantis,
|
||||
)
|
||||
])
|
||||
], mm_item_counts)
|
||||
|
||||
prompt_ids, prompt, _ = self._apply_prompt_updates(
|
||||
result["prompt_token_ids"],
|
||||
mantis_mm_repls,
|
||||
mm_item_counts,
|
||||
)
|
||||
|
||||
unbound_orig_repls = self._get_prompt_updates(
|
||||
orig_repls = self._get_mm_prompt_updates(
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_kwargs,
|
||||
)
|
||||
orig_repls = self._bind_and_group_updates(unbound_orig_repls)
|
||||
|
||||
mm_placeholders = self._find_mm_placeholders(
|
||||
orig_repls,
|
||||
prompt_ids,
|
||||
mm_item_counts,
|
||||
)
|
||||
mm_placeholders = self._find_mm_placeholders(prompt_ids, orig_repls)
|
||||
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
|
||||
|
||||
mm_placeholder_ranges = {
|
||||
|
||||
@@ -38,7 +38,8 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, BoundPromptUpdate,
|
||||
BaseProcessingInfo,
|
||||
MultiModalPromptUpdates,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement, PromptUpdate)
|
||||
# yapf: enable
|
||||
@@ -431,24 +432,21 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
|
||||
return [_IMAGE_TOKEN_ID] * num_image_tokens
|
||||
|
||||
num_images = mm_items.get_count("image", strict=False)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=image_tokens.__getitem__,
|
||||
replacement=get_replacement_phi3v,
|
||||
) for image_token in image_tokens[:num_images]
|
||||
)
|
||||
]
|
||||
|
||||
def _apply_prompt_updates(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
|
||||
mm_item_counts: Mapping[str, int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
# align to hf behavior when there are images
|
||||
if len(mm_item_counts):
|
||||
if len(mm_prompt_updates):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
# to decode token_ids to the original text, we need to
|
||||
# 1. remove the first bos token
|
||||
@@ -484,7 +482,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
token_ids, text, placeholders = super()._apply_prompt_updates(
|
||||
token_ids=token_ids,
|
||||
mm_prompt_updates=mm_prompt_updates,
|
||||
mm_item_counts=mm_item_counts,
|
||||
)
|
||||
|
||||
# Keep the behavior in line with HF processor
|
||||
|
||||
@@ -1032,8 +1032,8 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_token_id = tokenizer.vocab[tokenizer.image_token]
|
||||
audio_token_id = tokenizer.vocab[tokenizer.audio_token]
|
||||
image_token_id: int = tokenizer.vocab[tokenizer.image_token]
|
||||
audio_token_id: int = tokenizer.vocab[tokenizer.audio_token]
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
audio_processor = self.info.get_feature_extractor(
|
||||
@@ -1053,9 +1053,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
processor=hf_processor,
|
||||
)
|
||||
|
||||
image_tokens = [image_token_id] * num_image_tokens
|
||||
|
||||
return image_tokens
|
||||
return [image_token_id] * num_image_tokens
|
||||
|
||||
def get_audio_replacement_phi4mm(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
@@ -1066,9 +1064,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
audio_embed_size = self.info._compute_audio_embed_size(
|
||||
audio_frames)
|
||||
|
||||
audio_tokens = [audio_token_id] * audio_embed_size
|
||||
|
||||
return audio_tokens
|
||||
return [audio_token_id] * audio_embed_size
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
|
||||
@@ -824,9 +824,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
processor=hf_processor,
|
||||
)
|
||||
|
||||
image_tokens = [_IMAGE_PLACEHOLDER_TOKEN_ID] * num_image_tokens
|
||||
|
||||
return image_tokens
|
||||
return [_IMAGE_PLACEHOLDER_TOKEN_ID] * num_image_tokens
|
||||
|
||||
def get_audio_replacement_phi4mm(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
@@ -837,28 +835,20 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
|
||||
audio_embed_size = self.info._compute_audio_embed_size(
|
||||
audio_frames)
|
||||
|
||||
audio_tokens = [_AUDIO_PLACEHOLDER_TOKEN_ID] * audio_embed_size
|
||||
return [_AUDIO_PLACEHOLDER_TOKEN_ID] * audio_embed_size
|
||||
|
||||
return audio_tokens
|
||||
|
||||
num_images = mm_items.get_count("image", strict=False)
|
||||
num_audios = mm_items.get_count("audio", strict=False)
|
||||
|
||||
image_repl = [
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=image_tokens.__getitem__,
|
||||
replacement=get_image_replacement_phi4mm,
|
||||
) for image_token in image_tokens[:num_images]
|
||||
]
|
||||
audio_repl = [
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
target=audio_tokens.__getitem__,
|
||||
replacement=get_audio_replacement_phi4mm,
|
||||
) for audio_token in audio_tokens[:num_audios]
|
||||
),
|
||||
]
|
||||
return image_repl + audio_repl
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
@@ -309,9 +309,8 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
|
||||
if is_update_applied:
|
||||
mm_placeholders = self._find_mm_placeholders(
|
||||
mm_prompt_updates,
|
||||
prompt_ids,
|
||||
mm_item_counts,
|
||||
mm_prompt_updates,
|
||||
)
|
||||
self._validate_mm_placeholders(
|
||||
mm_placeholders,
|
||||
@@ -328,7 +327,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
) = self._apply_prompt_updates(
|
||||
prompt_ids,
|
||||
mm_prompt_updates,
|
||||
mm_item_counts,
|
||||
)
|
||||
self._validate_mm_placeholders(
|
||||
mm_placeholders,
|
||||
|
||||
Reference in New Issue
Block a user