[Refactor] Dynamic target and content for prompt updates (#23411)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -38,7 +38,8 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, BoundPromptUpdate,
|
||||
BaseProcessingInfo,
|
||||
MultiModalPromptUpdates,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement, PromptUpdate)
|
||||
# yapf: enable
|
||||
@@ -431,24 +432,21 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
|
||||
return [_IMAGE_TOKEN_ID] * num_image_tokens
|
||||
|
||||
num_images = mm_items.get_count("image", strict=False)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
target=image_tokens.__getitem__,
|
||||
replacement=get_replacement_phi3v,
|
||||
) for image_token in image_tokens[:num_images]
|
||||
)
|
||||
]
|
||||
|
||||
def _apply_prompt_updates(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
|
||||
mm_item_counts: Mapping[str, int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
# align to hf behavior when there are images
|
||||
if len(mm_item_counts):
|
||||
if len(mm_prompt_updates):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
# to decode token_ids to the original text, we need to
|
||||
# 1. remove the first bos token
|
||||
@@ -484,7 +482,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
token_ids, text, placeholders = super()._apply_prompt_updates(
|
||||
token_ids=token_ids,
|
||||
mm_prompt_updates=mm_prompt_updates,
|
||||
mm_item_counts=mm_item_counts,
|
||||
)
|
||||
|
||||
# Keep the behavior in line with HF processor
|
||||
|
||||
Reference in New Issue
Block a user