[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
@@ -450,6 +451,10 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
image_pad_token_ids = cached_encode(
|
||||
tokenizer, "<|image_pad|>", add_special_tokens=False
|
||||
)
|
||||
|
||||
pixel_shuffle_scale = getattr(image_processor, "pixel_shuffle_scale", 2)
|
||||
merge_length = pixel_shuffle_scale**2
|
||||
@@ -460,13 +465,13 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
|
||||
feature_size = int(grid_thw.prod()) // merge_length
|
||||
repl_full = "<|image_pad|>" * feature_size
|
||||
return PromptUpdateDetails.select_text(repl_full, "<|image_pad|>")
|
||||
repl_full = image_pad_token_ids * feature_size
|
||||
return PromptUpdateDetails.select_token_ids(repl_full, image_pad_token_ids)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target="<image>",
|
||||
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
|
||||
replacement=get_replacement_isaac,
|
||||
)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user