[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
Cyrus Leung
2026-08-23 14:50:32 +08:00
committed by GitHub
co-authored by Qwen Code
parent a3561ef8e4
commit ebcd606467
53 changed files with 831 additions and 855 deletions
+7 -2
View File
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processors.internvl import (
@@ -264,6 +265,8 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
hf_processor: InternVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor)
@@ -298,7 +301,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
return PromptReplacement(
modality="image",
target="<image>",
target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
replacement=get_replacement_internvl,
)
@@ -498,6 +501,8 @@ class InternVLMultiModalProcessor(
hf_processor: InternVLProcessor,
out_mm_data: BatchedTensorInputs,
):
tokenizer = self.info.get_tokenizer()
if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor)
@@ -514,7 +519,7 @@ class InternVLMultiModalProcessor(
return PromptReplacement(
modality="video",
target="<video>",
target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
replacement=get_video_replacement_internvl,
)