[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
Cyrus Leung
2026-08-23 14:50:32 +08:00
committed by GitHub
co-authored by Qwen Code
parent a3561ef8e4
commit ebcd606467
53 changed files with 831 additions and 855 deletions
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
PromptUpdateDetails,
TimingContext,
cached_encode,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -700,6 +701,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"""Replace each modality's placeholder token with the token ids that item's
replacement text encodes to, marking which of them hold embeddings."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
if isinstance(token, str):
return cached_encode(tokenizer, token, add_special_tokens=False)
if isinstance(token, int):
return [token]
return list(token)
updates = []
for modality, items in out_mm_kwargs.items():
# Popped so they are neither cached nor sent to the model; the updates
@@ -711,10 +721,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
for item in items
]
token = getattr(hf_processor, f"{modality}_token")
updates.append(
PromptReplacement(
modality=modality,
target=getattr(hf_processor, f"{modality}_token"),
target=get_target_token_ids(token),
replacement=replacements.__getitem__,
)
)