[MM] Simplify prompt updates: replace PromptSeq with list[int] (#53372)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: Qwen Code <noreply@alibaba-inc.com>
This commit is contained in:
Cyrus Leung
2026-08-23 14:50:32 +08:00
committed by GitHub
co-authored by Qwen Code
parent a3561ef8e4
commit ebcd606467
53 changed files with 831 additions and 855 deletions
+6 -6
View File
@@ -722,7 +722,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
image_end_id = vocab[IMAGE_END]
merge_size = self.info.image_processor.merge_size # type: ignore[union-attr]
def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def image_replacement(item_idx: int) -> PromptUpdateDetails:
grid = out_mm_kwargs["image"][item_idx]["image_grid_thw"].data
assert isinstance(grid, torch.Tensor)
num_tokens = int(grid.prod()) // merge_size**2
@@ -750,7 +750,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
* int(config.get("merge_factor", 1))
)
def audio_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def audio_replacement(item_idx: int) -> PromptUpdateDetails:
length = out_mm_kwargs["audio"][item_idx]["audio_lengths"].data
assert isinstance(length, torch.Tensor)
num_tokens = math.ceil(int(length.item()) / stride)
@@ -768,7 +768,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
)
if "video" in out_mm_kwargs:
def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
def video_replacement(item_idx: int) -> PromptUpdateDetails:
item = out_mm_kwargs["video"][item_idx]
input_ids_data = item["video_input_ids"].data
embed_mask_data = item["video_embed_mask"].data
@@ -777,8 +777,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
input_ids = input_ids_data.tolist()
embed_mask = embed_mask_data.bool()
def select_video_embeds(tokenizer, full) -> torch.Tensor:
del tokenizer, full
def select_video_embeds(full) -> torch.Tensor:
del full
return embed_mask
return PromptUpdateDetails(
@@ -789,7 +789,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
updates.append(
PromptReplacement(
modality="video",
target=VIDEO_PLACEHOLDER,
target=[vocab[VIDEO_PLACEHOLDER]],
replacement=video_replacement,
)
)