[Core] Simplify multimodal masking (#34246)

Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
This commit is contained in:
Lukas Geiger
2026-04-01 01:18:22 -07:00
committed by GitHub
parent 36d7f19897
commit 4f6eed3bd4
9 changed files with 54 additions and 51 deletions
+3 -1
View File
@@ -362,7 +362,9 @@ class SupportsMultiModal(Protocol):
# to ensure that any external configuration requiring offset tracking,
# e.g., LoRA, are applied correctly regardless of whether or not
# we have multimodal tokens.
in_vocab_ids = input_ids.masked_fill(is_multimodal, 0)
in_vocab_ids = input_ids.masked_fill(
is_multimodal.to(device=input_ids.device, non_blocking=True), 0
)
return embed_input_ids(in_vocab_ids)
return embed_input_ids(input_ids)