Increase Flexibility for OOV Multimodal Token Handling (#34858)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
@@ -130,6 +130,13 @@ class SupportsMultiModal(Protocol):
|
||||
Set internally by `_mark_tower_model`.
|
||||
"""
|
||||
|
||||
_has_oov_mm_tokens: bool = False
|
||||
"""
|
||||
In general, this should be set at init time by invoking
|
||||
`configure_mm_token_handling` models & passing all potentially
|
||||
OOV multimodal tokens.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
"""
|
||||
@@ -149,6 +156,17 @@ class SupportsMultiModal(Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
def configure_mm_token_handling(self, vocab_size: int, mm_token_ids: list[int]):
|
||||
"""Check if any multimodal tokens are out of vocabulary. If so, we will
|
||||
explicitly mask all multimodal tokens out when computing text embeddings,
|
||||
since the multimodal embeddings will be scattered over the results.
|
||||
"""
|
||||
self._has_oov_mm_tokens = any(tok_id >= vocab_size for tok_id in mm_token_ids)
|
||||
logger.info(
|
||||
"Contains out of vocabulary multimodal tokens? %s",
|
||||
self._has_oov_mm_tokens,
|
||||
)
|
||||
|
||||
def get_language_model(self) -> VllmModel:
|
||||
"""
|
||||
Returns the underlying language model used for text generation.
|
||||
@@ -324,7 +342,6 @@ class SupportsMultiModal(Protocol):
|
||||
multimodal_embeddings: MultiModalEmbeddings,
|
||||
*,
|
||||
is_multimodal: torch.Tensor,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> Tensor: ...
|
||||
|
||||
def _embed_text_input_ids(
|
||||
@@ -333,17 +350,14 @@ class SupportsMultiModal(Protocol):
|
||||
embed_input_ids: Callable[[Tensor], Tensor],
|
||||
*,
|
||||
is_multimodal: Tensor | None,
|
||||
handle_oov_mm_token: bool,
|
||||
) -> Tensor:
|
||||
if handle_oov_mm_token and is_multimodal is not None:
|
||||
is_text = ~is_multimodal
|
||||
text_embeds = embed_input_ids(input_ids[is_text])
|
||||
|
||||
return torch.empty(
|
||||
(input_ids.shape[0], text_embeds.shape[1]),
|
||||
dtype=text_embeds.dtype,
|
||||
device=text_embeds.device,
|
||||
).masked_scatter_(is_text.unsqueeze_(-1), text_embeds)
|
||||
if is_multimodal is not None and self._has_oov_mm_tokens:
|
||||
# Force all input IDs to be in vocab; we do this instead of squeezing
|
||||
# to ensure that any external configuration requiring offset tracking,
|
||||
# e.g., LoRA, are applied correctly regardless of whether or not
|
||||
# we have multimodal tokens.
|
||||
in_vocab_ids = input_ids.masked_fill(is_multimodal, 0)
|
||||
return embed_input_ids(in_vocab_ids)
|
||||
|
||||
return embed_input_ids(input_ids)
|
||||
|
||||
@@ -353,7 +367,6 @@ class SupportsMultiModal(Protocol):
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> Tensor:
|
||||
"""
|
||||
Apply token embeddings to `input_ids`.
|
||||
@@ -361,19 +374,19 @@ class SupportsMultiModal(Protocol):
|
||||
If `multimodal_embeddings` is passed, scatter them into
|
||||
`input_ids` according to the mask `is_multimodal`.
|
||||
|
||||
In case the multi-modal token IDs exceed the vocabulary size of
|
||||
the language model, you can set `handle_oov_mm_token=False`
|
||||
to avoid calling the language model's `embed_input_ids` method
|
||||
on those tokens. Note however that doing so increases memory usage
|
||||
as an additional buffer is needed to hold the input embeddings.
|
||||
NOTE: If this model has multimodal tokens that are of vocabulary
|
||||
(i.e., self._has_oov_mm_tokens=True), the input_ids will be copied
|
||||
and masked to 0 during the forward pass for the text embeddings.
|
||||
"""
|
||||
from .utils import _merge_multimodal_embeddings
|
||||
|
||||
# Get text embeddings first; multimodal embeddings will clobber
|
||||
# any invalid contents in the indices of multimodal embeddings
|
||||
# for the in vocabulary and out of vocabulary case.
|
||||
inputs_embeds = self._embed_text_input_ids(
|
||||
input_ids,
|
||||
self.get_language_model().embed_input_ids,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
|
||||
Reference in New Issue
Block a user