[Bugfix][Multimodal] Encode text in mixed CLIP/SigLIP pooling batches (#53165)
Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4f686e182a
commit
342b8ebd8b
@@ -57,6 +57,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .clip import dual_encoder_has_text_tokens, merge_dual_encoder_text_and_vision
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
@@ -943,7 +944,9 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
|
||||
self.pooler = DispatchPooler.for_embedding(pooler_config)
|
||||
|
||||
self._is_text_input = True
|
||||
# Set in embed_input_ids; consumed by forward.
|
||||
self._has_text_tokens = True
|
||||
self._mm_token_mask: torch.Tensor | None = None
|
||||
|
||||
def get_text_features(
|
||||
self,
|
||||
@@ -1088,8 +1091,12 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
self._is_text_input = (
|
||||
multimodal_embeddings is None or len(multimodal_embeddings) == 0
|
||||
has_mm_embeddings = (
|
||||
multimodal_embeddings is not None and len(multimodal_embeddings) > 0
|
||||
)
|
||||
self._mm_token_mask = is_multimodal
|
||||
self._has_text_tokens = dual_encoder_has_text_tokens(
|
||||
has_mm_embeddings, is_multimodal
|
||||
)
|
||||
|
||||
if multimodal_embeddings is None or is_multimodal is None:
|
||||
@@ -1121,9 +1128,11 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
raise RuntimeError("PP is not supported for this model")
|
||||
|
||||
# Multimodal inputs (image embeddings)
|
||||
if not self._is_text_input:
|
||||
assert inputs_embeds is not None
|
||||
if not self._has_text_tokens:
|
||||
return inputs_embeds
|
||||
|
||||
vision_embeds = inputs_embeds
|
||||
# NOTE: inputs_embeds in model runner has size text_config.projection_size
|
||||
# (instead of text_config.hidden_size) to accommodate image embeddings
|
||||
hidden_size = self.text_embed_dim
|
||||
@@ -1133,7 +1142,10 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
# No need to handle this case for now
|
||||
raise NotImplementedError
|
||||
|
||||
return self.get_text_features(input_ids, positions, inputs_embeds)
|
||||
text_features = self.get_text_features(input_ids, positions, inputs_embeds)
|
||||
return merge_dual_encoder_text_and_vision(
|
||||
text_features, vision_embeds, self._mm_token_mask
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(
|
||||
|
||||
Reference in New Issue
Block a user