[Bugfix][Multimodal] Encode text in mixed CLIP/SigLIP pooling batches (#53165)

Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Prudhvi Vuda
2026-08-24 10:45:42 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4f686e182a
commit 342b8ebd8b
5 changed files with 154 additions and 11 deletions
+47 -6
View File
@@ -119,6 +119,38 @@ def _get_vision_feature_select_strategy(pooling_type: str):
) from None
def dual_encoder_has_text_tokens(
has_mm_embeddings: bool,
is_multimodal: torch.Tensor | None,
) -> bool:
"""Whether a dual-encoder pooling batch still contains text tokens.
``embed_input_ids`` used to treat "the batch has any image embeddings" as
"the whole batch is vision-only". Mixed image+text batches then skipped
the text encoder, so completion-style text embeddings collapsed (issue
#53091). Honor the per-token ``is_multimodal`` mask instead.
"""
if not has_mm_embeddings or is_multimodal is None:
return not has_mm_embeddings
return bool((~is_multimodal).any().item())
def merge_dual_encoder_text_and_vision(
text_features: torch.Tensor,
vision_embeds: torch.Tensor,
is_multimodal: torch.Tensor | None,
) -> torch.Tensor:
"""Restore vision embeddings on multimodal tokens after the text encoder.
The model runner builds ``is_multimodal`` on CPU; text and vision features
are on the compute device.
"""
if is_multimodal is None or not bool(is_multimodal.any().item()):
return text_features
mm_mask = is_multimodal.to(device=vision_embeds.device, non_blocking=True)
return torch.where(mm_mask.unsqueeze(-1), vision_embeds, text_features)
class CLIPProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(CLIPConfig)
@@ -809,8 +841,9 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
self.pooler = DispatchPooler.for_embedding(pooler_config)
# Assumes that self.forward is called after self.embed_input_ids
self._is_text_input = True
# Set in embed_input_ids; consumed by forward.
self._has_text_tokens = True
self._mm_token_mask: torch.Tensor | None = None
def get_text_features(
self,
@@ -908,8 +941,12 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
self._is_text_input = (
multimodal_embeddings is None or len(multimodal_embeddings) == 0
has_mm_embeddings = (
multimodal_embeddings is not None and len(multimodal_embeddings) > 0
)
self._mm_token_mask = is_multimodal
self._has_text_tokens = dual_encoder_has_text_tokens(
has_mm_embeddings, is_multimodal
)
# This is to satisfy the type checker for each overload
@@ -943,9 +980,10 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
# Multimodal inputs
assert inputs_embeds is not None
if not self._is_text_input:
if not self._has_text_tokens:
return inputs_embeds
vision_embeds = inputs_embeds
# NOTE: inputs_embeds in model runner has size text_config.projection_dim
# (instead of text_config.hidden_size) to accommodate image embeddings
hidden_size = self.text_embed_dim
@@ -955,7 +993,10 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
# No need to handle this case for now
raise NotImplementedError
return self.get_text_features(input_ids, positions, inputs_embeds)
text_features = self.get_text_features(input_ids, positions, inputs_embeds)
return merge_dual_encoder_text_and_vision(
text_features, vision_embeds, self._mm_token_mask
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(
+17 -5
View File
@@ -57,6 +57,7 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .clip import dual_encoder_has_text_tokens, merge_dual_encoder_text_and_vision
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
from .interfaces_base import default_pooling_type
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
@@ -943,7 +944,9 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
self.pooler = DispatchPooler.for_embedding(pooler_config)
self._is_text_input = True
# Set in embed_input_ids; consumed by forward.
self._has_text_tokens = True
self._mm_token_mask: torch.Tensor | None = None
def get_text_features(
self,
@@ -1088,8 +1091,12 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
self._is_text_input = (
multimodal_embeddings is None or len(multimodal_embeddings) == 0
has_mm_embeddings = (
multimodal_embeddings is not None and len(multimodal_embeddings) > 0
)
self._mm_token_mask = is_multimodal
self._has_text_tokens = dual_encoder_has_text_tokens(
has_mm_embeddings, is_multimodal
)
if multimodal_embeddings is None or is_multimodal is None:
@@ -1121,9 +1128,11 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
raise RuntimeError("PP is not supported for this model")
# Multimodal inputs (image embeddings)
if not self._is_text_input:
assert inputs_embeds is not None
if not self._has_text_tokens:
return inputs_embeds
vision_embeds = inputs_embeds
# NOTE: inputs_embeds in model runner has size text_config.projection_size
# (instead of text_config.hidden_size) to accommodate image embeddings
hidden_size = self.text_embed_dim
@@ -1133,7 +1142,10 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
# No need to handle this case for now
raise NotImplementedError
return self.get_text_features(input_ids, positions, inputs_embeds)
text_features = self.get_text_features(input_ids, positions, inputs_embeds)
return merge_dual_encoder_text_and_vision(
text_features, vision_embeds, self._mm_token_mask
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(