[Bugfix][Multimodal] Encode text in mixed CLIP/SigLIP pooling batches (#53165)
Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4f686e182a
commit
342b8ebd8b
@@ -52,6 +52,31 @@ def _run_test(
|
||||
vllm_model.embed(texts)
|
||||
vllm_model.embed([""], images=images)
|
||||
|
||||
# Mixed image+text batch must not skip the text encoder (#53091).
|
||||
mixed_outputs = vllm_outputs_per_case[2]
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[0][0]],
|
||||
embeddings_1_lst=[mixed_outputs[0]],
|
||||
name_0="text_only",
|
||||
name_1="mixed_text",
|
||||
)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[1][0]],
|
||||
embeddings_1_lst=[mixed_outputs[1]],
|
||||
name_0="image_only",
|
||||
name_1="mixed_image",
|
||||
)
|
||||
empty_text = vllm_model.embed([""])
|
||||
empty_sim = torch.nn.functional.cosine_similarity(
|
||||
torch.tensor(mixed_outputs[0]),
|
||||
torch.tensor(empty_text[0]),
|
||||
dim=0,
|
||||
)
|
||||
assert empty_sim < 0.99, (
|
||||
"Mixed-batch text embedding collapsed to the empty-string vector "
|
||||
f"(cosine={empty_sim:.4f})"
|
||||
)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=CLIPModel) as hf_model:
|
||||
hf_outputs_per_case = []
|
||||
for input_texts, input_images in input_cases:
|
||||
@@ -101,6 +126,7 @@ def test_models(
|
||||
input_cases = [
|
||||
(HF_TEXT_PROMPTS, text_images),
|
||||
(HF_IMAGE_PROMPTS, images),
|
||||
([HF_TEXT_PROMPTS[0], HF_IMAGE_PROMPTS[0]], [None, images[0]]),
|
||||
]
|
||||
|
||||
_run_test(
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.clip import (
|
||||
dual_encoder_has_text_tokens,
|
||||
merge_dual_encoder_text_and_vision,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
def test_text_only_batch_runs_text_encoder():
|
||||
assert dual_encoder_has_text_tokens(False, None)
|
||||
assert dual_encoder_has_text_tokens(False, torch.zeros(4, dtype=torch.bool))
|
||||
|
||||
|
||||
def test_vision_only_batch_skips_text_encoder():
|
||||
is_multimodal = torch.ones(8, dtype=torch.bool)
|
||||
assert not dual_encoder_has_text_tokens(True, is_multimodal)
|
||||
|
||||
|
||||
def test_mixed_batch_runs_text_encoder():
|
||||
# Image tokens followed by a text sequence: the old batch-wide flag
|
||||
# treated this as vision-only and skipped the text encoder (#53091).
|
||||
is_multimodal = torch.tensor([True, True, True, False, False, False])
|
||||
assert dual_encoder_has_text_tokens(True, is_multimodal)
|
||||
|
||||
|
||||
def test_missing_token_mask_with_mm_embeddings_is_vision_only():
|
||||
assert not dual_encoder_has_text_tokens(True, None)
|
||||
|
||||
|
||||
def test_merge_keeps_vision_on_mm_tokens():
|
||||
text = torch.zeros(4, 2)
|
||||
vision = torch.ones(4, 2)
|
||||
is_multimodal = torch.tensor([True, True, False, False])
|
||||
out = merge_dual_encoder_text_and_vision(text, vision, is_multimodal)
|
||||
assert torch.equal(out[:2], vision[:2])
|
||||
assert torch.equal(out[2:], text[2:])
|
||||
@@ -63,6 +63,20 @@ def _run_test(
|
||||
vllm_model.embed(texts)
|
||||
vllm_model.embed([""], images=images)
|
||||
|
||||
mixed_outputs = vllm_outputs_per_case[2]
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[0][0]],
|
||||
embeddings_1_lst=[mixed_outputs[0]],
|
||||
name_0="text_only",
|
||||
name_1="mixed_text",
|
||||
)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[1][0]],
|
||||
embeddings_1_lst=[mixed_outputs[1]],
|
||||
name_0="image_only",
|
||||
name_1="mixed_image",
|
||||
)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=SiglipModel) as hf_model:
|
||||
hf_outputs_per_case = []
|
||||
for input_texts, input_images, tokenization_kwargs in input_cases:
|
||||
@@ -122,6 +136,14 @@ def test_models(
|
||||
},
|
||||
),
|
||||
(HF_IMAGE_PROMPTS, images, {}),
|
||||
(
|
||||
[HF_TEXT_PROMPTS[0], HF_IMAGE_PROMPTS[0]],
|
||||
[None, images[0]],
|
||||
{
|
||||
"padding": "max_length",
|
||||
"max_length": 64,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
_run_test(
|
||||
|
||||
Reference in New Issue
Block a user