[Bugfix][MM] Fix JinaVL processing cache order (#53553)
Signed-off-by: JiataiWang <wangjiatai@proton.me> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
735daf8ef6
commit
06ecec7a84
@@ -390,14 +390,6 @@ def test_processing_correctness(
|
|||||||
"OpenVLA uses a custom vLLM processor because its HF remote "
|
"OpenVLA uses a custom vLLM processor because its HF remote "
|
||||||
"processor is incompatible with current Transformers."
|
"processor is incompatible with current Transformers."
|
||||||
)
|
)
|
||||||
if model_id == "jinaai/jina-reranker-m0":
|
|
||||||
pytest.skip(
|
|
||||||
"JinaVL reverses the multi-modal item order inside "
|
|
||||||
"_apply_hf_processor_main to match its document-before-query "
|
|
||||||
"score template, which breaks the positional correspondence "
|
|
||||||
"between multi-modal kwargs and hashes assumed by the "
|
|
||||||
"processing cache. Needs a fix that keeps both consistent."
|
|
||||||
)
|
|
||||||
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
|
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
|
||||||
pytest.skip(
|
pytest.skip(
|
||||||
"Voxtral Realtime doesn't make use of any place-holder "
|
"Voxtral Realtime doesn't make use of any place-holder "
|
||||||
|
|||||||
@@ -4,15 +4,66 @@
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
from packaging.version import Version
|
from packaging.version import Version
|
||||||
|
from PIL import Image
|
||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||||
|
|
||||||
from vllm.model_executor.models.vision import FusedInputNorm
|
from vllm.model_executor.models.vision import FusedInputNorm
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||||
|
from vllm.multimodal.inputs import batched_tensors_equal
|
||||||
|
|
||||||
from ....conftest import ImageTestAssets
|
from ....conftest import ImageTestAssets
|
||||||
from ...utils import build_model_context
|
from ...utils import build_model_context
|
||||||
|
|
||||||
|
|
||||||
|
def test_jina_vl_processing_order() -> None:
|
||||||
|
"""Jina's document-first prompt keeps cached features and hashes aligned."""
|
||||||
|
ctx = build_model_context(
|
||||||
|
"jinaai/jina-reranker-m0",
|
||||||
|
runner="pooling",
|
||||||
|
limit_mm_per_prompt={"image": 2},
|
||||||
|
mm_processor_cache_gb=1,
|
||||||
|
)
|
||||||
|
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||||
|
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||||
|
ctx.model_config,
|
||||||
|
tokenizer=ctx.tokenizer,
|
||||||
|
cache=cache,
|
||||||
|
)
|
||||||
|
|
||||||
|
placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||||
|
query_image = Image.new("RGB", (128, 160), color=(255, 0, 0))
|
||||||
|
document_image = Image.new("RGB", (192, 128), color=(0, 255, 0))
|
||||||
|
|
||||||
|
def process(images: list[Image.Image]):
|
||||||
|
return processor(
|
||||||
|
placeholder * len(images),
|
||||||
|
mm_items=processor.info.parse_mm_data({"image": images}),
|
||||||
|
)
|
||||||
|
|
||||||
|
query = process([query_image])
|
||||||
|
document = process([document_image])
|
||||||
|
pair = process([query_image, document_image])
|
||||||
|
|
||||||
|
pair_items = pair["mm_kwargs"]["image"]
|
||||||
|
assert pair["mm_hashes"]["image"] == [
|
||||||
|
document["mm_hashes"]["image"][0],
|
||||||
|
query["mm_hashes"]["image"][0],
|
||||||
|
]
|
||||||
|
assert batched_tensors_equal(
|
||||||
|
pair_items[0].get_data(),
|
||||||
|
document["mm_kwargs"]["image"][0].get_data(),
|
||||||
|
)
|
||||||
|
assert batched_tensors_equal(
|
||||||
|
pair_items[1].get_data(),
|
||||||
|
query["mm_kwargs"]["image"][0].get_data(),
|
||||||
|
)
|
||||||
|
assert [item.length for item in pair["mm_placeholders"]["image"]] == [
|
||||||
|
document["mm_placeholders"]["image"][0].length,
|
||||||
|
query["mm_placeholders"]["image"][0].length,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("image_mean", "image_std", "rescale_factor", "is_identity"),
|
("image_mean", "image_std", "rescale_factor", "is_identity"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from collections.abc import Iterable, Mapping
|
from collections.abc import Iterable
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from transformers import BatchFeature
|
|
||||||
|
|
||||||
from vllm.config import ModelConfig, VllmConfig
|
from vllm.config import ModelConfig, VllmConfig
|
||||||
from vllm.inputs import TokensPrompt
|
from vllm.inputs import TokensPrompt
|
||||||
@@ -12,7 +11,12 @@ from vllm.logger import init_logger
|
|||||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.parse import MultiModalDataItems
|
from vllm.multimodal.inputs import MultiModalKwargsItems
|
||||||
|
from vllm.multimodal.processing.processor import (
|
||||||
|
MultiModalProcessingInfo,
|
||||||
|
ProcessorInputs,
|
||||||
|
TimingContext,
|
||||||
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
|
|
||||||
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
|
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
|
||||||
@@ -55,28 +59,40 @@ class JinaVLScorer(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||||
def _apply_hf_processor_main(
|
def _cached_apply_hf_processor(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
inputs: ProcessorInputs,
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
timing_ctx: TimingContext,
|
||||||
) -> BatchFeature:
|
) -> MultiModalProcessingInfo:
|
||||||
valid_mm_items = mm_items.select(
|
mm_info = super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
|
||||||
)
|
|
||||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
|
||||||
|
|
||||||
if not mm_data:
|
# Score inputs are query-first, while the prompt template is document-first.
|
||||||
return BatchFeature(dict(passthrough_data))
|
mm_kwargs = MultiModalKwargsItems(
|
||||||
|
{
|
||||||
|
modality: list(reversed(items))
|
||||||
|
for modality, items in mm_info.kwargs.items()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
mm_hashes = {
|
||||||
|
modality: list(reversed(hashes))
|
||||||
|
for modality, hashes in mm_info.hashes.items()
|
||||||
|
}
|
||||||
|
mm_prompt_updates = {
|
||||||
|
modality: [
|
||||||
|
[
|
||||||
|
self._recompute_cached_prompt_update(update, item_idx)
|
||||||
|
for update in updates
|
||||||
|
]
|
||||||
|
for item_idx, updates in enumerate(reversed(item_updates))
|
||||||
|
]
|
||||||
|
for modality, item_updates in mm_info.prompt_updates.items()
|
||||||
|
}
|
||||||
|
|
||||||
for _, value in mm_data.items():
|
return MultiModalProcessingInfo(
|
||||||
value.reverse()
|
kwargs=mm_kwargs,
|
||||||
processed_data = self.info.ctx.call_hf_processor(
|
hashes=mm_hashes,
|
||||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
prompt_updates=mm_prompt_updates,
|
||||||
mm_data,
|
|
||||||
hf_processor_mm_kwargs,
|
|
||||||
)
|
)
|
||||||
processed_data.update(passthrough_data)
|
|
||||||
return processed_data
|
|
||||||
|
|
||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
@MULTIMODAL_REGISTRY.register_processor(
|
||||||
|
|||||||
Reference in New Issue
Block a user