[VLM] Merged multi-modal processor for LLaVA-NeXT (#11682)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,70 +0,0 @@
|
||||
import pytest
|
||||
|
||||
from vllm.inputs import InputContext
|
||||
|
||||
from ....utils import build_model_context
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def get_max_llava_next_image_tokens():
|
||||
from vllm.model_executor.models.llava_next import (
|
||||
get_max_llava_next_image_tokens)
|
||||
return get_max_llava_next_image_tokens
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def dummy_data_for_llava_next():
|
||||
from vllm.model_executor.models.llava_next import dummy_data_for_llava_next
|
||||
return dummy_data_for_llava_next
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gridpoints,expected_max_tokens", [
|
||||
([[336, 336]], 1176),
|
||||
([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]], 2928),
|
||||
])
|
||||
def test_get_max_llava_next_image_tokens(gridpoints, expected_max_tokens,
|
||||
get_max_llava_next_image_tokens):
|
||||
ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
|
||||
|
||||
# Update the config image_grid_pinpoints
|
||||
# and calculate the resulting max tokens
|
||||
ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
|
||||
|
||||
actual_max_tokens = get_max_llava_next_image_tokens(
|
||||
InputContext(ctx.model_config))
|
||||
|
||||
assert expected_max_tokens == actual_max_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gridpoints,expected_size",
|
||||
[
|
||||
# One point; it has to be the largest
|
||||
([[336, 336]], (336, 336)),
|
||||
# Default for most llava next models; the 2x2 tile is the largest
|
||||
([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]],
|
||||
(672, 672)),
|
||||
# If two rectangular gridpoints are the same, the more vertical
|
||||
# one has the higher feature count due to newline features
|
||||
([[336, 672], [672, 336]], (672, 336))
|
||||
])
|
||||
def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
|
||||
gridpoints, expected_size):
|
||||
ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
|
||||
|
||||
# Update the config image_grid_pinpoints
|
||||
ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
|
||||
seq_len = 5000 # bigger than the max feature size for any image
|
||||
|
||||
dummy_data = dummy_data_for_llava_next(
|
||||
ctx,
|
||||
seq_len=seq_len,
|
||||
mm_counts={"image": 1},
|
||||
)
|
||||
seq_data = dummy_data.seq_data
|
||||
mm_data = dummy_data.multi_modal_data
|
||||
|
||||
# The dummy data dims should match the gridpoint with the biggest feat size
|
||||
assert mm_data["image"].height == expected_size[0]
|
||||
assert mm_data["image"].width == expected_size[1]
|
||||
assert len(seq_data.get_token_ids()) >= seq_len
|
||||
@@ -1,118 +0,0 @@
|
||||
from contextlib import nullcontext
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import LlavaNextImageProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MultiModalRegistry
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mm_registry():
|
||||
return MultiModalRegistry()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", ["half", "float"])
|
||||
@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
|
||||
def test_llava_next_image_processor(image_assets, mm_registry, dtype,
|
||||
size_factor):
|
||||
MODEL_NAME = "llava-hf/llava-v1.6-vicuna-7b-hf"
|
||||
|
||||
hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
|
||||
assert isinstance(hf_processor, LlavaNextImageProcessor)
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=MODEL_NAME,
|
||||
task="auto",
|
||||
tokenizer=MODEL_NAME,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
|
||||
mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
for asset in image_assets:
|
||||
image = rescale_image_size(asset.pil_image, size_factor)
|
||||
|
||||
hf_result = hf_processor.preprocess(
|
||||
image,
|
||||
return_tensors="pt",
|
||||
)
|
||||
vllm_result = mm_registry.map_input(
|
||||
model_config,
|
||||
{"image": image},
|
||||
)
|
||||
|
||||
assert hf_result.keys() == vllm_result.keys()
|
||||
for key, hf_tensor in hf_result.items():
|
||||
hf_arr: np.ndarray = hf_tensor.numpy()
|
||||
vllm_arr: np.ndarray = vllm_result[key].numpy()
|
||||
|
||||
assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
|
||||
assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_images", "limit", "is_valid"),
|
||||
[(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
|
||||
(2, 1, False), (2, 2, True)],
|
||||
)
|
||||
def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
|
||||
MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=MODEL_NAME,
|
||||
task="auto",
|
||||
tokenizer=MODEL_NAME,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
seed=0,
|
||||
dtype="half",
|
||||
revision=None,
|
||||
limit_mm_per_prompt={"image": limit},
|
||||
)
|
||||
|
||||
mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
if num_images == 0:
|
||||
mm_inputs = {}
|
||||
elif num_images == 1:
|
||||
mm_inputs = {"image": image}
|
||||
else:
|
||||
mm_inputs = {"image": [image] * num_images}
|
||||
|
||||
with nullcontext() if is_valid else pytest.raises(ValueError):
|
||||
mm_registry.map_input(model_config, mm_inputs)
|
||||
|
||||
|
||||
# NOTE: We don't test zero images since the HF processor doesn't support it
|
||||
@pytest.mark.parametrize("num_images", [1, 2])
|
||||
def test_image_mapper_multi(image_assets, mm_registry, num_images):
|
||||
MODEL_NAME = "llava-hf/llava-v1.6-mistral-7b-hf"
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=MODEL_NAME,
|
||||
task="auto",
|
||||
tokenizer=MODEL_NAME,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
seed=0,
|
||||
dtype="half",
|
||||
revision=None,
|
||||
limit_mm_per_prompt={"image": num_images},
|
||||
)
|
||||
|
||||
mm_registry.init_mm_limits_per_prompt(model_config)
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
mm_inputs = {"image": [image] * num_images}
|
||||
|
||||
mapped_inputs = mm_registry.map_input(model_config, mm_inputs)
|
||||
assert len(mapped_inputs["pixel_values"]) == num_images
|
||||
@@ -1,5 +1,7 @@
|
||||
from contextlib import nullcontext
|
||||
from functools import partial
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -526,6 +528,100 @@ def _rand_audio(
|
||||
return rng.rand(audio_len), sr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize(
|
||||
("limit", "num_supported", "is_valid"),
|
||||
[(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
|
||||
(2, 1, False), (2, 2, True)],
|
||||
)
|
||||
def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
|
||||
limit_mm_per_prompt = {"image": limit}
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_id,
|
||||
task="auto",
|
||||
tokenizer=model_id,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
seed=0,
|
||||
dtype="half",
|
||||
revision=None,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
|
||||
|
||||
processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
|
||||
ctx = InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_get_tokenizer(model_config.tokenizer),
|
||||
)
|
||||
|
||||
processor = processor_factory(ctx, cache=None)
|
||||
|
||||
mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
|
||||
processor.get_supported_mm_limits = mock_supported_mm_limits
|
||||
|
||||
if is_valid:
|
||||
exc_ctx = nullcontext()
|
||||
else:
|
||||
exc_ctx = pytest.raises(ValueError, match="this model only supports")
|
||||
|
||||
with exc_ctx:
|
||||
processor._get_and_validate_dummy_mm_counts()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize(
|
||||
("num_images", "limit", "is_valid"),
|
||||
[(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
|
||||
(2, 1, False), (2, 2, True)],
|
||||
)
|
||||
def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
|
||||
limit_mm_per_prompt = {"image": limit}
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_id,
|
||||
task="auto",
|
||||
tokenizer=model_id,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
seed=0,
|
||||
dtype="half",
|
||||
revision=None,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
|
||||
|
||||
processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
|
||||
ctx = InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_get_tokenizer(model_config.tokenizer),
|
||||
)
|
||||
|
||||
processor = processor_factory(ctx, cache=None)
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
image = _rand_img(rng, min_wh=128, max_wh=256)
|
||||
if num_images == 0:
|
||||
mm_data = {}
|
||||
elif num_images == 1:
|
||||
mm_data = {"image": image}
|
||||
else:
|
||||
mm_data = {"image": [image] * num_images}
|
||||
|
||||
if is_valid:
|
||||
exc_ctx = nullcontext()
|
||||
else:
|
||||
exc_ctx = pytest.raises(ValueError, match=f"passed {num_images} image")
|
||||
|
||||
with exc_ctx:
|
||||
processor.apply(
|
||||
"<image>" * num_images,
|
||||
mm_data=mm_data,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def _test_processing_cache_correctness(
|
||||
model_id: str,
|
||||
modalities: dict[str, bool],
|
||||
@@ -631,6 +727,7 @@ def _test_processing_cache_correctness(
|
||||
("facebook/chameleon-7b", {"image": False}),
|
||||
("adept/fuyu-8b", {"image": False}),
|
||||
("llava-hf/llava-1.5-7b-hf", {"image": True}),
|
||||
("llava-hf/llava-v1.6-mistral-7b-hf", {"image": True}),
|
||||
("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
|
||||
("mistral-community/pixtral-12b", {"image": True}),
|
||||
("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
|
||||
|
||||
@@ -3,13 +3,11 @@ from typing import Optional
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
|
||||
LlavaMultiModalProcessor,
|
||||
get_max_llava_image_tokens)
|
||||
LlavaMultiModalProcessor)
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
|
||||
@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
|
||||
class MyLlava(LlavaForConditionalGeneration):
|
||||
|
||||
|
||||
Reference in New Issue
Block a user