[Core] Support image processor (#4197)
This commit is contained in:
+24
-21
@@ -15,7 +15,9 @@ from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
|
||||
from vllm.distributed import destroy_model_parallel
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import MultiModalData, SampleLogprobs
|
||||
from vllm.multimodal import MultiModalData
|
||||
from vllm.multimodal.image import ImageFeatureData, ImagePixelData
|
||||
from vllm.sequence import SampleLogprobs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -24,6 +26,7 @@ _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
|
||||
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
|
||||
|
||||
# Multi modal related
|
||||
# You can use `.buildkite/download-images.sh` to download the assets
|
||||
_PIXEL_VALUES_FILES = [
|
||||
os.path.join(_TEST_DIR, "images", filename) for filename in
|
||||
["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
|
||||
@@ -89,17 +92,23 @@ def hf_images() -> List[Image.Image]:
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def vllm_images(request) -> "torch.Tensor":
|
||||
def vllm_images(request) -> List[MultiModalData]:
|
||||
vision_language_config = request.getfixturevalue("model_and_config")[1]
|
||||
all_images = []
|
||||
if vision_language_config.image_input_type == (
|
||||
VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
|
||||
filenames = _IMAGE_FEATURES_FILES
|
||||
return [
|
||||
ImageFeatureData(torch.load(filename))
|
||||
for filename in _IMAGE_FEATURES_FILES
|
||||
]
|
||||
else:
|
||||
filenames = _PIXEL_VALUES_FILES
|
||||
for filename in filenames:
|
||||
all_images.append(torch.load(filename))
|
||||
return torch.concat(all_images, dim=0)
|
||||
return [
|
||||
ImagePixelData(Image.open(filename)) for filename in _IMAGE_FILES
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def vllm_image_tensors(request) -> List[torch.Tensor]:
|
||||
return [torch.load(filename) for filename in _PIXEL_VALUES_FILES]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@@ -392,23 +401,17 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
sampling_params: SamplingParams,
|
||||
images: Optional[torch.Tensor] = None,
|
||||
images: Optional[List[MultiModalData]] = None,
|
||||
) -> List[Tuple[List[List[int]], List[str]]]:
|
||||
if images is not None:
|
||||
assert len(prompts) == len(images)
|
||||
|
||||
prompt_inputs: List[TextPrompt] = []
|
||||
for i, prompt in enumerate(prompts):
|
||||
prompt = TextPrompt(prompt=prompt)
|
||||
if images is not None:
|
||||
prompt["multi_modal_data"] = MultiModalData(
|
||||
type=MultiModalData.Type.IMAGE,
|
||||
data=images[i:i + 1],
|
||||
)
|
||||
inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
|
||||
if images is not None:
|
||||
for i, image in enumerate(images):
|
||||
inputs[i]["multi_modal_data"] = image
|
||||
|
||||
prompt_inputs.append(prompt)
|
||||
|
||||
req_outputs = self.model.generate(prompt_inputs,
|
||||
req_outputs = self.model.generate(inputs,
|
||||
sampling_params=sampling_params)
|
||||
|
||||
outputs: List[Tuple[List[List[int]], List[str]]] = []
|
||||
@@ -447,7 +450,7 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
images: Optional[torch.Tensor] = None,
|
||||
images: Optional[List[MultiModalData]] = None,
|
||||
) -> List[Tuple[List[int], str]]:
|
||||
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
|
||||
outputs = self.generate(prompts, greedy_params, images=images)
|
||||
|
||||
+40
-20
@@ -1,7 +1,7 @@
|
||||
import gc
|
||||
from dataclasses import fields
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Tuple
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -9,36 +9,50 @@ from transformers import AutoTokenizer
|
||||
|
||||
from vllm.config import VisionLanguageConfig
|
||||
|
||||
|
||||
def iter_llava_configs(model_name: str):
|
||||
image_hw_to_feature_size = {
|
||||
(336, 336): 576,
|
||||
}
|
||||
|
||||
for (h, w), f in image_hw_to_feature_size.items():
|
||||
for input_type, input_shape in [
|
||||
(VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
|
||||
(VisionLanguageConfig.ImageInputType.IMAGE_FEATURES, (1, f, 1024)),
|
||||
]:
|
||||
yield (model_name,
|
||||
VisionLanguageConfig(image_input_type=input_type,
|
||||
image_feature_size=f,
|
||||
image_token_id=32000,
|
||||
image_input_shape=input_shape,
|
||||
image_processor=model_name,
|
||||
image_processor_revision=None))
|
||||
|
||||
|
||||
model_and_vl_config = [
|
||||
("llava-hf/llava-1.5-7b-hf",
|
||||
VisionLanguageConfig(
|
||||
image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
|
||||
image_feature_size=576,
|
||||
image_token_id=32000,
|
||||
image_input_shape=(1, 3, 336, 336))),
|
||||
("llava-hf/llava-1.5-7b-hf",
|
||||
VisionLanguageConfig(
|
||||
image_input_type=VisionLanguageConfig.ImageInputType.IMAGE_FEATURES,
|
||||
image_feature_size=576,
|
||||
image_token_id=32000,
|
||||
image_input_shape=(1, 576, 1024)))
|
||||
*iter_llava_configs("llava-hf/llava-1.5-7b-hf"),
|
||||
# Not enough memory
|
||||
# *iter_llava_configs("llava-hf/llava-1.5-13b-hf"),
|
||||
]
|
||||
|
||||
|
||||
def as_dict(vision_language_config: VisionLanguageConfig) -> Dict:
|
||||
def as_dict(vlm_config: VisionLanguageConfig) -> Dict[str, Any]:
|
||||
"""Flatten vision language config to pure args.
|
||||
|
||||
Compatible with what llm entrypoint expects.
|
||||
"""
|
||||
result = {}
|
||||
for field in fields(vision_language_config):
|
||||
value = getattr(vision_language_config, field.name)
|
||||
for field in fields(vlm_config):
|
||||
value = getattr(vlm_config, field.name)
|
||||
if isinstance(value, Enum):
|
||||
result[field.name] = value.name.lower()
|
||||
elif isinstance(value, tuple):
|
||||
result[field.name] = ",".join([str(item) for item in value])
|
||||
else:
|
||||
result[field.name] = value
|
||||
|
||||
result["disable_image_processor"] = vlm_config.image_processor is None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -67,18 +81,19 @@ def sanitize_vllm_output(vllm_output: Tuple[List[int], str],
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
|
||||
vllm_image_prompts, vllm_images, model_and_config: tuple,
|
||||
dtype: str, max_tokens: int, worker_use_ray: bool) -> None:
|
||||
vllm_image_prompts, vllm_images, model_and_config, dtype: str,
|
||||
max_tokens: int, worker_use_ray: bool) -> None:
|
||||
"""Inference result should be the same between hf and vllm.
|
||||
|
||||
All the image fixtures for the test is under tests/images.
|
||||
For huggingface runner, we provide the raw images as input.
|
||||
For vllm runner, we provide image tensors and corresponding
|
||||
For huggingface runner, we provide the PIL images as input.
|
||||
For vllm runner, we provide MultiModalData objects and corresponding
|
||||
vision language config as input.
|
||||
Note, the text input is also adjusted to abide by vllm contract.
|
||||
The text output is sanitized to be able to compare with hf.
|
||||
"""
|
||||
model_id, vision_language_config = model_and_config
|
||||
|
||||
hf_model = hf_runner(model_id, dtype=dtype)
|
||||
hf_outputs = hf_model.generate_greedy(hf_image_prompts,
|
||||
max_tokens,
|
||||
@@ -88,6 +103,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
|
||||
vllm_model = vllm_runner(model_id,
|
||||
dtype=dtype,
|
||||
worker_use_ray=worker_use_ray,
|
||||
enforce_eager=True,
|
||||
**as_dict(vision_language_config))
|
||||
vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
|
||||
max_tokens,
|
||||
@@ -105,3 +121,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
|
||||
f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
|
||||
assert hf_output_ids == vllm_output_ids, (
|
||||
f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
|
||||
|
||||
|
||||
# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
|
||||
# (Requires multiple GPUs)
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import CLIPImageProcessor
|
||||
|
||||
from vllm.config import ModelConfig, VisionLanguageConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import ImagePixelData
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", ["half", "bfloat16", "float"])
|
||||
def test_clip_image_processor(hf_images, dtype):
|
||||
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
|
||||
IMAGE_HEIGHT = IMAGE_WIDTH = 33
|
||||
|
||||
hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
|
||||
assert isinstance(hf_processor, CLIPImageProcessor)
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=MODEL_NAME,
|
||||
tokenizer=MODEL_NAME,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
)
|
||||
vlm_config = VisionLanguageConfig(
|
||||
image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
|
||||
image_token_id=32000,
|
||||
image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
|
||||
image_feature_size=576,
|
||||
image_processor=MODEL_NAME,
|
||||
image_processor_revision=None,
|
||||
)
|
||||
|
||||
for image in hf_images:
|
||||
hf_result = hf_processor.preprocess(
|
||||
image,
|
||||
return_tensors="np",
|
||||
)
|
||||
vllm_result = MULTIMODAL_REGISTRY.process_input(
|
||||
ImagePixelData(image),
|
||||
model_config=model_config,
|
||||
vlm_config=vlm_config,
|
||||
)
|
||||
|
||||
assert hf_result.keys() == vllm_result.keys()
|
||||
for key, hf_arr in hf_result.items():
|
||||
vllm_arr: np.ndarray = vllm_result[key].numpy()
|
||||
|
||||
assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
|
||||
assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
|
||||
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
|
||||
IMAGE_HEIGHT = IMAGE_WIDTH = 33
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=MODEL_NAME,
|
||||
tokenizer=MODEL_NAME,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
)
|
||||
vlm_config = VisionLanguageConfig(
|
||||
image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
|
||||
image_token_id=32000,
|
||||
image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
|
||||
image_feature_size=576,
|
||||
image_processor=MODEL_NAME,
|
||||
image_processor_revision=None,
|
||||
)
|
||||
|
||||
for image, tensor in zip(hf_images, vllm_image_tensors):
|
||||
image_result = MULTIMODAL_REGISTRY.process_input(
|
||||
ImagePixelData(image),
|
||||
model_config=model_config,
|
||||
vlm_config=vlm_config,
|
||||
)
|
||||
tensor_result = MULTIMODAL_REGISTRY.process_input(
|
||||
ImagePixelData(tensor),
|
||||
model_config=model_config,
|
||||
vlm_config=vlm_config,
|
||||
)
|
||||
|
||||
assert image_result.keys() == tensor_result.keys()
|
||||
for key, image_arr in image_result.items():
|
||||
tensor_arr: np.ndarray = tensor_result[key].numpy()
|
||||
|
||||
assert image_arr.shape == tensor_arr.shape, f"Failed for key={key}"
|
||||
|
||||
# The examples in PR#3042 have slightly different preprocessing from
|
||||
# HuggingFace's LlavaProcessor, causing the test to fail.
|
||||
# assert np.allclose(image_arr, tensor_arr), f"Failed for key={key}"
|
||||
@@ -18,9 +18,10 @@ from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.utils import set_random_seed
|
||||
from vllm.multimodal import MultiModalData
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import Logprob, MultiModalData
|
||||
from vllm.sequence import Logprob
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils import Counter, random_uuid
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import pytest
|
||||
from transformers.image_processing_utils import BaseImageProcessor
|
||||
|
||||
from vllm.transformers_utils.image_processor import get_image_processor
|
||||
|
||||
IMAGE_PROCESSOR_NAMES = [
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
"llava-hf/llava-v1.6-34b-hf",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_name", IMAGE_PROCESSOR_NAMES)
|
||||
def test_image_processor_revision(processor_name: str):
|
||||
# Assume that "main" branch always exists
|
||||
image_processor = get_image_processor(processor_name, revision="main")
|
||||
assert isinstance(image_processor, BaseImageProcessor)
|
||||
|
||||
# Assume that "never" branch always does not exist
|
||||
with pytest.raises(OSError, match='not a valid git identifier'):
|
||||
get_image_processor(processor_name, revision="never")
|
||||
Reference in New Issue
Block a user