[VLM] Remove image_input_type from VLM config (#5852)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
Cyrus Leung
Roger Wang
parent
2c37540aa6
commit
98d6682cd1
+9
-31
@@ -17,20 +17,18 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
|
||||
AutoTokenizer, BatchEncoding)
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
|
||||
from vllm.config import TokenizerPoolConfig
|
||||
from vllm.distributed import (destroy_distributed_environment,
|
||||
destroy_model_parallel)
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.multimodal import MultiModalData
|
||||
else:
|
||||
# it will call torch.cuda.device_count()
|
||||
MultiModalData = None
|
||||
from vllm.sequence import SampleLogprobs
|
||||
from vllm.utils import cuda_device_count_stateless, is_cpu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# it will call torch.cuda.device_count()
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_TEST_DIR = os.path.dirname(__file__)
|
||||
@@ -51,14 +49,6 @@ def _read_prompts(filename: str) -> List[str]:
|
||||
class ImageAsset:
|
||||
name: Literal["stop_sign", "cherry_blossom"]
|
||||
|
||||
@cached_property
|
||||
def pixel_values(self) -> torch.Tensor:
|
||||
return torch.load(_IMAGE_DIR / f"{self.name}_pixel_values.pt")
|
||||
|
||||
@cached_property
|
||||
def image_features(self) -> torch.Tensor:
|
||||
return torch.load(_IMAGE_DIR / f"{self.name}_image_features.pt")
|
||||
|
||||
@cached_property
|
||||
def pil_image(self) -> Image.Image:
|
||||
return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
|
||||
@@ -66,20 +56,8 @@ class ImageAsset:
|
||||
def for_hf(self) -> Image.Image:
|
||||
return self.pil_image
|
||||
|
||||
def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
|
||||
# don't put this import at the top level
|
||||
# it will call torch.cuda.device_count()
|
||||
from vllm.multimodal.image import ImageFeatureData # noqa: F401
|
||||
from vllm.multimodal.image import ImagePixelData
|
||||
image_input_type = vision_config.image_input_type
|
||||
ImageInputType = VisionLanguageConfig.ImageInputType
|
||||
|
||||
if image_input_type == ImageInputType.IMAGE_FEATURES:
|
||||
return ImageFeatureData(self.image_features)
|
||||
if image_input_type == ImageInputType.PIXEL_VALUES:
|
||||
return ImagePixelData(self.pil_image)
|
||||
|
||||
raise NotImplementedError
|
||||
def for_vllm(self) -> Dict[str, Any]:
|
||||
return {"image": self.pil_image}
|
||||
|
||||
|
||||
class _ImageAssetPrompts(TypedDict):
|
||||
@@ -453,7 +431,7 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
sampling_params: SamplingParams,
|
||||
images: Optional[List[MultiModalData]] = None,
|
||||
images: Optional[List["MultiModalDataDict"]] = None,
|
||||
) -> List[Tuple[List[List[int]], List[str]]]:
|
||||
if images is not None:
|
||||
assert len(prompts) == len(images)
|
||||
@@ -502,7 +480,7 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
images: Optional[List[MultiModalData]] = None,
|
||||
images: Optional[List["MultiModalDataDict"]] = None,
|
||||
) -> List[Tuple[List[int], str]]:
|
||||
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
|
||||
outputs = self.generate(prompts, greedy_params, images=images)
|
||||
|
||||
@@ -39,8 +39,6 @@ def server():
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--image-input-type",
|
||||
"pixel_values",
|
||||
"--image-token-id",
|
||||
"32000",
|
||||
"--image-input-shape",
|
||||
|
||||
@@ -25,17 +25,11 @@ def iter_llava_configs(model_name: str):
|
||||
}
|
||||
|
||||
for (h, w), f in image_hw_to_feature_size.items():
|
||||
for input_type, input_shape in [
|
||||
(VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
|
||||
(VisionLanguageConfig.ImageInputType.IMAGE_FEATURES, (1, f, 1024)),
|
||||
]:
|
||||
yield (model_name,
|
||||
VisionLanguageConfig(image_input_type=input_type,
|
||||
image_feature_size=f,
|
||||
image_token_id=32000,
|
||||
image_input_shape=input_shape,
|
||||
image_processor=model_name,
|
||||
image_processor_revision=None))
|
||||
input_shape = (1, 3, h, w)
|
||||
yield (model_name,
|
||||
VisionLanguageConfig(image_feature_size=f,
|
||||
image_token_id=32000,
|
||||
image_input_shape=input_shape))
|
||||
|
||||
|
||||
model_and_vl_config = [
|
||||
@@ -81,8 +75,8 @@ def run_test(
|
||||
|
||||
All the image fixtures for the test is under tests/images.
|
||||
For huggingface runner, we provide the PIL images as input.
|
||||
For vllm runner, we provide MultiModalData objects and corresponding
|
||||
vision language config as input.
|
||||
For vllm runner, we provide MultiModalDataDict objects
|
||||
and corresponding vision language config as input.
|
||||
Note, the text input is also adjusted to abide by vllm contract.
|
||||
The text output is sanitized to be able to compare with hf.
|
||||
"""
|
||||
@@ -104,7 +98,7 @@ def run_test(
|
||||
# NOTE: `asset.for_vllm` will call `torch.cuda.device_count()`
|
||||
# we must put it inside the vllm_runner context manager
|
||||
# i.e. after creating vLLM instance.
|
||||
vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
|
||||
vllm_images = [asset.for_vllm() for asset in image_assets]
|
||||
|
||||
vllm_image_prompts = [
|
||||
p.replace("<image>", "<image>" * vlm_config.image_feature_size)
|
||||
|
||||
@@ -33,16 +33,13 @@ def iter_llava_next_configs(model_name: str):
|
||||
}
|
||||
|
||||
for (h, w), f in image_hw_to_feature_size.items():
|
||||
for input_type, input_shape in [
|
||||
(VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
|
||||
]:
|
||||
yield (model_name,
|
||||
VisionLanguageConfig(image_input_type=input_type,
|
||||
image_feature_size=f,
|
||||
image_token_id=32000,
|
||||
image_input_shape=input_shape,
|
||||
image_processor=model_name,
|
||||
image_processor_revision=None))
|
||||
input_shape = (1, 3, h, w)
|
||||
yield (model_name,
|
||||
VisionLanguageConfig(
|
||||
image_feature_size=f,
|
||||
image_token_id=32000,
|
||||
image_input_shape=input_shape,
|
||||
))
|
||||
|
||||
|
||||
model_and_vl_config = [
|
||||
@@ -85,14 +82,14 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
|
||||
|
||||
All the image fixtures for the test is under tests/images.
|
||||
For huggingface runner, we provide the PIL images as input.
|
||||
For vllm runner, we provide MultiModalData objects and corresponding
|
||||
vision language config as input.
|
||||
For vllm runner, we provide MultiModalDataDict objects
|
||||
and corresponding vision language config as input.
|
||||
Note, the text input is also adjusted to abide by vllm contract.
|
||||
The text output is sanitized to be able to compare with hf.
|
||||
"""
|
||||
model_id, vlm_config = model_and_config
|
||||
hf_images = [asset.for_hf() for asset in image_assets]
|
||||
vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
|
||||
vllm_images = [asset.for_vllm() for asset in image_assets]
|
||||
|
||||
with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
|
||||
|
||||
@@ -27,16 +27,11 @@ def iter_phi3v_configs(model_name: str):
|
||||
}
|
||||
|
||||
for (h, w), f in image_hw_to_feature_size.items():
|
||||
for input_type, input_shape in [
|
||||
(VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
|
||||
]:
|
||||
yield (model_name,
|
||||
VisionLanguageConfig(image_input_type=input_type,
|
||||
image_feature_size=f,
|
||||
image_token_id=32044,
|
||||
image_input_shape=input_shape,
|
||||
image_processor=model_name,
|
||||
image_processor_revision=None))
|
||||
input_shape = (1, 3, h, w)
|
||||
yield (model_name,
|
||||
VisionLanguageConfig(image_feature_size=f,
|
||||
image_token_id=32044,
|
||||
image_input_shape=input_shape))
|
||||
|
||||
|
||||
model_and_vl_config = [
|
||||
@@ -89,8 +84,8 @@ def run_test(
|
||||
|
||||
All the image fixtures for the test is under tests/images.
|
||||
For huggingface runner, we provide the PIL images as input.
|
||||
For vllm runner, we provide MultiModalData objects and corresponding
|
||||
vision language config as input.
|
||||
For vllm runner, we provide MultiModalDataDict objects
|
||||
and corresponding vision language config as input.
|
||||
Note, the text input is also adjusted to abide by vllm contract.
|
||||
The text output is sanitized to be able to compare with hf.
|
||||
"""
|
||||
@@ -113,7 +108,7 @@ def run_test(
|
||||
# we must put it inside the vllm_runner context manager
|
||||
# i.e. after creating vLLM instance.
|
||||
|
||||
vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
|
||||
vllm_images = [asset.for_vllm() for asset in image_assets]
|
||||
|
||||
vllm_image_prompts = [
|
||||
p.replace("<|image_1|>",
|
||||
|
||||
@@ -2,9 +2,8 @@ import numpy as np
|
||||
import pytest
|
||||
from transformers import CLIPImageProcessor, LlavaNextImageProcessor
|
||||
|
||||
from vllm.config import ModelConfig, VisionLanguageConfig
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import ImagePixelData
|
||||
|
||||
from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
|
||||
|
||||
@@ -12,7 +11,6 @@ from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
|
||||
@pytest.mark.parametrize("dtype", ["half", "float"])
|
||||
def test_clip_image_processor(image_assets, dtype):
|
||||
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
|
||||
IMAGE_HEIGHT = IMAGE_WIDTH = 560
|
||||
|
||||
hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
|
||||
assert isinstance(hf_processor, CLIPImageProcessor)
|
||||
@@ -25,14 +23,6 @@ def test_clip_image_processor(image_assets, dtype):
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
multimodal_config=VisionLanguageConfig(
|
||||
image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
|
||||
image_token_id=32000,
|
||||
image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
|
||||
image_feature_size=576,
|
||||
image_processor=MODEL_NAME,
|
||||
image_processor_revision=None,
|
||||
),
|
||||
)
|
||||
|
||||
for asset in image_assets:
|
||||
@@ -42,7 +32,7 @@ def test_clip_image_processor(image_assets, dtype):
|
||||
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
|
||||
vllm_result = MULTIMODAL_REGISTRY.map_input(
|
||||
model_config,
|
||||
ImagePixelData(asset.pil_image),
|
||||
{"image": asset.pil_image},
|
||||
)
|
||||
|
||||
assert hf_result.keys() == vllm_result.keys()
|
||||
@@ -60,7 +50,6 @@ def test_clip_image_processor(image_assets, dtype):
|
||||
@pytest.mark.parametrize("dtype", ["half", "float"])
|
||||
def test_llava_next_image_processor(image_assets, dtype):
|
||||
MODEL_NAME = "llava-hf/llava-v1.6-34b-hf"
|
||||
IMAGE_HEIGHT = IMAGE_WIDTH = 560
|
||||
|
||||
hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
|
||||
assert isinstance(hf_processor, LlavaNextImageProcessor)
|
||||
@@ -73,14 +62,6 @@ def test_llava_next_image_processor(image_assets, dtype):
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
multimodal_config=VisionLanguageConfig(
|
||||
image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
|
||||
image_token_id=64000,
|
||||
image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
|
||||
image_feature_size=2928,
|
||||
image_processor=MODEL_NAME,
|
||||
image_processor_revision=None,
|
||||
),
|
||||
)
|
||||
|
||||
for asset in image_assets:
|
||||
@@ -90,7 +71,7 @@ def test_llava_next_image_processor(image_assets, dtype):
|
||||
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
|
||||
vllm_result = MULTIMODAL_REGISTRY.map_input(
|
||||
model_config,
|
||||
ImagePixelData(asset.pil_image),
|
||||
{"image": asset.pil_image},
|
||||
)
|
||||
|
||||
assert hf_result.keys() == vllm_result.keys()
|
||||
@@ -107,7 +88,6 @@ def test_llava_next_image_processor(image_assets, dtype):
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
def test_image_pixel_types(image_assets, dtype):
|
||||
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
|
||||
IMAGE_HEIGHT = IMAGE_WIDTH = 560
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=MODEL_NAME,
|
||||
@@ -117,23 +97,15 @@ def test_image_pixel_types(image_assets, dtype):
|
||||
seed=0,
|
||||
dtype=dtype,
|
||||
revision=None,
|
||||
multimodal_config=VisionLanguageConfig(
|
||||
image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
|
||||
image_token_id=32000,
|
||||
image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
|
||||
image_feature_size=576,
|
||||
image_processor=MODEL_NAME,
|
||||
image_processor_revision=None,
|
||||
))
|
||||
|
||||
)
|
||||
for asset in image_assets:
|
||||
image_result = MULTIMODAL_REGISTRY.map_input(
|
||||
model_config,
|
||||
ImagePixelData(asset.pil_image),
|
||||
{"image": asset.pil_image},
|
||||
)
|
||||
tensor_result = MULTIMODAL_REGISTRY.map_input(
|
||||
model_config,
|
||||
ImagePixelData(asset.pixel_values),
|
||||
{"image": asset.pil_image},
|
||||
)
|
||||
|
||||
assert image_result.keys() == tensor_result.keys()
|
||||
|
||||
@@ -11,7 +11,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.async_llm_engine import AsyncLLMEngine
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.utils import set_random_seed
|
||||
from vllm.multimodal import MultiModalData
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import Logprob
|
||||
@@ -91,7 +91,7 @@ class AsyncLLM:
|
||||
prompt_token_ids: Optional[List[List[int]]] = None,
|
||||
use_tqdm: bool = True,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
multi_modal_data: Optional[MultiModalData] = None,
|
||||
multi_modal_data: Optional[MultiModalDataDict] = None,
|
||||
) -> List[RequestOutput]:
|
||||
|
||||
if prompts is None:
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
import pytest
|
||||
from transformers.image_processing_utils import BaseImageProcessor
|
||||
|
||||
from vllm.transformers_utils.image_processor import get_image_processor
|
||||
|
||||
IMAGE_PROCESSOR_NAMES = [
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
"llava-hf/llava-v1.6-34b-hf",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_name", IMAGE_PROCESSOR_NAMES)
|
||||
def test_image_processor_revision(processor_name: str):
|
||||
# Assume that "main" branch always exists
|
||||
image_processor = get_image_processor(processor_name, revision="main")
|
||||
assert isinstance(image_processor, BaseImageProcessor)
|
||||
|
||||
# Assume that "never" branch always does not exist
|
||||
with pytest.raises(OSError, match='not a valid git identifier'):
|
||||
get_image_processor(processor_name, revision="never")
|
||||
Reference in New Issue
Block a user