[CI/Build] Refactor image test assets (#5821)
This commit is contained in:
+70
-41
@@ -1,7 +1,12 @@
|
||||
import contextlib
|
||||
import gc
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypeVar
|
||||
from collections import UserList
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
|
||||
TypeVar)
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -28,21 +33,8 @@ _TEST_DIR = os.path.dirname(__file__)
|
||||
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
|
||||
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
|
||||
|
||||
# Multi modal related
|
||||
# You can use `.buildkite/download-images.sh` to download the assets
|
||||
PIXEL_VALUES_FILES = [
|
||||
os.path.join(_TEST_DIR, "images", filename) for filename in
|
||||
["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
|
||||
]
|
||||
IMAGE_FEATURES_FILES = [
|
||||
os.path.join(_TEST_DIR, "images", filename) for filename in
|
||||
["stop_sign_image_features.pt", "cherry_blossom_image_features.pt"]
|
||||
]
|
||||
IMAGE_FILES = [
|
||||
os.path.join(_TEST_DIR, "images", filename)
|
||||
for filename in ["stop_sign.jpg", "cherry_blossom.jpg"]
|
||||
]
|
||||
assert len(PIXEL_VALUES_FILES) == len(IMAGE_FEATURES_FILES) == len(IMAGE_FILES)
|
||||
_IMAGE_DIR = Path(_TEST_DIR) / "images"
|
||||
"""You can use `.buildkite/download-images.sh` to download the assets."""
|
||||
|
||||
|
||||
def _read_prompts(filename: str) -> List[str]:
|
||||
@@ -51,6 +43,63 @@ def _read_prompts(filename: str) -> List[str]:
|
||||
return prompts
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageAsset:
|
||||
name: Literal["stop_sign", "cherry_blossom"]
|
||||
|
||||
@cached_property
|
||||
def pixel_values(self) -> torch.Tensor:
|
||||
return torch.load(_IMAGE_DIR / f"{self.name}_pixel_values.pt")
|
||||
|
||||
@cached_property
|
||||
def image_features(self) -> torch.Tensor:
|
||||
return torch.load(_IMAGE_DIR / f"{self.name}_image_features.pt")
|
||||
|
||||
@cached_property
|
||||
def pil_image(self) -> Image.Image:
|
||||
return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
|
||||
|
||||
def for_hf(self) -> Image.Image:
|
||||
return self.pil_image
|
||||
|
||||
def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
|
||||
image_input_type = vision_config.image_input_type
|
||||
ImageInputType = VisionLanguageConfig.ImageInputType
|
||||
|
||||
if image_input_type == ImageInputType.IMAGE_FEATURES:
|
||||
return ImageFeatureData(self.image_features)
|
||||
if image_input_type == ImageInputType.PIXEL_VALUES:
|
||||
return ImagePixelData(self.pil_image)
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class _ImageAssetPrompts(TypedDict):
|
||||
stop_sign: str
|
||||
cherry_blossom: str
|
||||
|
||||
|
||||
class _ImageAssets(UserList[ImageAsset]):
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
[ImageAsset("stop_sign"),
|
||||
ImageAsset("cherry_blossom")])
|
||||
|
||||
def prompts(self, prompts: _ImageAssetPrompts) -> List[str]:
|
||||
"""
|
||||
Convenience method to define the prompt for each test image.
|
||||
|
||||
The order of the returned prompts matches the order of the
|
||||
assets when iterating through this object.
|
||||
"""
|
||||
return [prompts["stop_sign"], prompts["cherry_blossom"]]
|
||||
|
||||
|
||||
IMAGE_ASSETS = _ImageAssets()
|
||||
"""Singleton instance of :class:`_ImageAssets`."""
|
||||
|
||||
|
||||
def cleanup():
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
@@ -81,31 +130,6 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
|
||||
cleanup()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def hf_images() -> List[Image.Image]:
|
||||
return [Image.open(filename) for filename in IMAGE_FILES]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def vllm_images(request) -> List[MultiModalData]:
|
||||
vision_language_config = request.getfixturevalue("model_and_config")[1]
|
||||
if vision_language_config.image_input_type == (
|
||||
VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
|
||||
return [
|
||||
ImageFeatureData(torch.load(filename))
|
||||
for filename in IMAGE_FEATURES_FILES
|
||||
]
|
||||
else:
|
||||
return [
|
||||
ImagePixelData(Image.open(filename)) for filename in IMAGE_FILES
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def vllm_image_tensors(request) -> List[torch.Tensor]:
|
||||
return [torch.load(filename) for filename in PIXEL_VALUES_FILES]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_prompts() -> List[str]:
|
||||
prompts = []
|
||||
@@ -122,6 +146,11 @@ def example_long_prompts() -> List[str]:
|
||||
return prompts
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def image_assets() -> _ImageAssets:
|
||||
return IMAGE_ASSETS
|
||||
|
||||
|
||||
_STR_DTYPE_TO_TORCH_DTYPE = {
|
||||
"half": torch.half,
|
||||
"bfloat16": torch.bfloat16,
|
||||
|
||||
Reference in New Issue
Block a user