[Core] Dynamic image size support for VLMs (#5276)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: ywang96 <ywang@roblox.com> Co-authored-by: xwjiang2010 <87673679+xwjiang2010@users.noreply.github.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
This commit is contained in:
co-authored by
Xiaowei Jiang
ywang96
xwjiang2010
Roger Wang
parent
482045ee77
commit
9831aec49f
@@ -1,4 +1,5 @@
|
||||
from .base import MultiModalDataDict, MultiModalPlugin
|
||||
from .base import (BatchedTensors, MultiModalDataDict, MultiModalInputs,
|
||||
MultiModalPlugin)
|
||||
from .registry import MultiModalRegistry
|
||||
|
||||
MULTIMODAL_REGISTRY = MultiModalRegistry()
|
||||
@@ -11,8 +12,10 @@ See also:
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"BatchedTensors",
|
||||
"MultiModalDataDict",
|
||||
"MultiModalInputs",
|
||||
"MultiModalPlugin",
|
||||
"MULTIMODAL_REGISTRY",
|
||||
"MultiModalRegistry",
|
||||
"MultiModalDataDict",
|
||||
]
|
||||
|
||||
+85
-16
@@ -1,23 +1,90 @@
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import (TYPE_CHECKING, Any, Callable, Dict, Optional, Type,
|
||||
TypedDict, TypeVar, Union)
|
||||
from collections import UserDict, defaultdict
|
||||
from typing import (Any, Callable, Dict, List, Optional, Type, TypedDict,
|
||||
TypeVar, Union)
|
||||
|
||||
import torch
|
||||
import torch.types
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.inputs import InputContext
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
N = TypeVar("N", bound=Type["nn.Module"])
|
||||
BatchedTensors = Union[torch.Tensor, List[torch.Tensor]]
|
||||
"""
|
||||
If each input tensor in the batch has the same size, this is a single batched
|
||||
tensor; otherwise, this is a list of tensors with one element per batch.
|
||||
"""
|
||||
|
||||
if sys.version_info < (3, 9):
|
||||
# UserDict cannot be subscripted
|
||||
class _MultiModalInputsBase(UserDict):
|
||||
pass
|
||||
else:
|
||||
|
||||
class _MultiModalInputsBase(UserDict[str, torch.Tensor]):
|
||||
pass
|
||||
|
||||
|
||||
class MultiModalInputs(_MultiModalInputsBase):
|
||||
"""
|
||||
A dictionary that represents the keyword arguments to
|
||||
:meth:`~torch.nn.Module.forward`.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def try_concat(
|
||||
tensors: List[torch.Tensor],
|
||||
*,
|
||||
device: torch.types.Device,
|
||||
) -> BatchedTensors:
|
||||
# Avoid initializing CUDA too early
|
||||
import torch
|
||||
|
||||
unbatched_shape = tensors[0].shape[1:]
|
||||
|
||||
for tensor in tensors:
|
||||
if tensor.shape[1:] != unbatched_shape:
|
||||
return [
|
||||
tensor.squeeze(0).to(device=device) for tensor in tensors
|
||||
]
|
||||
|
||||
return torch.cat(tensors, dim=0).to(device=device)
|
||||
|
||||
@staticmethod
|
||||
def batch(
|
||||
inputs_list: List["MultiModalInputs"],
|
||||
device: torch.types.Device,
|
||||
) -> Dict[str, BatchedTensors]:
|
||||
"""Batch multiple inputs together into a dictionary."""
|
||||
if len(inputs_list) == 0:
|
||||
return {}
|
||||
|
||||
keys = inputs_list[0].keys()
|
||||
|
||||
item_lists: Dict[str, List[torch.Tensor]] = defaultdict(list)
|
||||
|
||||
for inputs in inputs_list:
|
||||
if inputs.keys() != keys:
|
||||
msg = f"Inputs do not share the same keys ({keys})"
|
||||
raise ValueError(msg)
|
||||
|
||||
for k, v in inputs.items():
|
||||
item_lists[k].append(v)
|
||||
|
||||
return {
|
||||
k: MultiModalInputs.try_concat(item_list, device=device)
|
||||
for k, item_list in item_lists.items()
|
||||
}
|
||||
|
||||
|
||||
class MultiModalDataBuiltins(TypedDict, total=False):
|
||||
image: "Image.Image"
|
||||
image: Image.Image
|
||||
|
||||
|
||||
MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
|
||||
@@ -29,12 +96,13 @@ to the model by the corresponding mapper. By default, the mapper of
|
||||
the corresponding plugin with the same modality key is applied.
|
||||
"""
|
||||
|
||||
MultiModalInputMapper = Callable[[InputContext, object], Dict[str,
|
||||
"torch.Tensor"]]
|
||||
MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
|
||||
"""Return a dictionary to be passed as keyword arguments to
|
||||
:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
|
||||
and processors in HuggingFace Transformers."""
|
||||
|
||||
N = TypeVar("N", bound=Type[nn.Module])
|
||||
|
||||
|
||||
class MultiModalPlugin(ABC):
|
||||
"""
|
||||
@@ -48,8 +116,7 @@ class MultiModalPlugin(ABC):
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._input_mappers: Dict[Type["nn.Module"],
|
||||
MultiModalInputMapper] = {}
|
||||
self._input_mappers: Dict[Type[nn.Module], MultiModalInputMapper] = {}
|
||||
|
||||
@abstractmethod
|
||||
def get_data_key(self) -> str:
|
||||
@@ -60,7 +127,7 @@ class MultiModalPlugin(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def _default_input_mapper(self, ctx: InputContext,
|
||||
data: object) -> Dict[str, "torch.Tensor"]:
|
||||
data: object) -> MultiModalInputs:
|
||||
"""Return a dictionary to be passed as keyword arguments to
|
||||
:meth:`~torch.nn.Module.forward`. This is similar in concept to
|
||||
tokenizers and processors in HuggingFace Transformers.
|
||||
@@ -80,6 +147,7 @@ class MultiModalPlugin(ABC):
|
||||
|
||||
See also:
|
||||
:ref:`input_processing_pipeline`
|
||||
:ref:`adding_a_new_multimodal_model`
|
||||
"""
|
||||
|
||||
def wrapper(model_cls: N) -> N:
|
||||
@@ -97,7 +165,7 @@ class MultiModalPlugin(ABC):
|
||||
return wrapper
|
||||
|
||||
def map_input(self, model_config: ModelConfig,
|
||||
data: object) -> Dict[str, "torch.Tensor"]:
|
||||
data: object) -> MultiModalInputs:
|
||||
"""
|
||||
Apply an input mapper to a data passed
|
||||
to the model, transforming the data into a dictionary of model inputs.
|
||||
@@ -106,7 +174,8 @@ class MultiModalPlugin(ABC):
|
||||
|
||||
The model is identified by ``model_config``.
|
||||
|
||||
TODO: Add guide [ref: PR #5276]
|
||||
See also:
|
||||
:ref:`adding_a_new_multimodal_model`
|
||||
"""
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.model_loader import get_model_architecture
|
||||
|
||||
@@ -1,19 +1,102 @@
|
||||
from functools import lru_cache
|
||||
from typing import Dict
|
||||
from typing import List, Optional, Tuple, TypeVar
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.inputs.registry import InputContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.image_processor import get_image_processor
|
||||
from vllm.transformers_utils.tokenizer import get_tokenizer
|
||||
|
||||
from .base import MultiModalPlugin
|
||||
from .base import MultiModalInputs, MultiModalPlugin
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
cached_get_image_processor = lru_cache(get_image_processor)
|
||||
cached_get_tokenizer = lru_cache(get_tokenizer)
|
||||
|
||||
# Utilities for image input processors
|
||||
_T = TypeVar("_T", str, int)
|
||||
|
||||
|
||||
def repeat_and_pad_token(
|
||||
token: _T,
|
||||
*,
|
||||
repeat_count: int = 1,
|
||||
pad_token_left: Optional[_T] = None,
|
||||
pad_token_right: Optional[_T] = None,
|
||||
) -> List[_T]:
|
||||
replacement = [token] * repeat_count
|
||||
if pad_token_left is not None:
|
||||
replacement = [pad_token_left] + replacement
|
||||
if pad_token_right is not None:
|
||||
replacement = replacement + [pad_token_right]
|
||||
|
||||
return replacement
|
||||
|
||||
|
||||
def repeat_and_pad_image_tokens(
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
prompt: Optional[str],
|
||||
prompt_token_ids: List[int],
|
||||
*,
|
||||
image_token_id: int,
|
||||
repeat_count: int = 1,
|
||||
pad_token_left: Optional[int] = None,
|
||||
pad_token_right: Optional[int] = None,
|
||||
) -> Tuple[Optional[str], List[int]]:
|
||||
if prompt is None:
|
||||
new_prompt = None
|
||||
else:
|
||||
image_token_str = tokenizer.decode(image_token_id)
|
||||
pad_token_str_left = (None if pad_token_left is None else
|
||||
tokenizer.decode(pad_token_left))
|
||||
pad_token_str_right = (None if pad_token_right is None else
|
||||
tokenizer.decode(pad_token_right))
|
||||
replacement_str = "".join(
|
||||
repeat_and_pad_token(
|
||||
image_token_str,
|
||||
repeat_count=repeat_count,
|
||||
pad_token_left=pad_token_str_left,
|
||||
pad_token_right=pad_token_str_right,
|
||||
))
|
||||
|
||||
image_token_count = prompt.count(image_token_str)
|
||||
# This is an arbitrary number to distinguish between the two cases
|
||||
if image_token_count > 16:
|
||||
logger.warning(
|
||||
"Please follow the prompt format that is "
|
||||
"documented on HuggingFace which does not involve "
|
||||
"repeating %s tokens.", image_token_str)
|
||||
elif image_token_count > 1:
|
||||
logger.warning("Multiple image input is not supported yet, "
|
||||
"so any extra image tokens will be treated "
|
||||
"as plain text.")
|
||||
|
||||
# The image tokens are removed to be consistent with HuggingFace
|
||||
new_prompt = prompt.replace(image_token_str, replacement_str, 1)
|
||||
|
||||
new_token_ids: List[int] = []
|
||||
for i, token in enumerate(prompt_token_ids):
|
||||
if token == image_token_id:
|
||||
replacement_ids = repeat_and_pad_token(
|
||||
image_token_id,
|
||||
repeat_count=repeat_count,
|
||||
pad_token_left=pad_token_left,
|
||||
pad_token_right=pad_token_right,
|
||||
)
|
||||
new_token_ids.extend(replacement_ids)
|
||||
|
||||
# No need to further scan the list since we only replace once
|
||||
new_token_ids.extend(prompt_token_ids[i + 1:])
|
||||
break
|
||||
else:
|
||||
new_token_ids.append(token)
|
||||
|
||||
return new_prompt, new_token_ids
|
||||
|
||||
|
||||
class ImagePlugin(MultiModalPlugin):
|
||||
@@ -27,7 +110,7 @@ class ImagePlugin(MultiModalPlugin):
|
||||
trust_remote_code=model_config.trust_remote_code)
|
||||
|
||||
def _default_input_mapper(self, ctx: InputContext,
|
||||
data: object) -> Dict[str, torch.Tensor]:
|
||||
data: object) -> MultiModalInputs:
|
||||
model_config = ctx.model_config
|
||||
if isinstance(data, Image.Image):
|
||||
image_processor = self._get_hf_image_processor(model_config)
|
||||
@@ -35,10 +118,15 @@ class ImagePlugin(MultiModalPlugin):
|
||||
raise RuntimeError("No HuggingFace processor is available"
|
||||
"to process the image object")
|
||||
try:
|
||||
return image_processor.preprocess(data, return_tensors="pt") \
|
||||
.to(model_config.dtype).data
|
||||
batch_data = image_processor \
|
||||
.preprocess(data, return_tensors="pt") \
|
||||
.data
|
||||
except Exception:
|
||||
logger.error("Failed to process image (%s)", data)
|
||||
raise
|
||||
|
||||
raise TypeError(f"Invalid type for 'image': {type(data)}")
|
||||
return MultiModalInputs(batch_data)
|
||||
elif isinstance(data, torch.Tensor):
|
||||
raise NotImplementedError("Embeddings input is not supported yet")
|
||||
|
||||
raise TypeError(f"Invalid image type: {type(data)}")
|
||||
|
||||
+22
-11
@@ -1,18 +1,17 @@
|
||||
import functools
|
||||
from typing import Optional, Sequence, Type, TypeVar
|
||||
from typing import Dict, Optional, Sequence
|
||||
|
||||
from torch import nn
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .base import MultiModalDataDict, MultiModalInputMapper, MultiModalPlugin
|
||||
from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
|
||||
MultiModalPlugin)
|
||||
from .image import ImagePlugin
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
N = TypeVar("N", bound=Type[nn.Module])
|
||||
|
||||
|
||||
class MultiModalRegistry:
|
||||
"""
|
||||
@@ -61,7 +60,7 @@ class MultiModalRegistry:
|
||||
return self.register_input_mapper("image", mapper)
|
||||
|
||||
def _process_input(self, key: str, value: object,
|
||||
model_config: ModelConfig):
|
||||
model_config: ModelConfig) -> MultiModalInputs:
|
||||
plugin = self._plugins.get(key)
|
||||
if plugin:
|
||||
return plugin.map_input(model_config, value)
|
||||
@@ -93,16 +92,28 @@ class MultiModalRegistry:
|
||||
"""
|
||||
return self.register_input_mapper("image", mapper)
|
||||
|
||||
def map_input(self, model_config: ModelConfig, data: MultiModalDataDict):
|
||||
def map_input(self, model_config: ModelConfig,
|
||||
data: MultiModalDataDict) -> MultiModalInputs:
|
||||
"""
|
||||
Apply an input mapper to the data passed to the model.
|
||||
|
||||
See :meth:`MultiModalPlugin.map_input` for more details.
|
||||
"""
|
||||
result_list = [
|
||||
self._process_input(k, v, model_config) for k, v in data.items()
|
||||
]
|
||||
return {k: v for d in result_list for k, v in d.items()}
|
||||
merged_dict: Dict[str, torch.Tensor] = {}
|
||||
|
||||
for data_key, data_value in data.items():
|
||||
input_dict = self._process_input(data_key, data_value,
|
||||
model_config)
|
||||
|
||||
for input_key, input_tensor in input_dict.items():
|
||||
if input_key in merged_dict:
|
||||
raise ValueError(f"The input mappers (keys={set(data)}) "
|
||||
f"resulted in a conflicting keyword "
|
||||
f"argument to `forward()`: {input_key}")
|
||||
|
||||
merged_dict[input_key] = input_tensor
|
||||
|
||||
return MultiModalInputs(merged_dict)
|
||||
|
||||
def create_input_mapper(self, model_config: ModelConfig):
|
||||
"""
|
||||
|
||||
+61
-34
@@ -4,11 +4,56 @@ from typing import Optional, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import aiohttp
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
|
||||
from vllm.multimodal.base import MultiModalDataDict
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
|
||||
def _validate_remote_url(url: str, *, name: str):
|
||||
parsed_url = urlparse(url)
|
||||
if parsed_url.scheme not in ["http", "https"]:
|
||||
raise ValueError(f"Invalid '{name}': A valid '{name}' "
|
||||
"must have scheme 'http' or 'https'.")
|
||||
|
||||
|
||||
def _get_request_headers():
|
||||
return {"User-Agent": f"vLLM/{VLLM_VERSION}"}
|
||||
|
||||
|
||||
def _load_image_from_bytes(b: bytes):
|
||||
image = Image.open(BytesIO(b))
|
||||
image.load()
|
||||
return image
|
||||
|
||||
|
||||
def _load_image_from_data_url(image_url: str):
|
||||
# Only split once and assume the second part is the base64 encoded image
|
||||
_, image_base64 = image_url.split(",", 1)
|
||||
return load_image_from_base64(image_base64)
|
||||
|
||||
|
||||
def fetch_image(image_url: str) -> Image.Image:
|
||||
"""Load PIL image from a url or base64 encoded openai GPT4V format"""
|
||||
if image_url.startswith('http'):
|
||||
_validate_remote_url(image_url, name="image_url")
|
||||
|
||||
headers = _get_request_headers()
|
||||
|
||||
with requests.get(url=image_url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
image_raw = response.content
|
||||
image = _load_image_from_bytes(image_raw)
|
||||
|
||||
elif image_url.startswith('data:image'):
|
||||
image = _load_image_from_data_url(image_url)
|
||||
else:
|
||||
raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
|
||||
"with either 'data:image' or 'http'.")
|
||||
|
||||
return image
|
||||
|
||||
|
||||
class ImageFetchAiohttp:
|
||||
@@ -29,34 +74,31 @@ class ImageFetchAiohttp:
|
||||
"""Load PIL image from a url or base64 encoded openai GPT4V format"""
|
||||
|
||||
if image_url.startswith('http'):
|
||||
parsed_url = urlparse(image_url)
|
||||
if parsed_url.scheme not in ["http", "https"]:
|
||||
raise ValueError("Invalid 'image_url': A valid 'image_url' "
|
||||
"must have scheme 'http' or 'https'.")
|
||||
# Avoid circular import
|
||||
from vllm import __version__ as VLLM_VERSION
|
||||
_validate_remote_url(image_url, name="image_url")
|
||||
|
||||
client = cls.get_aiohttp_client()
|
||||
headers = {"User-Agent": f"vLLM/{VLLM_VERSION}"}
|
||||
headers = _get_request_headers()
|
||||
|
||||
async with client.get(url=image_url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
image_raw = await response.read()
|
||||
image = Image.open(BytesIO(image_raw))
|
||||
image = _load_image_from_bytes(image_raw)
|
||||
|
||||
# Only split once and assume the second part is the base64 encoded image
|
||||
elif image_url.startswith('data:image'):
|
||||
image = load_image_from_base64(image_url.split(',', 1)[1])
|
||||
|
||||
image = _load_image_from_data_url(image_url)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid 'image_url': A valid 'image_url' must start "
|
||||
"with either 'data:image' or 'http'.")
|
||||
|
||||
image.load()
|
||||
return image
|
||||
|
||||
|
||||
async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
|
||||
image = await ImageFetchAiohttp.fetch_image(image_url)
|
||||
return {"image": image}
|
||||
|
||||
|
||||
def encode_image_base64(image: Image.Image, format: str = 'JPEG') -> str:
|
||||
"""Encode a pillow image to base64 format."""
|
||||
|
||||
@@ -69,26 +111,11 @@ def encode_image_base64(image: Image.Image, format: str = 'JPEG') -> str:
|
||||
|
||||
def load_image_from_base64(image: Union[bytes, str]) -> Image.Image:
|
||||
"""Load image from base64 format."""
|
||||
return Image.open(BytesIO(base64.b64decode(image)))
|
||||
return _load_image_from_bytes(base64.b64decode(image))
|
||||
|
||||
|
||||
# TODO(ywang96): move this to a model registry for preprocessing vision
|
||||
# language prompts based on the model type.
|
||||
def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
|
||||
config: ModelConfig) -> str:
|
||||
"""Combine image and text prompts for vision language model depending on
|
||||
the model architecture."""
|
||||
|
||||
if config.hf_config.model_type in ("llava", "llava_next"):
|
||||
full_prompt = f"{image_prompt}\n{text_prompt}"
|
||||
elif config.hf_config.model_type == 'phi3_v':
|
||||
full_prompt = f"{image_prompt}<s>\n{text_prompt}"
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported model type: {config.hf_config.model_type}")
|
||||
return full_prompt
|
||||
|
||||
|
||||
async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
|
||||
image = await ImageFetchAiohttp.fetch_image(image_url)
|
||||
return {"image": image}
|
||||
def rescale_image_size(image: Image.Image, size_factor: float) -> Image.Image:
|
||||
"""Rescale the dimensions of an image by a constant factor."""
|
||||
new_width = int(image.width * size_factor)
|
||||
new_height = int(image.height * size_factor)
|
||||
return image.resize((new_width, new_height))
|
||||
|
||||
Reference in New Issue
Block a user