[Core] Dynamic image size support for VLMs (#5276)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: ywang96 <ywang@roblox.com> Co-authored-by: xwjiang2010 <87673679+xwjiang2010@users.noreply.github.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
This commit is contained in:
co-authored by
Xiaowei Jiang
ywang96
xwjiang2010
Roger Wang
parent
482045ee77
commit
9831aec49f
+85
-16
@@ -1,23 +1,90 @@
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import (TYPE_CHECKING, Any, Callable, Dict, Optional, Type,
|
||||
TypedDict, TypeVar, Union)
|
||||
from collections import UserDict, defaultdict
|
||||
from typing import (Any, Callable, Dict, List, Optional, Type, TypedDict,
|
||||
TypeVar, Union)
|
||||
|
||||
import torch
|
||||
import torch.types
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.inputs import InputContext
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
N = TypeVar("N", bound=Type["nn.Module"])
|
||||
BatchedTensors = Union[torch.Tensor, List[torch.Tensor]]
|
||||
"""
|
||||
If each input tensor in the batch has the same size, this is a single batched
|
||||
tensor; otherwise, this is a list of tensors with one element per batch.
|
||||
"""
|
||||
|
||||
if sys.version_info < (3, 9):
|
||||
# UserDict cannot be subscripted
|
||||
class _MultiModalInputsBase(UserDict):
|
||||
pass
|
||||
else:
|
||||
|
||||
class _MultiModalInputsBase(UserDict[str, torch.Tensor]):
|
||||
pass
|
||||
|
||||
|
||||
class MultiModalInputs(_MultiModalInputsBase):
|
||||
"""
|
||||
A dictionary that represents the keyword arguments to
|
||||
:meth:`~torch.nn.Module.forward`.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def try_concat(
|
||||
tensors: List[torch.Tensor],
|
||||
*,
|
||||
device: torch.types.Device,
|
||||
) -> BatchedTensors:
|
||||
# Avoid initializing CUDA too early
|
||||
import torch
|
||||
|
||||
unbatched_shape = tensors[0].shape[1:]
|
||||
|
||||
for tensor in tensors:
|
||||
if tensor.shape[1:] != unbatched_shape:
|
||||
return [
|
||||
tensor.squeeze(0).to(device=device) for tensor in tensors
|
||||
]
|
||||
|
||||
return torch.cat(tensors, dim=0).to(device=device)
|
||||
|
||||
@staticmethod
|
||||
def batch(
|
||||
inputs_list: List["MultiModalInputs"],
|
||||
device: torch.types.Device,
|
||||
) -> Dict[str, BatchedTensors]:
|
||||
"""Batch multiple inputs together into a dictionary."""
|
||||
if len(inputs_list) == 0:
|
||||
return {}
|
||||
|
||||
keys = inputs_list[0].keys()
|
||||
|
||||
item_lists: Dict[str, List[torch.Tensor]] = defaultdict(list)
|
||||
|
||||
for inputs in inputs_list:
|
||||
if inputs.keys() != keys:
|
||||
msg = f"Inputs do not share the same keys ({keys})"
|
||||
raise ValueError(msg)
|
||||
|
||||
for k, v in inputs.items():
|
||||
item_lists[k].append(v)
|
||||
|
||||
return {
|
||||
k: MultiModalInputs.try_concat(item_list, device=device)
|
||||
for k, item_list in item_lists.items()
|
||||
}
|
||||
|
||||
|
||||
class MultiModalDataBuiltins(TypedDict, total=False):
|
||||
image: "Image.Image"
|
||||
image: Image.Image
|
||||
|
||||
|
||||
MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
|
||||
@@ -29,12 +96,13 @@ to the model by the corresponding mapper. By default, the mapper of
|
||||
the corresponding plugin with the same modality key is applied.
|
||||
"""
|
||||
|
||||
MultiModalInputMapper = Callable[[InputContext, object], Dict[str,
|
||||
"torch.Tensor"]]
|
||||
MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
|
||||
"""Return a dictionary to be passed as keyword arguments to
|
||||
:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
|
||||
and processors in HuggingFace Transformers."""
|
||||
|
||||
N = TypeVar("N", bound=Type[nn.Module])
|
||||
|
||||
|
||||
class MultiModalPlugin(ABC):
|
||||
"""
|
||||
@@ -48,8 +116,7 @@ class MultiModalPlugin(ABC):
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._input_mappers: Dict[Type["nn.Module"],
|
||||
MultiModalInputMapper] = {}
|
||||
self._input_mappers: Dict[Type[nn.Module], MultiModalInputMapper] = {}
|
||||
|
||||
@abstractmethod
|
||||
def get_data_key(self) -> str:
|
||||
@@ -60,7 +127,7 @@ class MultiModalPlugin(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def _default_input_mapper(self, ctx: InputContext,
|
||||
data: object) -> Dict[str, "torch.Tensor"]:
|
||||
data: object) -> MultiModalInputs:
|
||||
"""Return a dictionary to be passed as keyword arguments to
|
||||
:meth:`~torch.nn.Module.forward`. This is similar in concept to
|
||||
tokenizers and processors in HuggingFace Transformers.
|
||||
@@ -80,6 +147,7 @@ class MultiModalPlugin(ABC):
|
||||
|
||||
See also:
|
||||
:ref:`input_processing_pipeline`
|
||||
:ref:`adding_a_new_multimodal_model`
|
||||
"""
|
||||
|
||||
def wrapper(model_cls: N) -> N:
|
||||
@@ -97,7 +165,7 @@ class MultiModalPlugin(ABC):
|
||||
return wrapper
|
||||
|
||||
def map_input(self, model_config: ModelConfig,
|
||||
data: object) -> Dict[str, "torch.Tensor"]:
|
||||
data: object) -> MultiModalInputs:
|
||||
"""
|
||||
Apply an input mapper to a data passed
|
||||
to the model, transforming the data into a dictionary of model inputs.
|
||||
@@ -106,7 +174,8 @@ class MultiModalPlugin(ABC):
|
||||
|
||||
The model is identified by ``model_config``.
|
||||
|
||||
TODO: Add guide [ref: PR #5276]
|
||||
See also:
|
||||
:ref:`adding_a_new_multimodal_model`
|
||||
"""
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.model_loader import get_model_architecture
|
||||
|
||||
Reference in New Issue
Block a user