[Core] Dynamic image size support for VLMs (#5276)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: ywang96 <ywang@roblox.com> Co-authored-by: xwjiang2010 <87673679+xwjiang2010@users.noreply.github.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
This commit is contained in:
co-authored by
Xiaowei Jiang
ywang96
xwjiang2010
Roger Wang
parent
482045ee77
commit
9831aec49f
+81
-33
@@ -1,12 +1,13 @@
|
||||
import contextlib
|
||||
import gc
|
||||
import os
|
||||
import sys
|
||||
from collections import UserList
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple,
|
||||
TypedDict, TypeVar)
|
||||
from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
|
||||
TypeVar)
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -22,13 +23,10 @@ from vllm.distributed import (destroy_distributed_environment,
|
||||
destroy_model_parallel)
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.utils import fetch_image
|
||||
from vllm.sequence import SampleLogprobs
|
||||
from vllm.utils import cuda_device_count_stateless, is_cpu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# it will call torch.cuda.device_count()
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_TEST_DIR = os.path.dirname(__file__)
|
||||
@@ -47,30 +45,42 @@ def _read_prompts(filename: str) -> List[str]:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageAsset:
|
||||
name: Literal["stop_sign", "cherry_blossom"]
|
||||
name: Literal["stop_sign", "cherry_blossom", "boardwalk"]
|
||||
|
||||
@cached_property
|
||||
def pil_image(self) -> Image.Image:
|
||||
if self.name == "boardwalk":
|
||||
return fetch_image(
|
||||
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||
)
|
||||
|
||||
return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
|
||||
|
||||
def for_hf(self) -> Image.Image:
|
||||
return self.pil_image
|
||||
|
||||
def for_vllm(self) -> Dict[str, Any]:
|
||||
return {"image": self.pil_image}
|
||||
|
||||
|
||||
class _ImageAssetPrompts(TypedDict):
|
||||
stop_sign: str
|
||||
cherry_blossom: str
|
||||
boardwalk: str
|
||||
|
||||
|
||||
class _ImageAssets(UserList):
|
||||
if sys.version_info < (3, 9):
|
||||
# UserList cannot be subscripted
|
||||
class _ImageAssetsBase(UserList):
|
||||
pass
|
||||
else:
|
||||
|
||||
class _ImageAssetsBase(UserList[ImageAsset]):
|
||||
pass
|
||||
|
||||
|
||||
class _ImageAssets(_ImageAssetsBase):
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
[ImageAsset("stop_sign"),
|
||||
ImageAsset("cherry_blossom")])
|
||||
super().__init__([
|
||||
ImageAsset("stop_sign"),
|
||||
ImageAsset("cherry_blossom"),
|
||||
ImageAsset("boardwalk")
|
||||
])
|
||||
|
||||
def prompts(self, prompts: _ImageAssetPrompts) -> List[str]:
|
||||
"""
|
||||
@@ -79,7 +89,10 @@ class _ImageAssets(UserList):
|
||||
The order of the returned prompts matches the order of the
|
||||
assets when iterating through this object.
|
||||
"""
|
||||
return [prompts["stop_sign"], prompts["cherry_blossom"]]
|
||||
return [
|
||||
prompts["stop_sign"], prompts["cherry_blossom"],
|
||||
prompts["boardwalk"]
|
||||
]
|
||||
|
||||
|
||||
IMAGE_ASSETS = _ImageAssets()
|
||||
@@ -220,7 +233,7 @@ class HfRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> List[Tuple[List[List[int]], List[str]]]:
|
||||
if images:
|
||||
assert len(prompts) == len(images)
|
||||
@@ -255,7 +268,7 @@ class HfRunner:
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> List[Tuple[List[int], str]]:
|
||||
outputs = self.generate(prompts,
|
||||
do_sample=False,
|
||||
@@ -291,19 +304,30 @@ class HfRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
**kwargs: Any,
|
||||
) -> List[List[torch.Tensor]]:
|
||||
all_logprobs = []
|
||||
for prompt in prompts:
|
||||
input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
|
||||
all_logprobs: List[List[torch.Tensor]] = []
|
||||
for i, prompt in enumerate(prompts):
|
||||
processor_kwargs: Dict[str, Any] = {
|
||||
"text": prompt,
|
||||
"return_tensors": "pt",
|
||||
}
|
||||
if images is not None and images[i] is not None:
|
||||
processor_kwargs["images"] = images[i]
|
||||
|
||||
inputs = self.processor(**processor_kwargs)
|
||||
|
||||
output = self.model.generate(
|
||||
self.wrap_device(input_ids),
|
||||
**self.wrap_device(inputs),
|
||||
use_cache=True,
|
||||
do_sample=False,
|
||||
max_new_tokens=max_tokens,
|
||||
output_hidden_states=True,
|
||||
return_dict_in_generate=True,
|
||||
**kwargs,
|
||||
)
|
||||
seq_logprobs = []
|
||||
seq_logprobs: List[torch.Tensor] = []
|
||||
for hidden_states in output.hidden_states:
|
||||
last_hidden_states = hidden_states[-1][0]
|
||||
logits = torch.matmul(
|
||||
@@ -323,20 +347,32 @@ class HfRunner:
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
**kwargs: Any,
|
||||
) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
|
||||
all_logprobs: List[List[Dict[int, float]]] = []
|
||||
all_output_ids: List[List[int]] = []
|
||||
all_output_strs: List[str] = []
|
||||
|
||||
for prompt in prompts:
|
||||
input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
|
||||
for i, prompt in enumerate(prompts):
|
||||
processor_kwargs: Dict[str, Any] = {
|
||||
"text": prompt,
|
||||
"return_tensors": "pt",
|
||||
}
|
||||
if images is not None and images[i] is not None:
|
||||
processor_kwargs["images"] = images[i]
|
||||
|
||||
inputs = self.processor(**processor_kwargs)
|
||||
input_ids = inputs.input_ids
|
||||
|
||||
output = self.model.generate(
|
||||
self.wrap_device(input_ids),
|
||||
**self.wrap_device(inputs),
|
||||
use_cache=True,
|
||||
do_sample=False,
|
||||
max_new_tokens=max_tokens,
|
||||
output_hidden_states=True,
|
||||
return_dict_in_generate=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
seq_logprobs: List[torch.Tensor] = []
|
||||
@@ -431,7 +467,7 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
sampling_params: SamplingParams,
|
||||
images: Optional[List["MultiModalDataDict"]] = None,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
) -> List[Tuple[List[List[int]], List[str]]]:
|
||||
if images is not None:
|
||||
assert len(prompts) == len(images)
|
||||
@@ -439,7 +475,7 @@ class VllmRunner:
|
||||
inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
|
||||
if images is not None:
|
||||
for i, image in enumerate(images):
|
||||
inputs[i]["multi_modal_data"] = image
|
||||
inputs[i]["multi_modal_data"] = {"image": image}
|
||||
|
||||
req_outputs = self.model.generate(inputs,
|
||||
sampling_params=sampling_params)
|
||||
@@ -462,10 +498,19 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
sampling_params: SamplingParams,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
|
||||
assert sampling_params.logprobs is not None
|
||||
|
||||
req_outputs = self.model.generate(prompts,
|
||||
if images is not None:
|
||||
assert len(prompts) == len(images)
|
||||
|
||||
inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
|
||||
if images is not None:
|
||||
for i, image in enumerate(images):
|
||||
inputs[i]["multi_modal_data"] = {"image": image}
|
||||
|
||||
req_outputs = self.model.generate(inputs,
|
||||
sampling_params=sampling_params)
|
||||
outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
|
||||
for req_output in req_outputs:
|
||||
@@ -480,7 +525,7 @@ class VllmRunner:
|
||||
self,
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
images: Optional[List["MultiModalDataDict"]] = None,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
) -> List[Tuple[List[int], str]]:
|
||||
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
|
||||
outputs = self.generate(prompts, greedy_params, images=images)
|
||||
@@ -492,11 +537,14 @@ class VllmRunner:
|
||||
prompts: List[str],
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
images: Optional[List[Image.Image]] = None,
|
||||
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
|
||||
greedy_logprobs_params = SamplingParams(temperature=0.0,
|
||||
max_tokens=max_tokens,
|
||||
logprobs=num_logprobs)
|
||||
outputs = self.generate_w_logprobs(prompts, greedy_logprobs_params)
|
||||
outputs = self.generate_w_logprobs(prompts,
|
||||
greedy_logprobs_params,
|
||||
images=images)
|
||||
|
||||
return [(output_ids, output_str, output_logprobs)
|
||||
for output_ids, output_str, output_logprobs in outputs]
|
||||
|
||||
Reference in New Issue
Block a user