[Misc] Support local image encoding in benchmarks (#43843)

Signed-off-by: xiaoz <Sukra1@outlook.com>
This commit is contained in:
XiaoZ
2026-06-02 15:15:06 +00:00
committed by GitHub
parent 4d93bc35c9
commit 53fa09d085
3 changed files with 238 additions and 15 deletions
+8 -2
View File
@@ -246,6 +246,12 @@ Every image listed in "image_files" is added to the request in the listed order
The "image" shorthand accepts the same values as "image_files". The "image_url" field accepts either an OpenAI-style object with a "url" field or a URL string. The "image" shorthand accepts the same values as "image_files". The "image_url" field accepts either an OpenAI-style object with a "url" field or a URL string.
By default, image references are sent to the serving endpoint as provided, with local image paths converted to `file://` URLs.
If the benchmark client should load local and HTTP(S) images before sending requests, pass `--custom-ensure-client-side-data` to encode them as base64 data URLs on the client side.
Existing `data:image/...` URLs are already self-contained and are kept unchanged.
```bash ```bash
# need a model with vision capability here # need a model with vision capability here
vllm serve Qwen/Qwen2-VL-7B-Instruct vllm serve Qwen/Qwen2-VL-7B-Instruct
@@ -253,13 +259,13 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct
```bash ```bash
# run benchmarking script # run benchmarking script
vllm bench serve--save-result --save-detailed \ vllm bench serve --save-result --save-detailed \
--backend openai-chat \ --backend openai-chat \
--model Qwen/Qwen2-VL-7B-Instruct \ --model Qwen/Qwen2-VL-7B-Instruct \
--endpoint /v1/chat/completions \ --endpoint /v1/chat/completions \
--dataset-name custom_image \ --dataset-name custom_image \
--dataset-path <path-to-your-image-data-jsonl> \ --dataset-path <path-to-your-image-data-jsonl> \
--allowed-local-media-path /path/to/image/folder --custom-ensure-client-side-data
``` ```
Note that we need to use the `openai-chat` backend and `/v1/chat/completions` endpoint for multimodal inputs. Note that we need to use the `openai-chat` backend and `/v1/chat/completions` endpoint for multimodal inputs.
@@ -2,11 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json import json
from argparse import Namespace from argparse import Namespace
from io import BytesIO
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
import pybase64 as base64
import pytest import pytest
from PIL import Image
import vllm.benchmarks.datasets.datasets as datasets_module
from vllm.benchmarks.datasets import CustomImageDataset, get_samples from vllm.benchmarks.datasets import CustomImageDataset, get_samples
from vllm.benchmarks.lib.endpoint_request_func import ( from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput, RequestFuncInput,
@@ -33,6 +37,22 @@ def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
f.write(json.dumps(row) + "\n") f.write(json.dumps(row) + "\n")
def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None:
Image.new("RGB", (1, 1), color=color).save(path)
def _decode_data_url(data_url: str) -> tuple[str, bytes]:
prefix, image_base64 = data_url.split(",", 1)
return prefix, base64.b64decode(image_base64)
def _assert_png_data_url(data_url: str) -> None:
prefix, image_bytes = _decode_data_url(data_url)
assert prefix == "data:image/png;base64"
with Image.open(BytesIO(image_bytes)) as image:
image.verify()
def _args_for_custom_image(dataset_path: Path) -> Namespace: def _args_for_custom_image(dataset_path: Path) -> Namespace:
return Namespace( return Namespace(
dataset_name="custom_image", dataset_name="custom_image",
@@ -42,6 +62,7 @@ def _args_for_custom_image(dataset_path: Path) -> Namespace:
num_prompts=2, num_prompts=2,
custom_output_len=32, custom_output_len=32,
enable_multimodal_chat=False, enable_multimodal_chat=False,
custom_ensure_client_side_data=False,
request_id_prefix="req-", request_id_prefix="req-",
no_oversample=False, no_oversample=False,
) )
@@ -230,6 +251,125 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
assert _get_chat_messages(request_input) == sample.prompt assert _get_chat_messages(request_input) == sample.prompt
@pytest.mark.benchmark
def test_custom_image_dataset_encodes_image_media_when_requested(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart b.png"
_write_png(image_a, color=(255, 0, 0))
_write_png(image_b, color=(0, 255, 0))
data_url = "data:image/png;base64,Zm9v"
remote_url = "https://example.com/chart.png"
original_fetch_image = datasets_module.fetch_image
def fake_fetch_image(image_url: str) -> Image.Image:
if image_url == remote_url:
return Image.new("RGB", (1, 1), color=(0, 0, 255))
return original_fetch_image(image_url)
monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image)
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the charts.",
"image_files": [
str(image_a),
image_b.as_uri(),
remote_url,
data_url,
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
assert len(samples) == 1
assert isinstance(samples[0].multi_modal_data, list)
image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data]
_assert_png_data_url(image_urls[0])
_assert_png_data_url(image_urls[1])
_assert_png_data_url(image_urls[2])
assert image_urls[3] == data_url
@pytest.mark.benchmark
def test_custom_image_dataset_encodes_interleaved_image_media(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
_write_png(image_a, color=(255, 0, 0))
_write_png(image_b, color=(0, 255, 0))
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Compare "},
{"type": "image", "image": str(image_a)},
{
"type": "image_url",
"image_url": {
"url": image_b.as_uri(),
"detail": "low",
},
},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
sample = samples[0]
assert isinstance(sample.prompt, list)
_assert_png_data_url(sample.prompt[1]["image_url"]["url"])
_assert_png_data_url(sample.prompt[2]["image_url"]["url"])
assert sample.prompt[2]["image_url"]["detail"] == "low"
@pytest.mark.benchmark
def test_custom_image_dataset_rejects_invalid_image_media(
tmp_path: Path,
) -> None:
invalid_image = tmp_path / "not_an_image.png"
invalid_image.write_text("not an image")
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="Invalid image URL"):
dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
ensure_client_side_data=True,
)
@pytest.mark.benchmark @pytest.mark.benchmark
def test_custom_image_dataset_rejects_invalid_content_part( def test_custom_image_dataset_rejects_invalid_content_part(
tmp_path: Path, tmp_path: Path,
+90 -13
View File
@@ -44,6 +44,7 @@ from vllm.lora.request import LoRARequest
from vllm.lora.utils import get_adapter_absolute_path from vllm.lora.utils import get_adapter_absolute_path
from vllm.multimodal.audio import get_audio_duration from vllm.multimodal.audio import get_audio_duration
from vllm.multimodal.image import convert_image_mode from vllm.multimodal.image import convert_image_mode
from vllm.multimodal.utils import encode_image_url, fetch_image
from vllm.tokenizers import TokenizerLike from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.repo_utils import hf_api from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -363,7 +364,11 @@ def lora_path_on_disk(lora_path: str) -> str:
lora_tokenizer_cache: dict[int, TokenizerLike] = {} lora_tokenizer_cache: dict[int, TokenizerLike] = {}
def process_image(image: Any) -> Mapping[str, Any]: def process_image(
image: Any,
*,
ensure_client_side_data: bool = False,
) -> Mapping[str, Any]:
""" """
Process a single image input and return a multimedia content dictionary. Process a single image input and return a multimedia content dictionary.
@@ -380,6 +385,9 @@ def process_image(image: Any) -> Mapping[str, Any]:
encoded data. - If string starts with "data:image/", treats as base64. encoded data. - If string starts with "data:image/", treats as base64.
- If string starts with "http://", "https://", or "file://", treats as URL. - If string starts with "http://", "https://", or "file://", treats as URL.
- Otherwise treats as local file path and prepends "file://". - Otherwise treats as local file path and prepends "file://".
- If ensure_client_side_data is True, local and HTTP(S) image references
are loaded and encoded as base64 image data URLs. Existing data:image
URLs are kept unchanged.
- Returns a dictionary with the image URL or base64 data. - Returns a dictionary with the image URL or base64 data.
Raises: Raises:
@@ -403,6 +411,13 @@ def process_image(image: Any) -> Mapping[str, Any]:
if image.startswith(("http://", "https://", "file://", "data:image/")) if image.startswith(("http://", "https://", "file://", "data:image/"))
else f"file://{image}" else f"file://{image}"
) )
if ensure_client_side_data and not image_url.startswith("data:image/"):
try:
fetched_image = fetch_image(image_url)
image_url = encode_image_url(fetched_image)
except Exception as e:
raise ValueError(f"Invalid image URL: {image_url}") from e
return {"type": "image_url", "image_url": {"url": image_url}} return {"type": "image_url", "image_url": {"url": image_url}}
raise ValueError( raise ValueError(
@@ -1645,6 +1660,16 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"value overrides potential output length loaded from the dataset. It is " "value overrides potential output length loaded from the dataset. It is "
"used only for custom dataset.", "used only for custom dataset.",
) )
custom_group.add_argument(
"--custom-ensure-client-side-data",
action="store_true",
help=(
"Ensure custom dataset media is sent as client-side data instead "
"of references. For custom_image datasets, this loads local and "
"HTTP(S) images on the benchmark client and encodes them as "
"base64 data URLs. Existing data:image URLs are kept unchanged."
),
)
spec_bench_group = parser.add_argument_group("spec bench dataset options") spec_bench_group = parser.add_argument_group("spec bench dataset options")
spec_bench_group.add_argument( spec_bench_group.add_argument(
@@ -2075,6 +2100,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
tokenizer=tokenizer, tokenizer=tokenizer,
output_len=args.custom_output_len, output_len=args.custom_output_len,
enable_multimodal_chat=args.enable_multimodal_chat, enable_multimodal_chat=args.enable_multimodal_chat,
ensure_client_side_data=getattr(
args, "custom_ensure_client_side_data", False
),
request_id_prefix=args.request_id_prefix, request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample, no_oversample=args.no_oversample,
) )
@@ -2627,7 +2655,12 @@ class CustomImageDataset(CustomDataset):
return parts return parts
@classmethod @classmethod
def _process_content_part(cls, part: dict[str, Any]) -> dict[str, Any]: def _process_content_part(
cls,
part: dict[str, Any],
*,
ensure_client_side_data: bool = False,
) -> dict[str, Any]:
content_type = part.get("type") content_type = part.get("type")
if content_type == "text": if content_type == "text":
text = part.get("text") text = part.get("text")
@@ -2638,12 +2671,22 @@ class CustomImageDataset(CustomDataset):
if content_type == "image": if content_type == "image":
if "image" not in part: if "image" not in part:
raise ValueError("Image content parts must contain an 'image' field.") raise ValueError("Image content parts must contain an 'image' field.")
return dict(process_image(part["image"])) return dict(
process_image(
part["image"],
ensure_client_side_data=ensure_client_side_data,
)
)
if content_type == "image_url": if content_type == "image_url":
image_url = part.get("image_url") image_url = part.get("image_url")
if isinstance(image_url, str): if isinstance(image_url, str):
return dict(process_image(image_url)) return dict(
process_image(
image_url,
ensure_client_side_data=ensure_client_side_data,
)
)
if isinstance(image_url, dict): if isinstance(image_url, dict):
url = image_url.get("url") url = image_url.get("url")
@@ -2652,7 +2695,12 @@ class CustomImageDataset(CustomDataset):
"Image URL content parts must contain a string 'image_url.url'." "Image URL content parts must contain a string 'image_url.url'."
) )
processed_part = dict(process_image(url)) processed_part = dict(
process_image(
url,
ensure_client_side_data=ensure_client_side_data,
)
)
processed_image_url = dict(processed_part["image_url"]) processed_image_url = dict(processed_part["image_url"])
processed_image_url.update( processed_image_url.update(
{key: value for key, value in image_url.items() if key != "url"} {key: value for key, value in image_url.items() if key != "url"}
@@ -2671,9 +2719,17 @@ class CustomImageDataset(CustomDataset):
) )
@classmethod @classmethod
def _process_interleaved_content(cls, content: Any) -> list[dict[str, Any]]: def _process_interleaved_content(
cls,
content: Any,
*,
ensure_client_side_data: bool = False,
) -> list[dict[str, Any]]:
return [ return [
cls._process_content_part(part) cls._process_content_part(
part,
ensure_client_side_data=ensure_client_side_data,
)
for part in cls._validate_content_parts(content) for part in cls._validate_content_parts(content)
] ]
@@ -2682,11 +2738,23 @@ class CustomImageDataset(CustomDataset):
return "".join(part["text"] for part in content if part.get("type") == "text") return "".join(part["text"] for part in content if part.get("type") == "text")
@staticmethod @staticmethod
def _process_image_files(images: Any) -> dict[str, Any] | list[dict[str, Any]]: def _process_image_files(
images: Any,
*,
ensure_client_side_data: bool = False,
) -> dict[str, Any] | list[dict[str, Any]]:
if not isinstance(images, list) or not images: if not isinstance(images, list) or not images:
raise ValueError("'image_files' must be a non-empty list.") raise ValueError("'image_files' must be a non-empty list.")
mm_content = [dict(process_image(image)) for image in images] mm_content = [
dict(
process_image(
image,
ensure_client_side_data=ensure_client_side_data,
)
)
for image in images
]
if len(mm_content) == 1: if len(mm_content) == 1:
return mm_content[0] return mm_content[0]
@@ -2698,6 +2766,7 @@ class CustomImageDataset(CustomDataset):
num_requests: int, num_requests: int,
output_len: int | None = None, output_len: int | None = None,
enable_multimodal_chat: bool = False, enable_multimodal_chat: bool = False,
ensure_client_side_data: bool = False,
request_id_prefix: str = "", request_id_prefix: str = "",
no_oversample: bool = False, no_oversample: bool = False,
**kwargs, **kwargs,
@@ -2718,9 +2787,14 @@ class CustomImageDataset(CustomDataset):
break break
if "content" in item: if "content" in item:
content = self._process_interleaved_content(item["content"]) content = self._process_interleaved_content(
item["content"],
ensure_client_side_data=ensure_client_side_data,
)
text_prompt = self._get_text_from_content(content) text_prompt = self._get_text_from_content(content)
prompt_len = len(tokenizer(text_prompt).input_ids) prompt_len = (
1 if tokenizer is None else len(tokenizer(text_prompt).input_ids)
)
prompt = ( prompt = (
[{"role": "user", "content": content}] [{"role": "user", "content": content}]
if enable_multimodal_chat if enable_multimodal_chat
@@ -2741,8 +2815,11 @@ class CustomImageDataset(CustomDataset):
if not isinstance(prompt, str): if not isinstance(prompt, str):
raise ValueError("'prompt' must be a string.") raise ValueError("'prompt' must be a string.")
prompt_len = len(tokenizer(prompt).input_ids) prompt_len = 1 if tokenizer is None else len(tokenizer(prompt).input_ids)
mm_content = self._process_image_files(item["image_files"]) mm_content = self._process_image_files(
item["image_files"],
ensure_client_side_data=ensure_client_side_data,
)
if enable_multimodal_chat: if enable_multimodal_chat:
# Note: when chat is enabled the request prompt_len is no longer # Note: when chat is enabled the request prompt_len is no longer
# accurate and we will be using request output to count the # accurate and we will be using request output to count the