iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Misc] Support interleaved custom image benchmark datasets (#43636)

Browse Source 
Signed-off-by: ThibaultCastells <thib.castells@icloud.com>
This commit is contained in:
Thibault Castells
2026-05-26 03:37:25 -07:00
committed by GitHub
parent 681d7dd38b
commit 5d09f471f4
4 changed files with 471 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/benchmarking/cli.md
+9 -1
View File
@@ -231,13 +231,21 @@ vllm bench serve \
#### Custom Image Dataset
If the image dataset you want to benchmark is not supported yet in vLLM, then you can benchmark on it using `CustomImageDataset`. At inference time, use the option `--dataset-name custom_image`. Your data needs to be in the `.jsonl` format and needs to have "prompt" and "image_files" fields per entry, e.g., `image_data.jsonl`:
If the image dataset you want to benchmark is not supported yet in vLLM, then you can benchmark on it using `CustomImageDataset`. At inference time, use the option `--dataset-name custom_image`. Your data needs to be in the `.jsonl` format and can use "prompt" and "image_files" fields per entry, e.g., `image_data.jsonl`:
```json
{"prompt": "How many animals are present in the given image?", "image_files": ["/path/to/image/folder/horsepony.jpg"]}
{"prompt": "What colour is the bird shown in the image?", "image_files": ["/path/to/image/folder/flycatcher.jpeg"]}
```
Every image listed in "image_files" is added to the request in the listed order after the prompt text. To preserve an interleaved order of text and images, use a "content" field with OpenAI-compatible content parts:
```json
{"content": [{"type": "text", "text": "Compare "}, {"type": "image", "image": "/path/to/image/folder/chart_a.png"}, {"type": "text", "text": " with "}, {"type": "image_url", "image_url": {"url": "/path/to/image/folder/chart_b.png"}}]}
```
The "image" shorthand accepts the same values as "image_files". The "image_url" field accepts either an OpenAI-style object with a "url" field or a URL string.
```bash
# need a model with vision capability here
vllm serve Qwen/Qwen2-VL-7B-Instruct
tests/benchmarks/test_custom_image_dataset.py
+246
View File
@@ -0,0 +1,246 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from argparse import Namespace
from pathlib import Path
from typing import Any
import pytest
from vllm.benchmarks.datasets import CustomImageDataset, get_samples
from vllm.benchmarks.lib.endpoint_request_func import (
RequestFuncInput,
_get_chat_content,
_get_chat_messages,
)
pytestmark = pytest.mark.skip_global_cleanup
class _TokenizedPrompt:
def __init__(self, prompt: str) -> None:
self.input_ids = prompt.split()
class _Tokenizer:
def __call__(self, prompt: str) -> _TokenizedPrompt:
return _TokenizedPrompt(prompt)
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("w") as f:
for row in rows:
f.write(json.dumps(row) + "\n")
def _args_for_custom_image(dataset_path: Path) -> Namespace:
return Namespace(
dataset_name="custom_image",
dataset_path=str(dataset_path),
disable_shuffle=True,
seed=0,
num_prompts=2,
custom_output_len=32,
enable_multimodal_chat=False,
request_id_prefix="req-",
no_oversample=False,
)
@pytest.mark.benchmark
def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
image_c = tmp_path / "chart_c.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the first two charts.",
"image_files": [str(image_a), str(image_b)],
},
{
"content": [
{"type": "text", "text": "Now compare "},
{"type": "image", "image": str(image_c)},
],
},
],
)
samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer())
assert len(samples) == 2
assert samples[0].request_id == "req-0"
assert isinstance(samples[0].multi_modal_data, list)
assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [
f"file://{image_a}",
f"file://{image_b}",
]
assert samples[1].request_id == "req-1"
assert samples[1].multi_modal_data is None
assert isinstance(samples[1].prompt, list)
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
@pytest.mark.benchmark
def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"prompt": "Compare the charts.",
"image_files": [str(image_a), str(image_b)],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
)
assert len(samples) == 1
sample = samples[0]
assert sample.prompt == "Compare the charts."
assert sample.prompt_len == 3
assert isinstance(sample.multi_modal_data, list)
assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [
f"file://{image_a}",
f"file://{image_b}",
]
@pytest.mark.benchmark
def test_custom_image_dataset_preserves_interleaved_content_order(
tmp_path: Path,
) -> None:
image_a = tmp_path / "chart_a.png"
image_b = tmp_path / "chart_b.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Compare "},
{"type": "image", "image": str(image_a)},
{"type": "text", "text": " with "},
{
"type": "image_url",
"image_url": {
"url": str(image_b),
"detail": "low",
},
},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
)
assert len(samples) == 1
sample = samples[0]
assert sample.multi_modal_data is None
assert sample.prompt_len == 2
assert isinstance(sample.prompt, list)
assert [part["type"] for part in sample.prompt] == [
"text",
"image_url",
"text",
"image_url",
]
assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}"
assert sample.prompt[3]["image_url"] == {
"url": f"file://{image_b}",
"detail": "low",
}
request_input = RequestFuncInput(
prompt=sample.prompt,
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=sample.prompt_len,
output_len=32,
model="test-model",
)
assert _get_chat_content(request_input) == sample.prompt
@pytest.mark.benchmark
def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
tmp_path: Path,
) -> None:
image = tmp_path / "chart.png"
jsonl = tmp_path / "images.jsonl"
_write_jsonl(
jsonl,
[
{
"content": [
{"type": "text", "text": "Describe "},
{"type": "image", "image": str(image)},
],
}
],
)
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
enable_multimodal_chat=True,
)
sample = samples[0]
assert sample.multi_modal_data is None
assert sample.prompt == [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe "},
{"type": "image_url", "image_url": {"url": f"file://{image}"}},
],
}
]
request_input = RequestFuncInput(
prompt=sample.prompt,
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=sample.prompt_len,
output_len=32,
model="test-model",
)
assert _get_chat_messages(request_input) == sample.prompt
@pytest.mark.benchmark
def test_custom_image_dataset_rejects_invalid_content_part(
tmp_path: Path,
) -> None:
jsonl = tmp_path / "images.jsonl"
_write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}])
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
)
vllm/benchmarks/datasets/datasets.py
+163 -10
View File
@@ -2323,15 +2323,152 @@ class CustomImageDataset(CustomDataset):
"prompt": "Which country has the most pokemons based on the given graphs?",
"image_files": ["path/to/image.png"],
}
{
"content": [
{"type": "text", "text": "Compare these images: "},
{"type": "image", "image": "path/to/image1.png"},
{"type": "text", "text": " and "},
{"type": "image_url", "image_url": {"url": "path/to/image2.png"}},
],
}
```
NOTE: Only the first image file in "image_files" is used for each sample request.
This is used to benchmark multimodal LLMs on arbitrary datasets.
"""
IS_MULTIMODAL = True
def load_data(self) -> None:
if self.dataset_path is None:
raise ValueError("dataset_path must be provided for loading data.")
self.data: list[dict] = []
if not self.dataset_path.endswith(".jsonl"):
raise NotImplementedError(
"Only JSONL format is supported for CustomImageDataset."
)
with open(self.dataset_path, encoding="utf-8") as f:
for line_number, line in enumerate(f, start=1):
line = line.strip()
if not line:
continue
try:
item = json.loads(line)
except json.JSONDecodeError as e:
raise ValueError(
f"Invalid JSON in custom image dataset line {line_number}: {e}"
) from e
if not isinstance(item, dict):
raise ValueError(
"Each custom image dataset line must contain a JSON object. "
f"Found {type(item)} on line {line_number}."
)
has_legacy_fields = "prompt" in item and "image_files" in item
has_interleaved_content = "content" in item
if not has_legacy_fields and not has_interleaved_content:
raise ValueError(
"Each custom image dataset line must contain either "
"'prompt' and 'image_files' fields, or a 'content' field. "
f"Invalid line: {line_number}."
)
self.data.append(item)
random.seed(self.random_seed)
if not getattr(self, "disable_shuffle", False):
random.shuffle(self.data)
@staticmethod
def _validate_content_parts(content: Any) -> list[dict[str, Any]]:
if not isinstance(content, list):
raise ValueError(
"'content' must be a list of text and image content dictionaries."
)
if not content:
raise ValueError("'content' must contain at least one item.")
parts: list[dict[str, Any]] = []
for part in content:
if not isinstance(part, dict):
raise ValueError(
f"Each item in 'content' must be a dictionary. Found {type(part)}."
)
parts.append(part)
return parts
@classmethod
def _process_content_part(cls, part: dict[str, Any]) -> dict[str, Any]:
content_type = part.get("type")
if content_type == "text":
text = part.get("text")
if not isinstance(text, str):
raise ValueError("Text content parts must contain a string 'text'.")
return {"type": "text", "text": text}
if content_type == "image":
if "image" not in part:
raise ValueError("Image content parts must contain an 'image' field.")
return dict(process_image(part["image"]))
if content_type == "image_url":
image_url = part.get("image_url")
if isinstance(image_url, str):
return dict(process_image(image_url))
if isinstance(image_url, dict):
url = image_url.get("url")
if not isinstance(url, str):
raise ValueError(
"Image URL content parts must contain a string 'image_url.url'."
)
processed_part = dict(process_image(url))
processed_image_url = dict(processed_part["image_url"])
processed_image_url.update(
{key: value for key, value in image_url.items() if key != "url"}
)
processed_part["image_url"] = processed_image_url
return processed_part
raise ValueError(
"Image URL content parts must contain an 'image_url' string "
"or dictionary."
)
raise ValueError(
"Content parts must have type 'text', 'image', or 'image_url'. "
f"Found: {content_type!r}."
)
@classmethod
def _process_interleaved_content(cls, content: Any) -> list[dict[str, Any]]:
return [
cls._process_content_part(part)
for part in cls._validate_content_parts(content)
]
@staticmethod
def _get_text_from_content(content: list[dict[str, Any]]) -> str:
return "".join(part["text"] for part in content if part.get("type") == "text")
@staticmethod
def _process_image_files(images: Any) -> dict[str, Any] | list[dict[str, Any]]:
if not isinstance(images, list) or not images:
raise ValueError("'image_files' must be a non-empty list.")
mm_content = [dict(process_image(image)) for image in images]
if len(mm_content) == 1:
return mm_content[0]
return mm_content
def sample(
self,
tokenizer: TokenizerLike,
@@ -2356,17 +2493,33 @@ class CustomImageDataset(CustomDataset):
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
if "content" in item:
content = self._process_interleaved_content(item["content"])
text_prompt = self._get_text_from_content(content)
prompt_len = len(tokenizer(text_prompt).input_ids)
prompt = (
[{"role": "user", "content": content}]
if enable_multimodal_chat
else content
)
sampled_requests.append(
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
multi_modal_data=None,
request_id=request_id_prefix + str(i),
)
)
continue
prompt = item["prompt"]
if not isinstance(prompt, str):
raise ValueError("'prompt' must be a string.")
prompt_len = len(tokenizer(prompt).input_ids)
images = item["image_files"]
if len(images) > 1:
logger.warning(
"Multiple image files found for sample %d. "
"Only the first image will be used.",
i,
)
mm_content = process_image(images[0])
mm_content = self._process_image_files(item["image_files"])
if enable_multimodal_chat:
# Note: when chat is enabled the request prompt_len is no longer
# accurate and we will be using request output to count the
vllm/benchmarks/lib/endpoint_request_func.py
+53 -11
View File
@@ -66,7 +66,7 @@ class StreamedResponseHandler:
class RequestFuncInput:
"""The input for the request function."""
prompt: str | list[str]
prompt: str | list[str] | list[dict[str, Any]]
api_url: str
prompt_len: int
output_len: int
@@ -268,8 +268,6 @@ def _get_chat_content(
request_func_input: RequestFuncInput,
mm_position: Literal["first", "last"] = "last",
) -> list[dict[str, Any]]:
text_contents = [{"type": "text", "text": request_func_input.prompt}]
mm_contents = []
if request_func_input.multi_modal_content:
mm_content = request_func_input.multi_modal_content
@@ -282,12 +280,60 @@ def _get_chat_content(
"multi_modal_content must be a dict or list[dict] for openai-chat"
)
prompt = request_func_input.prompt
if (
isinstance(prompt, list)
and prompt
and all(
isinstance(item, dict) and isinstance(item.get("type"), str)
for item in prompt
)
):
if mm_position == "first":
return mm_contents + prompt
return prompt + mm_contents
text_contents = [{"type": "text", "text": prompt}]
if mm_position == "first":
return mm_contents + text_contents
return text_contents + mm_contents
def _is_chat_messages(prompt: Any) -> bool:
return (
isinstance(prompt, list)
and prompt
and all(
isinstance(item, dict)
and isinstance(item.get("role"), str)
and isinstance(item.get("content"), (str, list))
for item in prompt
)
)
def _get_chat_messages(
request_func_input: RequestFuncInput,
mm_position: Literal["first", "last"] = "last",
) -> list[dict[str, Any]]:
prompt = request_func_input.prompt
if _is_chat_messages(prompt):
return prompt
return [
{
"role": "user",
"content": _get_chat_content(
request_func_input,
mm_position=mm_position,
),
}
]
async def async_request_openai_chat_completions(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
@@ -297,15 +343,13 @@ async def async_request_openai_chat_completions(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
content = _get_chat_content(request_func_input, mm_position=mm_position)
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
else request_func_input.model,
"messages": [
{"role": "user", "content": content},
],
"messages": messages,
"max_completion_tokens": request_func_input.output_len,
"stream": True,
"stream_options": {
@@ -608,15 +652,13 @@ async def async_request_openai_embeddings_chat(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Embeddings API", "embeddings")
content = _get_chat_content(request_func_input, mm_position=mm_position)
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
else request_func_input.model,
"messages": [
{"role": "user", "content": content},
],
"messages": messages,
# Many embedding models have short context length,
# this is to avoid dropping some of the requests.
"truncate_prompt_tokens": -1,