[Misc] Support interleaved custom image benchmark datasets (#43636)
Signed-off-by: ThibaultCastells <thib.castells@icloud.com>
This commit is contained in:
@@ -0,0 +1,246 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.datasets import CustomImageDataset, get_samples
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
RequestFuncInput,
|
||||
_get_chat_content,
|
||||
_get_chat_messages,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class _TokenizedPrompt:
|
||||
def __init__(self, prompt: str) -> None:
|
||||
self.input_ids = prompt.split()
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def __call__(self, prompt: str) -> _TokenizedPrompt:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
with path.open("w") as f:
|
||||
for row in rows:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
|
||||
|
||||
def _args_for_custom_image(dataset_path: Path) -> Namespace:
|
||||
return Namespace(
|
||||
dataset_name="custom_image",
|
||||
dataset_path=str(dataset_path),
|
||||
disable_shuffle=True,
|
||||
seed=0,
|
||||
num_prompts=2,
|
||||
custom_output_len=32,
|
||||
enable_multimodal_chat=False,
|
||||
request_id_prefix="req-",
|
||||
no_oversample=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_get_samples_custom_image_cli_path_supports_multi_image_and_content(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
image_c = tmp_path / "chart_c.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the first two charts.",
|
||||
"image_files": [str(image_a), str(image_b)],
|
||||
},
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Now compare "},
|
||||
{"type": "image", "image": str(image_c)},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer())
|
||||
|
||||
assert len(samples) == 2
|
||||
assert samples[0].request_id == "req-0"
|
||||
assert isinstance(samples[0].multi_modal_data, list)
|
||||
assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [
|
||||
f"file://{image_a}",
|
||||
f"file://{image_b}",
|
||||
]
|
||||
|
||||
assert samples[1].request_id == "req-1"
|
||||
assert samples[1].multi_modal_data is None
|
||||
assert isinstance(samples[1].prompt, list)
|
||||
assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "}
|
||||
assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}"
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"prompt": "Compare the charts.",
|
||||
"image_files": [str(image_a), str(image_b)],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
sample = samples[0]
|
||||
assert sample.prompt == "Compare the charts."
|
||||
assert sample.prompt_len == 3
|
||||
assert isinstance(sample.multi_modal_data, list)
|
||||
assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [
|
||||
f"file://{image_a}",
|
||||
f"file://{image_b}",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_preserves_interleaved_content_order(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image_a = tmp_path / "chart_a.png"
|
||||
image_b = tmp_path / "chart_b.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare "},
|
||||
{"type": "image", "image": str(image_a)},
|
||||
{"type": "text", "text": " with "},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": str(image_b),
|
||||
"detail": "low",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
sample = samples[0]
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt_len == 2
|
||||
assert isinstance(sample.prompt, list)
|
||||
assert [part["type"] for part in sample.prompt] == [
|
||||
"text",
|
||||
"image_url",
|
||||
"text",
|
||||
"image_url",
|
||||
]
|
||||
assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}"
|
||||
assert sample.prompt[3]["image_url"] == {
|
||||
"url": f"file://{image_b}",
|
||||
"detail": "low",
|
||||
}
|
||||
|
||||
request_input = RequestFuncInput(
|
||||
prompt=sample.prompt,
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=sample.prompt_len,
|
||||
output_len=32,
|
||||
model="test-model",
|
||||
)
|
||||
assert _get_chat_content(request_input) == sample.prompt
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
image = tmp_path / "chart.png"
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(
|
||||
jsonl,
|
||||
[
|
||||
{
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe "},
|
||||
{"type": "image", "image": str(image)},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
enable_multimodal_chat=True,
|
||||
)
|
||||
|
||||
sample = samples[0]
|
||||
assert sample.multi_modal_data is None
|
||||
assert sample.prompt == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe "},
|
||||
{"type": "image_url", "image_url": {"url": f"file://{image}"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
request_input = RequestFuncInput(
|
||||
prompt=sample.prompt,
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=sample.prompt_len,
|
||||
output_len=32,
|
||||
model="test-model",
|
||||
)
|
||||
assert _get_chat_messages(request_input) == sample.prompt
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_custom_image_dataset_rejects_invalid_content_part(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
jsonl = tmp_path / "images.jsonl"
|
||||
_write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}])
|
||||
|
||||
dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True)
|
||||
with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"):
|
||||
dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
)
|
||||
Reference in New Issue
Block a user