[Misc] Support local image encoding in benchmarks (#43843)

Signed-off-by: xiaoz <Sukra1@outlook.com>
This commit is contained in:
XiaoZ
2026-06-02 15:15:06 +00:00
committed by GitHub
parent 4d93bc35c9
commit 53fa09d085
3 changed files with 238 additions and 15 deletions
+90 -13
View File
@@ -44,6 +44,7 @@ from vllm.lora.request import LoRARequest
from vllm.lora.utils import get_adapter_absolute_path
from vllm.multimodal.audio import get_audio_duration
from vllm.multimodal.image import convert_image_mode
from vllm.multimodal.utils import encode_image_url, fetch_image
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -363,7 +364,11 @@ def lora_path_on_disk(lora_path: str) -> str:
lora_tokenizer_cache: dict[int, TokenizerLike] = {}
def process_image(image: Any) -> Mapping[str, Any]:
def process_image(
image: Any,
*,
ensure_client_side_data: bool = False,
) -> Mapping[str, Any]:
"""
Process a single image input and return a multimedia content dictionary.
@@ -380,6 +385,9 @@ def process_image(image: Any) -> Mapping[str, Any]:
encoded data. - If string starts with "data:image/", treats as base64.
- If string starts with "http://", "https://", or "file://", treats as URL.
- Otherwise treats as local file path and prepends "file://".
- If ensure_client_side_data is True, local and HTTP(S) image references
are loaded and encoded as base64 image data URLs. Existing data:image
URLs are kept unchanged.
- Returns a dictionary with the image URL or base64 data.
Raises:
@@ -403,6 +411,13 @@ def process_image(image: Any) -> Mapping[str, Any]:
if image.startswith(("http://", "https://", "file://", "data:image/"))
else f"file://{image}"
)
if ensure_client_side_data and not image_url.startswith("data:image/"):
try:
fetched_image = fetch_image(image_url)
image_url = encode_image_url(fetched_image)
except Exception as e:
raise ValueError(f"Invalid image URL: {image_url}") from e
return {"type": "image_url", "image_url": {"url": image_url}}
raise ValueError(
@@ -1645,6 +1660,16 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"value overrides potential output length loaded from the dataset. It is "
"used only for custom dataset.",
)
custom_group.add_argument(
"--custom-ensure-client-side-data",
action="store_true",
help=(
"Ensure custom dataset media is sent as client-side data instead "
"of references. For custom_image datasets, this loads local and "
"HTTP(S) images on the benchmark client and encodes them as "
"base64 data URLs. Existing data:image URLs are kept unchanged."
),
)
spec_bench_group = parser.add_argument_group("spec bench dataset options")
spec_bench_group.add_argument(
@@ -2075,6 +2100,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
tokenizer=tokenizer,
output_len=args.custom_output_len,
enable_multimodal_chat=args.enable_multimodal_chat,
ensure_client_side_data=getattr(
args, "custom_ensure_client_side_data", False
),
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
@@ -2627,7 +2655,12 @@ class CustomImageDataset(CustomDataset):
return parts
@classmethod
def _process_content_part(cls, part: dict[str, Any]) -> dict[str, Any]:
def _process_content_part(
cls,
part: dict[str, Any],
*,
ensure_client_side_data: bool = False,
) -> dict[str, Any]:
content_type = part.get("type")
if content_type == "text":
text = part.get("text")
@@ -2638,12 +2671,22 @@ class CustomImageDataset(CustomDataset):
if content_type == "image":
if "image" not in part:
raise ValueError("Image content parts must contain an 'image' field.")
return dict(process_image(part["image"]))
return dict(
process_image(
part["image"],
ensure_client_side_data=ensure_client_side_data,
)
)
if content_type == "image_url":
image_url = part.get("image_url")
if isinstance(image_url, str):
return dict(process_image(image_url))
return dict(
process_image(
image_url,
ensure_client_side_data=ensure_client_side_data,
)
)
if isinstance(image_url, dict):
url = image_url.get("url")
@@ -2652,7 +2695,12 @@ class CustomImageDataset(CustomDataset):
"Image URL content parts must contain a string 'image_url.url'."
)
processed_part = dict(process_image(url))
processed_part = dict(
process_image(
url,
ensure_client_side_data=ensure_client_side_data,
)
)
processed_image_url = dict(processed_part["image_url"])
processed_image_url.update(
{key: value for key, value in image_url.items() if key != "url"}
@@ -2671,9 +2719,17 @@ class CustomImageDataset(CustomDataset):
)
@classmethod
def _process_interleaved_content(cls, content: Any) -> list[dict[str, Any]]:
def _process_interleaved_content(
cls,
content: Any,
*,
ensure_client_side_data: bool = False,
) -> list[dict[str, Any]]:
return [
cls._process_content_part(part)
cls._process_content_part(
part,
ensure_client_side_data=ensure_client_side_data,
)
for part in cls._validate_content_parts(content)
]
@@ -2682,11 +2738,23 @@ class CustomImageDataset(CustomDataset):
return "".join(part["text"] for part in content if part.get("type") == "text")
@staticmethod
def _process_image_files(images: Any) -> dict[str, Any] | list[dict[str, Any]]:
def _process_image_files(
images: Any,
*,
ensure_client_side_data: bool = False,
) -> dict[str, Any] | list[dict[str, Any]]:
if not isinstance(images, list) or not images:
raise ValueError("'image_files' must be a non-empty list.")
mm_content = [dict(process_image(image)) for image in images]
mm_content = [
dict(
process_image(
image,
ensure_client_side_data=ensure_client_side_data,
)
)
for image in images
]
if len(mm_content) == 1:
return mm_content[0]
@@ -2698,6 +2766,7 @@ class CustomImageDataset(CustomDataset):
num_requests: int,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
ensure_client_side_data: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2718,9 +2787,14 @@ class CustomImageDataset(CustomDataset):
break
if "content" in item:
content = self._process_interleaved_content(item["content"])
content = self._process_interleaved_content(
item["content"],
ensure_client_side_data=ensure_client_side_data,
)
text_prompt = self._get_text_from_content(content)
prompt_len = len(tokenizer(text_prompt).input_ids)
prompt_len = (
1 if tokenizer is None else len(tokenizer(text_prompt).input_ids)
)
prompt = (
[{"role": "user", "content": content}]
if enable_multimodal_chat
@@ -2741,8 +2815,11 @@ class CustomImageDataset(CustomDataset):
if not isinstance(prompt, str):
raise ValueError("'prompt' must be a string.")
prompt_len = len(tokenizer(prompt).input_ids)
mm_content = self._process_image_files(item["image_files"])
prompt_len = 1 if tokenizer is None else len(tokenizer(prompt).input_ids)
mm_content = self._process_image_files(
item["image_files"],
ensure_client_side_data=ensure_client_side_data,
)
if enable_multimodal_chat:
# Note: when chat is enabled the request prompt_len is no longer
# accurate and we will be using request output to count the