[Misc] Support interleaved custom image benchmark datasets (#43636)

Signed-off-by: ThibaultCastells <thib.castells@icloud.com>
This commit is contained in:
Thibault Castells
2026-05-26 03:37:25 -07:00
committed by GitHub
parent 681d7dd38b
commit 5d09f471f4
4 changed files with 471 additions and 22 deletions
+53 -11
View File
@@ -66,7 +66,7 @@ class StreamedResponseHandler:
class RequestFuncInput:
"""The input for the request function."""
prompt: str | list[str]
prompt: str | list[str] | list[dict[str, Any]]
api_url: str
prompt_len: int
output_len: int
@@ -268,8 +268,6 @@ def _get_chat_content(
request_func_input: RequestFuncInput,
mm_position: Literal["first", "last"] = "last",
) -> list[dict[str, Any]]:
text_contents = [{"type": "text", "text": request_func_input.prompt}]
mm_contents = []
if request_func_input.multi_modal_content:
mm_content = request_func_input.multi_modal_content
@@ -282,12 +280,60 @@ def _get_chat_content(
"multi_modal_content must be a dict or list[dict] for openai-chat"
)
prompt = request_func_input.prompt
if (
isinstance(prompt, list)
and prompt
and all(
isinstance(item, dict) and isinstance(item.get("type"), str)
for item in prompt
)
):
if mm_position == "first":
return mm_contents + prompt
return prompt + mm_contents
text_contents = [{"type": "text", "text": prompt}]
if mm_position == "first":
return mm_contents + text_contents
return text_contents + mm_contents
def _is_chat_messages(prompt: Any) -> bool:
return (
isinstance(prompt, list)
and prompt
and all(
isinstance(item, dict)
and isinstance(item.get("role"), str)
and isinstance(item.get("content"), (str, list))
for item in prompt
)
)
def _get_chat_messages(
request_func_input: RequestFuncInput,
mm_position: Literal["first", "last"] = "last",
) -> list[dict[str, Any]]:
prompt = request_func_input.prompt
if _is_chat_messages(prompt):
return prompt
return [
{
"role": "user",
"content": _get_chat_content(
request_func_input,
mm_position=mm_position,
),
}
]
async def async_request_openai_chat_completions(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
@@ -297,15 +343,13 @@ async def async_request_openai_chat_completions(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
content = _get_chat_content(request_func_input, mm_position=mm_position)
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
else request_func_input.model,
"messages": [
{"role": "user", "content": content},
],
"messages": messages,
"max_completion_tokens": request_func_input.output_len,
"stream": True,
"stream_options": {
@@ -608,15 +652,13 @@ async def async_request_openai_embeddings_chat(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Embeddings API", "embeddings")
content = _get_chat_content(request_func_input, mm_position=mm_position)
messages = _get_chat_messages(request_func_input, mm_position=mm_position)
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
else request_func_input.model,
"messages": [
{"role": "user", "content": content},
],
"messages": messages,
# Many embedding models have short context length,
# this is to avoid dropping some of the requests.
"truncate_prompt_tokens": -1,