[Misc] support arbitrary MM datasets in spec dec bench (#33486)

Signed-off-by: kkt-cohere <komal@cohere.com>
Signed-off-by: Komal Kumar Teru <162363718+kkt-cohere@users.noreply.github.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Komal Kumar Teru
2026-02-02 08:49:48 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent ab374786c7
commit ba871fb788
3 changed files with 156 additions and 18 deletions
+102
View File
@@ -1335,6 +1335,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"random-rerank",
"hf",
"custom",
"custom_mm",
"prefix_repetition",
"spec_bench",
],
@@ -1363,6 +1364,11 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
action="store_true",
help="Skip applying chat template to prompt for datasets that support it.",
)
parser.add_argument(
"--enable-multimodal-chat",
action="store_true",
help="Enable multimodal chat transformation for datasets that support it.",
)
parser.add_argument(
"--disable-shuffle",
action="store_true",
@@ -1685,6 +1691,19 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
no_oversample=args.no_oversample,
)
elif args.dataset_name == "custom_mm":
dataset = CustomMMDataset(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
input_requests = dataset.sample(
num_requests=args.num_prompts,
tokenizer=tokenizer,
output_len=args.custom_output_len,
enable_multimodal_chat=args.enable_multimodal_chat,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
elif args.dataset_name == "sonnet":
dataset = SonnetDataset(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
@@ -1832,6 +1851,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
num_requests=args.num_prompts,
tokenizer=tokenizer,
output_len=args.hf_output_len,
enable_multimodal_chat=args.enable_multimodal_chat,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
skip_chat_template=args.skip_chat_template,
@@ -1849,6 +1869,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
num_requests=args.num_prompts,
tokenizer=tokenizer,
output_len=args.spec_bench_output_len,
enable_multimodal_chat=args.enable_multimodal_chat,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
),
@@ -1860,6 +1881,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
tokenizer=tokenizer,
num_requests=args.num_prompts,
output_len=args.sharegpt_output_len,
enable_multimodal_chat=args.enable_multimodal_chat,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
),
@@ -1903,6 +1925,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
limit_mm_per_prompt=args.random_mm_limit_mm_per_prompt,
num_mm_items_range_ratio=args.random_mm_num_mm_items_range_ratio,
bucket_config=args.random_mm_bucket_config,
enable_multimodal_chat=args.enable_multimodal_chat,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
),
@@ -2075,6 +2098,85 @@ class CustomDataset(BenchmarkDataset):
return sampled_requests
class CustomMMDataset(CustomDataset):
"""
Implements the Custom MultiModal dataset. Loads data from a JSONL file and generates
sample requests based on conversation turns. E.g.,
```
{
"prompt": "How many red blocks in the given images?",
"image_files": ["path/to/image1.png", "path/to/image2.png"],
}
{
"prompt": "Which country has the most pokemons based on the given graphs?",
"image_files": ["path/to/image.png"],
}
```
NOTE: Only the first image file in "image_files" is used for each sample request.
This is used to benchmark multimodal LLMs on arbitrary datasets.
"""
IS_MULTIMODAL = True
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
) -> list:
# load all data if needed
self.num_available_samples = len(self.data)
if num_requests <= 0:
num_requests = self.num_available_samples
logger.info(
"num_requests is set to 0 or negative, "
"so using all available samples: %d",
num_requests,
)
sampled_requests = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
prompt = item["prompt"]
prompt_len = len(tokenizer(prompt).input_ids)
images = item["image_files"]
if len(images) > 1:
logger.warning(
"Multiple image files found for sample %d. "
"Only the first image will be used.",
i,
)
mm_content = process_image(images[0])
if enable_multimodal_chat:
# Note: when chat is enabled the request prompt_len is no longer
# accurate and we will be using request output to count the
# actual prompt len
prompt = self.apply_multimodal_chat_transformation(prompt, mm_content)
sampled_requests.append(
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=output_len,
multi_modal_data=mm_content,
request_id=request_id_prefix + str(i),
)
)
self.maybe_oversample_requests(
sampled_requests, num_requests, request_id_prefix, no_oversample
)
return sampled_requests
# -----------------------------------------------------------------------------
# Spec Bench Dataset Implementation
# -----------------------------------------------------------------------------