From 9a7a273dfe6a89bbe00639fe99b0d61095fbc40a Mon Sep 17 00:00:00 2001 From: Sergei Skvortsov Date: Fri, 15 May 2026 21:01:21 +0100 Subject: [PATCH] Add HumanEval and GSM8K benchmarks to datasets (#42648) Signed-off-by: southfreebird Co-authored-by: Michael Goin --- docs/benchmarking/cli.md | 22 +++++ vllm/benchmarks/datasets/datasets.py | 133 +++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index 1520e92f3d..ea56b00ba2 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -34,6 +34,8 @@ th { | HuggingFace-AIMO | ✅ | ✅ | `AI-MO/aimo-validation-aime`, `AI-MO/NuminaMath-1.5`, `AI-MO/NuminaMath-CoT` | | HuggingFace-Other | ✅ | ✅ | `lmms-lab/LLaVA-OneVision-Data`, `Aeala/ShareGPT_Vicuna_unfiltered` | | HuggingFace-MTBench | ✅ | ✅ | `philschmid/mt-bench` | +| HuggingFace-HumanEval | ✅ | ✅ | `openai/openai_humaneval` | +| HuggingFace-GSM8K | ✅ | ✅ | `openai/gsm8k` | | HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` | | HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` | | Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` | @@ -443,6 +445,26 @@ vllm bench serve \ --num-prompts 80 ``` +`openai/openai_humaneval`: + +``` bash +vllm bench serve \ + --model NousResearch/Hermes-3-Llama-3.1-8B \ + --dataset-name hf \ + --dataset-path openai/openai_humaneval \ + --num-prompts 80 +``` + +`openai/gsm8k`: + +``` bash +vllm bench serve \ + --model NousResearch/Hermes-3-Llama-3.1-8B \ + --dataset-name hf \ + --dataset-path openai/gsm8k \ + --num-prompts 80 +``` + `vdaita/edit_5k_char` or `vdaita/edit_10k_char`: ``` bash diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index e71b4a3135..3e0a5451fd 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -1940,6 +1940,19 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: ): dataset_class = MTBenchDataset args.hf_split = args.hf_split if args.hf_split else "train" + elif ( + args.dataset_path in HumanEvalDataset.SUPPORTED_DATASET_PATHS + or args.hf_name in HumanEvalDataset.SUPPORTED_DATASET_PATHS + ): + dataset_class = HumanEvalDataset + args.hf_split = args.hf_split if args.hf_split else "test" + elif ( + args.dataset_path in GSM8KDataset.SUPPORTED_DATASET_PATHS + or args.hf_name in GSM8KDataset.SUPPORTED_DATASET_PATHS + ): + dataset_class = GSM8KDataset + args.hf_subset = args.hf_subset if args.hf_subset else "main" + args.hf_split = args.hf_split if args.hf_split else "test" elif ( args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS @@ -3122,6 +3135,126 @@ class MTBenchDataset(HuggingFaceDataset): return sampled_requests +# ----------------------------------------------------------------------------- +# HumanEval Dataset Implementation +# ----------------------------------------------------------------------------- + + +class HumanEvalDataset(HuggingFaceDataset): + """ + HumanEvalDataset Dataset. + https://huggingface.co/datasets/openai/openai_humaneval + + We create a single turn dataset for HumanEval. + """ + + DEFAULT_OUTPUT_LEN = 256 + SUPPORTED_DATASET_PATHS = { + "openai/openai_humaneval", + } + + def sample( + self, + tokenizer: TokenizerLike, + num_requests: int, + request_id_prefix: str = "", + no_oversample: bool = False, + output_len: int | None = None, + enable_multimodal_chat: bool = False, + skip_chat_template: bool = False, + **kwargs, + ) -> list[SampleRequest]: + output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN + sampled_requests = [] + + for i, item in enumerate(self.data): + if len(sampled_requests) >= num_requests: + break + prompt = item["prompt"] + + # apply template + if not skip_chat_template: + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + tokenize=False, + ) + + prompt_len = len(tokenizer(prompt).input_ids) + sampled_requests.append( + SampleRequest( + prompt=prompt, + prompt_len=prompt_len, + expected_output_len=output_len, + request_id=request_id_prefix + str(i), + ) + ) + self.maybe_oversample_requests( + sampled_requests, num_requests, request_id_prefix, no_oversample + ) + return sampled_requests + + +# ----------------------------------------------------------------------------- +# GSM8K Dataset Implementation +# ----------------------------------------------------------------------------- + + +class GSM8KDataset(HuggingFaceDataset): + """ + GSM8K Dataset. + https://huggingface.co/datasets/openai/gsm8k + + We create a single turn dataset for GSM8K. + """ + + DEFAULT_OUTPUT_LEN = 256 + SUPPORTED_DATASET_PATHS = { + "openai/gsm8k", + } + + def sample( + self, + tokenizer: TokenizerLike, + num_requests: int, + request_id_prefix: str = "", + no_oversample: bool = False, + output_len: int | None = None, + enable_multimodal_chat: bool = False, + skip_chat_template: bool = False, + **kwargs, + ) -> list[SampleRequest]: + output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN + sampled_requests = [] + + for i, item in enumerate(self.data): + if len(sampled_requests) >= num_requests: + break + prompt = item["question"] + + # apply template + if not skip_chat_template: + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + tokenize=False, + ) + + prompt_len = len(tokenizer(prompt).input_ids) + sampled_requests.append( + SampleRequest( + prompt=prompt, + prompt_len=prompt_len, + expected_output_len=output_len, + request_id=request_id_prefix + str(i), + ) + ) + self.maybe_oversample_requests( + sampled_requests, num_requests, request_id_prefix, no_oversample + ) + return sampled_requests + + # ----------------------------------------------------------------------------- # Blazedit Dataset Implementation # -----------------------------------------------------------------------------