From 894a02500b16fceb77d92110d02c9b5127de0e5f Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Mon, 4 May 2026 02:39:10 +0200 Subject: [PATCH] [Bench] Forward --seed to CustomDataset and CustomMMDataset shuffle (#40788) Signed-off-by: akimtsvigun --- tests/benchmarks/test_custom_dataset_seed.py | 77 ++++++++++++++++++++ vllm/benchmarks/datasets/datasets.py | 8 +- 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 tests/benchmarks/test_custom_dataset_seed.py diff --git a/tests/benchmarks/test_custom_dataset_seed.py b/tests/benchmarks/test_custom_dataset_seed.py new file mode 100644 index 0000000000..dac87e6e6d --- /dev/null +++ b/tests/benchmarks/test_custom_dataset_seed.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import argparse +import json +from pathlib import Path + +import pytest +from transformers import AutoTokenizer, PreTrainedTokenizerBase + +from vllm.benchmarks.datasets import get_samples + + +@pytest.fixture(scope="session") +def hf_tokenizer() -> PreTrainedTokenizerBase: + return AutoTokenizer.from_pretrained("gpt2") + + +def _write_jsonl(path: Path, n_rows: int) -> None: + with path.open("w") as f: + for i in range(n_rows): + f.write(json.dumps({"prompt": f"row {i}: unique prompt content."}) + "\n") + + +def _args_for_custom(dataset_path: str, seed: int) -> argparse.Namespace: + return argparse.Namespace( + dataset_name="custom", + dataset_path=dataset_path, + disable_shuffle=False, + num_prompts=30, + custom_output_len=32, + skip_chat_template=True, + no_oversample=False, + seed=seed, + request_id_prefix="", + ) + + +@pytest.mark.benchmark +def test_custom_dataset_seed_propagates( + hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path +) -> None: + """--seed must control the CustomDataset shuffle used by get_samples. + + Without the fix, CustomDataset was instantiated without random_seed, + so its load-time shuffle always used DEFAULT_SEED=0 regardless of + args.seed, causing every run with --dataset-name custom to pick the + same subset of rows from a larger file. + """ + jsonl = tmp_path / "data.jsonl" + _write_jsonl(jsonl, n_rows=60) + + samples_a = get_samples(_args_for_custom(str(jsonl), seed=0), hf_tokenizer) + samples_b = get_samples(_args_for_custom(str(jsonl), seed=42), hf_tokenizer) + + prompts_a = {s.prompt for s in samples_a} + prompts_b = {s.prompt for s in samples_b} + + assert len(prompts_a) == 30 + assert len(prompts_b) == 30 + assert prompts_a != prompts_b + + +@pytest.mark.benchmark +def test_custom_dataset_same_seed_is_deterministic( + hf_tokenizer: PreTrainedTokenizerBase, tmp_path: Path +) -> None: + """Same --seed must yield the same CustomDataset subset.""" + jsonl = tmp_path / "data.jsonl" + _write_jsonl(jsonl, n_rows=60) + + samples_a = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer) + samples_b = get_samples(_args_for_custom(str(jsonl), seed=7), hf_tokenizer) + + prompts_a = [s.prompt for s in samples_a] + prompts_b = [s.prompt for s in samples_b] + + assert prompts_a == prompts_b diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index 419275d2e6..b032c0a0d6 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -1803,7 +1803,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: if args.dataset_name == "custom": dataset = CustomDataset( - dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle + dataset_path=args.dataset_path, + disable_shuffle=args.disable_shuffle, + random_seed=args.seed, ) input_requests = dataset.sample( num_requests=args.num_prompts, @@ -1816,7 +1818,9 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: elif args.dataset_name == "custom_mm": dataset = CustomMMDataset( - dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle + dataset_path=args.dataset_path, + disable_shuffle=args.disable_shuffle, + random_seed=args.seed, ) input_requests = dataset.sample( num_requests=args.num_prompts,