From f0a61bd4386bf0d694cfe848e853f2450462e454 Mon Sep 17 00:00:00 2001 From: JinYan Su <751080330@qq.com> Date: Tue, 15 Sep 2026 07:48:23 +0800 Subject: [PATCH] [Rust][Benchmark] Support HF ShareGPT datasets in multi-turn mode (#51104) Signed-off-by: xiaguan <751080330@qq.com> Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- rust/src/bench/README.md | 1 + rust/src/bench/src/config.rs | 38 +++++++++++++++++++++++++++----- rust/src/bench/src/multi_turn.rs | 26 +++++++++++++++++++--- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/rust/src/bench/README.md b/rust/src/bench/README.md index d4e0c69b80..c8724d6cd6 100644 --- a/rust/src/bench/README.md +++ b/rust/src/bench/README.md @@ -667,6 +667,7 @@ With `--multi-turn`, `--num-prompts` controls the number of **conversations**, n - `--dataset-name random` — synthetic conversations with controllable per-turn token lengths. Auto-sets `min_tokens` to enforce output length without `ignore_eos`. - `--dataset-name sharegpt` — loads all turns (not just the first two); filters for entries with ≥ 2 real turns. +- `--dataset-name hf` — downloads a ShareGPT-format HuggingFace config, then loads it like `sharegpt`. **Prefix sharing** (random dataset): when `--multi-turn-prefix-global-ratio` or `--multi-turn-prefix-conversation-ratio` is > 0, each turn sends a fixed-length message (no history accumulation) composed of a global prefix + per-conversation prefix + unique suffix. The two ratios must sum to < 1.0. diff --git a/rust/src/bench/src/config.rs b/rust/src/bench/src/config.rs index 3a0936b5f4..b5a24d942f 100644 --- a/rust/src/bench/src/config.rs +++ b/rust/src/bench/src/config.rs @@ -549,13 +549,15 @@ impl BenchConfig { )); } - // Normalize and validate min/max turns. ShareGPT only consumes max_turns - // (the loader walks all available turns up to the cap), so the - // min/num/max coupling used for synthetic generation does not apply. - if args.dataset_name == DatasetName::ShareGpt { + // Normalize and validate min/max turns. Conversation datasets only + // consume max_turns (the loader walks all available turns up to the + // cap), so the min/num/max coupling used for synthetic generation + // does not apply. + if matches!(args.dataset_name, DatasetName::ShareGpt | DatasetName::Hf) { if args.multi_turn_max_turns == 1 { return Err(BenchError::Config( - "--multi-turn-max-turns must be at least 2 for ShareGPT multi-turn".into(), + "--multi-turn-max-turns must be at least 2 for ShareGPT-format multi-turn" + .into(), )); } } else { @@ -722,7 +724,7 @@ impl BenchConfig { multi_turn_min_turns, multi_turn_max_turns, sharegpt_multi_turn_max_turns: if args.multi_turn - && args.dataset_name == DatasetName::ShareGpt + && matches!(args.dataset_name, DatasetName::ShareGpt | DatasetName::Hf) && args.multi_turn_max_turns != 0 { Some(args.multi_turn_max_turns) @@ -1067,6 +1069,30 @@ mod tests { assert_eq!(config.sharegpt_multi_turn_max_turns, Some(20)); } + #[test] + fn test_hf_multi_turn_uses_conversation_turn_cap() { + let args = vec![ + "vllm-bench", + "--backend", + "openai-chat", + "--multi-turn", + "--model", + "test-model", + "--dataset-name", + "hf", + "--dataset-path", + "org/sharegpt-dataset", + "--hf-subset", + "sharegpt", + "--multi-turn-max-turns", + "2", + ]; + let args = parse_args(args); + let config = BenchConfig::from_args(&args).unwrap(); + + assert_eq!(config.sharegpt_multi_turn_max_turns, Some(2)); + } + #[test] fn test_sweep_summary_percentiles_default_empty() { let args = base_multi_turn_args(); diff --git a/rust/src/bench/src/multi_turn.rs b/rust/src/bench/src/multi_turn.rs index e0a3138170..8eb175a07f 100644 --- a/rust/src/bench/src/multi_turn.rs +++ b/rust/src/bench/src/multi_turn.rs @@ -186,9 +186,29 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result { - return Err(BenchError::Config( - "HF dataset multi-turn is not yet supported. Use 'random' or 'sharegpt' with --multi-turn.".into(), - )); + let tok = tokenizer + .as_ref() + .ok_or_else(|| BenchError::Config("HF dataset requires a tokenizer".into()))?; + let dataset_id = config.dataset_path.as_deref().ok_or_else(|| { + BenchError::Config("--dataset-path is required for --dataset-name hf".into()) + })?; + let (downloaded_path, _config, _split) = + crate::datasets::hf_dataset::download_hf_dataset( + dataset_id, + config.hf_subset.as_deref(), + config.hf_split.as_deref(), + config.num_prompts, + ) + .await?; + crate::datasets::multi_turn::load_sharegpt_multi_turn( + tok, + &downloaded_path, + config.num_prompts, + config.hf_output_len, + config.sharegpt_multi_turn_max_turns, + config.seed, + &config.request_id_prefix, + )? } };