[Rust][Benchmark] Support HF ShareGPT datasets in multi-turn mode (#51104)

Signed-off-by: xiaguan <751080330@qq.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
JinYan Su
2026-09-14 16:48:23 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ba2ae9f239
commit f0a61bd438
3 changed files with 56 additions and 9 deletions
+1
View File
@@ -667,6 +667,7 @@ With `--multi-turn`, `--num-prompts` controls the number of **conversations**, n
- `--dataset-name random` — synthetic conversations with controllable per-turn token lengths. Auto-sets `min_tokens` to enforce output length without `ignore_eos`.
- `--dataset-name sharegpt` — loads all turns (not just the first two); filters for entries with ≥ 2 real turns.
- `--dataset-name hf` — downloads a ShareGPT-format HuggingFace config, then loads it like `sharegpt`.
**Prefix sharing** (random dataset): when `--multi-turn-prefix-global-ratio` or `--multi-turn-prefix-conversation-ratio` is > 0, each turn sends a fixed-length message (no history accumulation) composed of a global prefix + per-conversation prefix + unique suffix. The two ratios must sum to < 1.0.
+32 -6
View File
@@ -549,13 +549,15 @@ impl BenchConfig {
));
}
// Normalize and validate min/max turns. ShareGPT only consumes max_turns
// (the loader walks all available turns up to the cap), so the
// min/num/max coupling used for synthetic generation does not apply.
if args.dataset_name == DatasetName::ShareGpt {
// Normalize and validate min/max turns. Conversation datasets only
// consume max_turns (the loader walks all available turns up to the
// cap), so the min/num/max coupling used for synthetic generation
// does not apply.
if matches!(args.dataset_name, DatasetName::ShareGpt | DatasetName::Hf) {
if args.multi_turn_max_turns == 1 {
return Err(BenchError::Config(
"--multi-turn-max-turns must be at least 2 for ShareGPT multi-turn".into(),
"--multi-turn-max-turns must be at least 2 for ShareGPT-format multi-turn"
.into(),
));
}
} else {
@@ -722,7 +724,7 @@ impl BenchConfig {
multi_turn_min_turns,
multi_turn_max_turns,
sharegpt_multi_turn_max_turns: if args.multi_turn
&& args.dataset_name == DatasetName::ShareGpt
&& matches!(args.dataset_name, DatasetName::ShareGpt | DatasetName::Hf)
&& args.multi_turn_max_turns != 0
{
Some(args.multi_turn_max_turns)
@@ -1067,6 +1069,30 @@ mod tests {
assert_eq!(config.sharegpt_multi_turn_max_turns, Some(20));
}
#[test]
fn test_hf_multi_turn_uses_conversation_turn_cap() {
let args = vec![
"vllm-bench",
"--backend",
"openai-chat",
"--multi-turn",
"--model",
"test-model",
"--dataset-name",
"hf",
"--dataset-path",
"org/sharegpt-dataset",
"--hf-subset",
"sharegpt",
"--multi-turn-max-turns",
"2",
];
let args = parse_args(args);
let config = BenchConfig::from_args(&args).unwrap();
assert_eq!(config.sharegpt_multi_turn_max_turns, Some(2));
}
#[test]
fn test_sweep_summary_percentiles_default_empty() {
let args = base_multi_turn_args();
+23 -3
View File
@@ -186,9 +186,29 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
));
}
DatasetName::Hf => {
return Err(BenchError::Config(
"HF dataset multi-turn is not yet supported. Use 'random' or 'sharegpt' with --multi-turn.".into(),
));
let tok = tokenizer
.as_ref()
.ok_or_else(|| BenchError::Config("HF dataset requires a tokenizer".into()))?;
let dataset_id = config.dataset_path.as_deref().ok_or_else(|| {
BenchError::Config("--dataset-path is required for --dataset-name hf".into())
})?;
let (downloaded_path, _config, _split) =
crate::datasets::hf_dataset::download_hf_dataset(
dataset_id,
config.hf_subset.as_deref(),
config.hf_split.as_deref(),
config.num_prompts,
)
.await?;
crate::datasets::multi_turn::load_sharegpt_multi_turn(
tok,
&downloaded_path,
config.num_prompts,
config.hf_output_len,
config.sharegpt_multi_turn_max_turns,
config.seed,
&config.request_id_prefix,
)?
}
};