[Rust][Benchmark] Support HF ShareGPT datasets in multi-turn mode (#51104)
Signed-off-by: xiaguan <751080330@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ba2ae9f239
commit
f0a61bd438
@@ -667,6 +667,7 @@ With `--multi-turn`, `--num-prompts` controls the number of **conversations**, n
|
||||
|
||||
- `--dataset-name random` — synthetic conversations with controllable per-turn token lengths. Auto-sets `min_tokens` to enforce output length without `ignore_eos`.
|
||||
- `--dataset-name sharegpt` — loads all turns (not just the first two); filters for entries with ≥ 2 real turns.
|
||||
- `--dataset-name hf` — downloads a ShareGPT-format HuggingFace config, then loads it like `sharegpt`.
|
||||
|
||||
**Prefix sharing** (random dataset): when `--multi-turn-prefix-global-ratio` or `--multi-turn-prefix-conversation-ratio` is > 0, each turn sends a fixed-length message (no history accumulation) composed of a global prefix + per-conversation prefix + unique suffix. The two ratios must sum to < 1.0.
|
||||
|
||||
|
||||
@@ -549,13 +549,15 @@ impl BenchConfig {
|
||||
));
|
||||
}
|
||||
|
||||
// Normalize and validate min/max turns. ShareGPT only consumes max_turns
|
||||
// (the loader walks all available turns up to the cap), so the
|
||||
// min/num/max coupling used for synthetic generation does not apply.
|
||||
if args.dataset_name == DatasetName::ShareGpt {
|
||||
// Normalize and validate min/max turns. Conversation datasets only
|
||||
// consume max_turns (the loader walks all available turns up to the
|
||||
// cap), so the min/num/max coupling used for synthetic generation
|
||||
// does not apply.
|
||||
if matches!(args.dataset_name, DatasetName::ShareGpt | DatasetName::Hf) {
|
||||
if args.multi_turn_max_turns == 1 {
|
||||
return Err(BenchError::Config(
|
||||
"--multi-turn-max-turns must be at least 2 for ShareGPT multi-turn".into(),
|
||||
"--multi-turn-max-turns must be at least 2 for ShareGPT-format multi-turn"
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
@@ -722,7 +724,7 @@ impl BenchConfig {
|
||||
multi_turn_min_turns,
|
||||
multi_turn_max_turns,
|
||||
sharegpt_multi_turn_max_turns: if args.multi_turn
|
||||
&& args.dataset_name == DatasetName::ShareGpt
|
||||
&& matches!(args.dataset_name, DatasetName::ShareGpt | DatasetName::Hf)
|
||||
&& args.multi_turn_max_turns != 0
|
||||
{
|
||||
Some(args.multi_turn_max_turns)
|
||||
@@ -1067,6 +1069,30 @@ mod tests {
|
||||
assert_eq!(config.sharegpt_multi_turn_max_turns, Some(20));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hf_multi_turn_uses_conversation_turn_cap() {
|
||||
let args = vec![
|
||||
"vllm-bench",
|
||||
"--backend",
|
||||
"openai-chat",
|
||||
"--multi-turn",
|
||||
"--model",
|
||||
"test-model",
|
||||
"--dataset-name",
|
||||
"hf",
|
||||
"--dataset-path",
|
||||
"org/sharegpt-dataset",
|
||||
"--hf-subset",
|
||||
"sharegpt",
|
||||
"--multi-turn-max-turns",
|
||||
"2",
|
||||
];
|
||||
let args = parse_args(args);
|
||||
let config = BenchConfig::from_args(&args).unwrap();
|
||||
|
||||
assert_eq!(config.sharegpt_multi_turn_max_turns, Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sweep_summary_percentiles_default_empty() {
|
||||
let args = base_multi_turn_args();
|
||||
|
||||
@@ -186,9 +186,29 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
|
||||
));
|
||||
}
|
||||
DatasetName::Hf => {
|
||||
return Err(BenchError::Config(
|
||||
"HF dataset multi-turn is not yet supported. Use 'random' or 'sharegpt' with --multi-turn.".into(),
|
||||
));
|
||||
let tok = tokenizer
|
||||
.as_ref()
|
||||
.ok_or_else(|| BenchError::Config("HF dataset requires a tokenizer".into()))?;
|
||||
let dataset_id = config.dataset_path.as_deref().ok_or_else(|| {
|
||||
BenchError::Config("--dataset-path is required for --dataset-name hf".into())
|
||||
})?;
|
||||
let (downloaded_path, _config, _split) =
|
||||
crate::datasets::hf_dataset::download_hf_dataset(
|
||||
dataset_id,
|
||||
config.hf_subset.as_deref(),
|
||||
config.hf_split.as_deref(),
|
||||
config.num_prompts,
|
||||
)
|
||||
.await?;
|
||||
crate::datasets::multi_turn::load_sharegpt_multi_turn(
|
||||
tok,
|
||||
&downloaded_path,
|
||||
config.num_prompts,
|
||||
config.hf_output_len,
|
||||
config.sharegpt_multi_turn_max_turns,
|
||||
config.seed,
|
||||
&config.request_id_prefix,
|
||||
)?
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user