[Bugfix][Rust] Fix HF multi-turn dataset integration (#56915)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a7576447b8
commit
772e4f05fa
@@ -350,6 +350,27 @@ pub fn load_sharegpt_multi_turn(
|
||||
.as_array()
|
||||
.ok_or_else(|| BenchError::Config("ShareGPT file must contain a JSON array".into()))?;
|
||||
|
||||
load_sharegpt_multi_turn_from_rows(
|
||||
tokenizer,
|
||||
entries,
|
||||
num_conversations,
|
||||
output_len_override,
|
||||
max_turns,
|
||||
seed,
|
||||
request_id_prefix,
|
||||
)
|
||||
}
|
||||
|
||||
/// Load multi-turn conversations from deserialized ShareGPT rows.
|
||||
pub fn load_sharegpt_multi_turn_from_rows(
|
||||
tokenizer: &TokenizerKind,
|
||||
entries: &[serde_json::Value],
|
||||
num_conversations: usize,
|
||||
output_len_override: Option<usize>,
|
||||
max_turns: Option<usize>,
|
||||
seed: u64,
|
||||
request_id_prefix: &str,
|
||||
) -> Result<Vec<MultiTurnConversation>> {
|
||||
// Filter entries with at least 4 messages (2 turns: user+assistant+user+assistant)
|
||||
let mut filtered: Vec<&serde_json::Value> = entries
|
||||
.iter()
|
||||
@@ -507,6 +528,42 @@ fn gen_prompt_to_target_len(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_load_sharegpt_multi_turn_from_hf_rows() {
|
||||
let tokenizer =
|
||||
TokenizerKind::Tiktoken(crate::tiktoken::load_builtin_tiktoken("gpt2").unwrap());
|
||||
let rows = vec![serde_json::json!({
|
||||
"conversations": [
|
||||
{"from": "human", "value": "hello"},
|
||||
{"from": "gpt", "value": "hi"},
|
||||
{"from": "human", "value": "how are you?"},
|
||||
{"from": "gpt", "value": "great"}
|
||||
]
|
||||
})];
|
||||
|
||||
let conversations =
|
||||
load_sharegpt_multi_turn_from_rows(&tokenizer, &rows, 1, Some(7), None, 42, "hf-")
|
||||
.unwrap();
|
||||
let actual = conversations
|
||||
.iter()
|
||||
.map(|conversation| {
|
||||
(
|
||||
conversation.conversation_id.as_str(),
|
||||
conversation
|
||||
.turns
|
||||
.iter()
|
||||
.map(|turn| (turn.user_message.as_ref(), turn.expected_output_len))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(
|
||||
actual,
|
||||
vec![("hf-conv-0", vec![("hello", 7), ("how are you?", 7)])]
|
||||
);
|
||||
}
|
||||
|
||||
fn common_prefix_bytes(strings: &[&str]) -> usize {
|
||||
if strings.is_empty() {
|
||||
return 0;
|
||||
|
||||
@@ -192,17 +192,18 @@ pub async fn run_multi_turn_benchmark(config: &BenchConfig) -> Result<serde_json
|
||||
let dataset_id = config.dataset_path.as_deref().ok_or_else(|| {
|
||||
BenchError::Config("--dataset-path is required for --dataset-name hf".into())
|
||||
})?;
|
||||
let (downloaded_path, _config, _split) =
|
||||
crate::datasets::hf_dataset::download_hf_dataset(
|
||||
dataset_id,
|
||||
config.hf_subset.as_deref(),
|
||||
config.hf_split.as_deref(),
|
||||
config.num_prompts,
|
||||
)
|
||||
.await?;
|
||||
crate::datasets::multi_turn::load_sharegpt_multi_turn(
|
||||
let (rows, _config, _split) = crate::datasets::hf_dataset::download_hf_dataset(
|
||||
dataset_id,
|
||||
config.hf_subset.as_deref(),
|
||||
config.hf_split.as_deref(),
|
||||
config.num_prompts,
|
||||
config.seed,
|
||||
config.disable_shuffle,
|
||||
)
|
||||
.await?;
|
||||
crate::datasets::multi_turn::load_sharegpt_multi_turn_from_rows(
|
||||
tok,
|
||||
&downloaded_path,
|
||||
&rows,
|
||||
config.num_prompts,
|
||||
config.hf_output_len,
|
||||
config.sharegpt_multi_turn_max_turns,
|
||||
|
||||
Reference in New Issue
Block a user