[Rust Frontend] Introduce unified parser interface & combined parser (#46583)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::DeepSeekParser as ExternalDeepSeekParser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{DeepSeekV3ToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<|tool▁calls▁begin|>",
|
||||
"<|tool▁call▁begin|>function<|tool▁sep|>get_weather\n",
|
||||
"```json\n",
|
||||
"{\"location\":\"Hangzhou\",\"days\":3}",
|
||||
"\n```<|tool▁call▁end|>",
|
||||
"<|tool▁call▁begin|>function<|tool▁sep|>get_weather\n",
|
||||
"```json\n",
|
||||
"{\"location\":\"San Francisco\",\"days\":2}",
|
||||
"\n```<|tool▁call▁end|>",
|
||||
"<|tool▁calls▁end|>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no DeepSeek V3 tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
DeepSeekV3ToolParser::create(tools).expect("DeepSeek V3 parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = ExternalDeepSeekParser::new();
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
ExternalDeepSeekParser::new,
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_deepseek_v3(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"deepseek_v3/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"deepseek_v3/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_deepseek_v3);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,128 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::DeepSeek31Parser as ExternalDeepSeek31Parser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{DeepSeekV31ToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<|tool▁calls▁begin|>",
|
||||
"<|tool▁call▁begin|>get_weather<|tool▁sep|>",
|
||||
"{\"location\":\"Hangzhou\",\"days\":3}",
|
||||
"<|tool▁call▁end|>",
|
||||
"<|tool▁call▁begin|>get_weather<|tool▁sep|>",
|
||||
"{\"location\":\"San Francisco\",\"days\":2}",
|
||||
"<|tool▁call▁end|>",
|
||||
"<|tool▁calls▁end|>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no DeepSeek V3.1 tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
DeepSeekV31ToolParser::create(tools).expect("DeepSeek V3.1 parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = ExternalDeepSeek31Parser::new();
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
ExternalDeepSeek31Parser::new,
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_deepseek_v31(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"deepseek_v31/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"deepseek_v31/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_deepseek_v31);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,113 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{DeepSeekV32ToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::feed_parser;
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<|DSML|function_calls>\n",
|
||||
"<|DSML|invoke name=\"get_weather\">\n",
|
||||
"<|DSML|parameter name=\"location\" string=\"true\">Hangzhou</|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"date\" string=\"true\">2026-04-28</|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"unit\" string=\"true\">celsius</|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"days\" string=\"false\">3</|DSML|parameter>\n",
|
||||
"</|DSML|invoke>\n",
|
||||
"<|DSML|invoke name=\"get_weather\">\n",
|
||||
"<|DSML|parameter name=\"location\" string=\"true\">San Francisco</|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"date\" string=\"true\">2026-04-28</|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"unit\" string=\"true\">fahrenheit</|DSML|parameter>\n",
|
||||
"<|DSML|parameter name=\"days\" string=\"false\">2</|DSML|parameter>\n",
|
||||
"</|DSML|invoke>\n",
|
||||
"</|DSML|function_calls>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no DSML tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
DeepSeekV32ToolParser::create(tools).expect("DeepSeek V3.2 parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("reuse_parser", |b| {
|
||||
let mut parser = parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_deepseek_v32(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"deepseek_v32/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"deepseek_v32/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_deepseek_v32);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,155 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{Gemma4ToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::feed_parser;
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
const LONG_TOOL_ARGUMENT_REPEATS: usize = 256;
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will inspect the data before answering.\n",
|
||||
"<|tool_call>",
|
||||
"call:convert{",
|
||||
"whole:114.514,",
|
||||
"flag:true,",
|
||||
"empty:<|\"|><|\"|>,",
|
||||
"payload:{",
|
||||
"name:<|\"|>demo<|\"|>,",
|
||||
"count:42,",
|
||||
"enabled:false,",
|
||||
"missing:null,",
|
||||
"nested:{level:2,label:<|\"|>deep<|\"|>},",
|
||||
"tags:[<|\"|>red<|\"|>,<|\"|>blue<|\"|>,3,true,null,{kind:<|\"|>leaf<|\"|>}]",
|
||||
"},",
|
||||
"items:[",
|
||||
"<|\"|>alpha<|\"|>,",
|
||||
"{key:<|\"|>value<|\"|>,score:0.75},",
|
||||
"[1,2,3]",
|
||||
"]",
|
||||
"}",
|
||||
"<tool_call|>",
|
||||
"<|tool_call>",
|
||||
"call:update_record{",
|
||||
"data:{id:7,active:true,notes:[<|\"|>keep<|\"|>,<|\"|>review<|\"|>]}",
|
||||
"}",
|
||||
"<tool_call|>",
|
||||
" Finished.",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no Gemma4 tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn long_tool_argument_fixture() -> String {
|
||||
let line =
|
||||
"<section><p>Literal } and <tool_call|> marker-shaped text inside content.</p></section>\n";
|
||||
format!(
|
||||
concat!(
|
||||
"I will write the file.\n",
|
||||
"<|tool_call>",
|
||||
"call:write_file{{",
|
||||
"path:<|\"|>index.html<|\"|>,",
|
||||
"content:<|\"|>{}<|\"|>",
|
||||
"}}",
|
||||
"<tool_call|>",
|
||||
"Done."
|
||||
),
|
||||
line.repeat(LONG_TOOL_ARGUMENT_REPEATS)
|
||||
)
|
||||
}
|
||||
|
||||
fn parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
Gemma4ToolParser::create(tools).expect("Gemma4 parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("reuse_parser", |b| {
|
||||
let mut parser = parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_gemma4(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_tool_argument = long_tool_argument_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"gemma4/mixed_complex_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will inspect the data before answering.\n Finished.",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"gemma4/long_tool_argument",
|
||||
&tools,
|
||||
&long_tool_argument,
|
||||
CHUNK_CHARS,
|
||||
"I will write the file.\nDone.",
|
||||
1,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"gemma4/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_gemma4);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,210 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::Glm4MoeParser as ExternalGlm4MoeParser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const GLM45_PARSER_NAME: &str = "glm45";
|
||||
const GLM47_PARSER_NAME: &str = "glm47";
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn glm45_mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<tool_call>get_weather\n",
|
||||
"<arg_key>city</arg_key>\n",
|
||||
"<arg_value>Hangzhou</arg_value>\n",
|
||||
"<arg_key>date</arg_key>\n",
|
||||
"<arg_value>2026-05-07</arg_value>\n",
|
||||
"<arg_key>unit</arg_key>\n",
|
||||
"<arg_value>celsius</arg_value>\n",
|
||||
"<arg_key>days</arg_key>\n",
|
||||
"<arg_value>3</arg_value>\n",
|
||||
"</tool_call>\n",
|
||||
"<tool_call>get_weather\n",
|
||||
"<arg_key>city</arg_key>\n",
|
||||
"<arg_value>San Francisco</arg_value>\n",
|
||||
"<arg_key>date</arg_key>\n",
|
||||
"<arg_value>2026-05-07</arg_value>\n",
|
||||
"<arg_key>unit</arg_key>\n",
|
||||
"<arg_value>fahrenheit</arg_value>\n",
|
||||
"<arg_key>days</arg_key>\n",
|
||||
"<arg_value>2</arg_value>\n",
|
||||
"</tool_call>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn glm47_mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<tool_call>get_weather",
|
||||
"<arg_key>city</arg_key>",
|
||||
"<arg_value>Hangzhou</arg_value>",
|
||||
"<arg_key>date</arg_key>",
|
||||
"<arg_value>2026-05-07</arg_value>",
|
||||
"<arg_key>unit</arg_key>",
|
||||
"<arg_value>celsius</arg_value>",
|
||||
"<arg_key>days</arg_key>",
|
||||
"<arg_value>3</arg_value>",
|
||||
"</tool_call>",
|
||||
"<tool_call>get_weather",
|
||||
"<arg_key>city</arg_key>",
|
||||
"<arg_value>San Francisco</arg_value>",
|
||||
"<arg_key>date</arg_key>",
|
||||
"<arg_value>2026-05-07</arg_value>",
|
||||
"<arg_key>unit</arg_key>",
|
||||
"<arg_value>fahrenheit</arg_value>",
|
||||
"<arg_key>days</arg_key>",
|
||||
"<arg_value>2</arg_value>",
|
||||
"</tool_call>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no GLM MoE tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn native_parser(name: &str, tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
match name {
|
||||
GLM45_PARSER_NAME => Glm45MoeToolParser::create(tools),
|
||||
GLM47_PARSER_NAME => Glm47MoeToolParser::create(tools),
|
||||
_ => unreachable!("unexpected GLM parser name"),
|
||||
}
|
||||
.expect("GLM MoE parser should initialize")
|
||||
}
|
||||
|
||||
fn external_parser(name: &str) -> ExternalGlm4MoeParser {
|
||||
match name {
|
||||
GLM45_PARSER_NAME => ExternalGlm4MoeParser::glm45(),
|
||||
GLM47_PARSER_NAME => ExternalGlm4MoeParser::glm47(),
|
||||
_ => unreachable!("unexpected GLM parser name"),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
parser_name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(parser_name, tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(parser_name, tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = external_parser(parser_name);
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| external_parser(parser_name),
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_glm45_moe(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let glm45_mixed_text = glm45_mixed_fixture();
|
||||
let glm47_mixed_text = glm47_mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"glm45/mixed_text_tool_call",
|
||||
GLM45_PARSER_NAME,
|
||||
&tools,
|
||||
&glm45_mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"glm47/mixed_text_tool_call",
|
||||
GLM47_PARSER_NAME,
|
||||
&tools,
|
||||
&glm47_mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"glm45/long_normal_text",
|
||||
GLM45_PARSER_NAME,
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"glm47/long_normal_text",
|
||||
GLM47_PARSER_NAME,
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_glm45_moe);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,149 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::KimiK2Parser as ExternalKimiK2Parser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{KimiK2ToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<|tool_calls_section_begin|>",
|
||||
"<|tool_call_begin|>functions.get_weather:0",
|
||||
"<|tool_call_argument_begin|>{\"location\":\"Hangzhou\",\"days\":3}",
|
||||
"<|tool_call_end|>",
|
||||
"<|tool_call_begin|>functions.get_weather:1",
|
||||
"<|tool_call_argument_begin|>{\"location\":\"San Francisco\",\"days\":2}",
|
||||
"<|tool_call_end|>",
|
||||
"<|tool_calls_section_end|>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn mixed_chunks() -> Vec<&'static str> {
|
||||
vec![
|
||||
"I will check two cities before answering.\n",
|
||||
"<|tool_calls_section_begin|>",
|
||||
"<|tool_call_begin|>functions.get_weather:0",
|
||||
"<|tool_call_argument_begin|>",
|
||||
"{\"location\":",
|
||||
"\"Hangzhou\",",
|
||||
"\"days\":3}",
|
||||
"<|tool_call_end|>",
|
||||
"<|tool_call_begin|>functions.get_weather:1",
|
||||
"<|tool_call_argument_begin|>",
|
||||
"{\"location\":",
|
||||
"\"San Francisco\",",
|
||||
"\"days\":2}",
|
||||
"<|tool_call_end|>",
|
||||
"<|tool_calls_section_end|>",
|
||||
]
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no Kimi K2 tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
KimiK2ToolParser::create(tools).expect("Kimi K2 parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunks: &[&str],
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = ExternalKimiK2Parser::new();
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(chunks));
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
ExternalKimiK2Parser::new,
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(chunks));
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_kimi_k2(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let mixed_chunks = mixed_chunks();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
let long_normal_chunks = split_by_chars(&long_normal_text, CHUNK_CHARS);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"kimi_k2/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
&mixed_chunks,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"kimi_k2/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
&long_normal_chunks,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_kimi_k2);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,128 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::LlamaParser as ExternalLlamaParser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{Llama3JsonToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn tool_call(function_name: &str, parameters: &str) -> String {
|
||||
format!(r#"{{"name":"{function_name}","parameters":{parameters}}}"#)
|
||||
}
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
format!(
|
||||
"{}; {}",
|
||||
tool_call("get_weather", r#"{"location":"Hangzhou","days":3}"#),
|
||||
tool_call(
|
||||
"convert",
|
||||
r#"{"whole":42.5,"flag":true,"payload":{"nested":["x",null]},"items":[1,2,3],"empty":""}"#
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no Llama JSON tool call at the root.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
Llama3JsonToolParser::create(tools).expect("Llama JSON parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = ExternalLlamaParser::new();
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
ExternalLlamaParser::new,
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_llama3_json(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"llama3_json/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"llama3_json/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_llama3_json);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,136 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::MinimaxM2Parser as ExternalMinimaxM2Parser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{MinimaxM2ToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<minimax:tool_call>",
|
||||
"<invoke name=\"get_weather\">",
|
||||
"<parameter name=\"city\">Hangzhou</parameter>",
|
||||
"<parameter name=\"date\">2026-04-30</parameter>",
|
||||
"<parameter name=\"unit\">celsius</parameter>",
|
||||
"<parameter name=\"days\">3</parameter>",
|
||||
"</invoke>",
|
||||
"<invoke name=\"get_weather\">",
|
||||
"<parameter name=\"city\">San Francisco</parameter>",
|
||||
"<parameter name=\"date\">2026-04-30</parameter>",
|
||||
"<parameter name=\"unit\">fahrenheit</parameter>",
|
||||
"<parameter name=\"days\">2</parameter>",
|
||||
"</invoke>",
|
||||
"</minimax:tool_call>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no MiniMax M2 tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
MinimaxM2ToolParser::create(tools).expect("MiniMax M2 parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = ExternalMinimaxM2Parser::new();
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
ExternalMinimaxM2Parser::new,
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_minimax_m2(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"minimax_m2/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"minimax_m2/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_minimax_m2);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,161 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
const LONG_TOOL_BODY_REPEATS: usize = 8192;
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
concat!(
|
||||
"I will check two cities before answering.\n",
|
||||
"<tool_call>\n",
|
||||
"<function=get_weather>\n",
|
||||
"<parameter=location>Hangzhou</parameter>\n",
|
||||
"<parameter=date>2026-04-29</parameter>\n",
|
||||
"<parameter=unit>celsius</parameter>\n",
|
||||
"<parameter=days>3</parameter>\n",
|
||||
"</function>\n",
|
||||
"</tool_call>\n",
|
||||
"<tool_call>\n",
|
||||
"<function=get_weather>\n",
|
||||
"<parameter=location>San Francisco</parameter>\n",
|
||||
"<parameter=date>2026-04-29</parameter>\n",
|
||||
"<parameter=unit>fahrenheit</parameter>\n",
|
||||
"<parameter=days>2</parameter>\n",
|
||||
"</function>\n",
|
||||
"</tool_call>",
|
||||
)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no Qwen Coder tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn long_tool_call_fixture() -> String {
|
||||
let location = "x".repeat(LONG_TOOL_BODY_REPEATS);
|
||||
format!(
|
||||
"<tool_call>\n\
|
||||
<function=get_weather>\n\
|
||||
<parameter=location>{location}</parameter>\n\
|
||||
</function>\n\
|
||||
</tool_call>"
|
||||
)
|
||||
}
|
||||
|
||||
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
Qwen3CoderToolParser::create(tools).expect("Qwen Coder parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = ExternalQwenCoderParser::new();
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
ExternalQwenCoderParser::new,
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_qwen3_coder(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
let long_tool_call = long_tool_call_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"qwen3_coder/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"qwen3_coder/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"qwen3_coder/long_tool_call_body",
|
||||
&tools,
|
||||
&long_tool_call,
|
||||
CHUNK_CHARS,
|
||||
"",
|
||||
1,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_qwen3_coder);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,125 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use tool_parser::parsers::QwenParser as ExternalQwenParser;
|
||||
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
|
||||
use vllm_parser::tool::{Qwen3XmlToolParser, Tool, ToolParser};
|
||||
|
||||
mod utils;
|
||||
use utils::{feed_external_parser, feed_parser, openai_tools};
|
||||
|
||||
const CHUNK_CHARS: usize = 7;
|
||||
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
|
||||
|
||||
fn tool_call(function_name: &str, arguments: &str) -> String {
|
||||
format!("<tool_call>\n{{\"name\":\"{function_name}\",\"arguments\":{arguments}}}\n</tool_call>")
|
||||
}
|
||||
|
||||
fn mixed_fixture() -> String {
|
||||
format!(
|
||||
"I will check two cities before answering.\n{}{}",
|
||||
tool_call("get_weather", r#"{"location":"Hangzhou","days":3}"#),
|
||||
tool_call("get_weather", r#"{"location":"San Francisco","days":2}"#),
|
||||
)
|
||||
}
|
||||
|
||||
fn long_normal_text_fixture() -> String {
|
||||
let line = "This is ordinary assistant text with no Qwen XML tool markers at all.\n";
|
||||
line.repeat(LONG_NORMAL_TEXT_REPEATS)
|
||||
}
|
||||
|
||||
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
|
||||
Qwen3XmlToolParser::create(tools).expect("Qwen XML parser should initialize")
|
||||
}
|
||||
|
||||
fn run_stream_group(
|
||||
c: &mut Criterion,
|
||||
name: &str,
|
||||
tools: &[Tool],
|
||||
text: &str,
|
||||
chunk_chars: usize,
|
||||
expected_normal_text: &str,
|
||||
expected_native_calls_len: usize,
|
||||
) {
|
||||
let chunks = split_by_chars(text, chunk_chars);
|
||||
let openai_tools = openai_tools(tools);
|
||||
|
||||
let mut group = c.benchmark_group(name);
|
||||
group.sample_size(50);
|
||||
group.warm_up_time(Duration::from_millis(300));
|
||||
group.measurement_time(Duration::from_secs(2));
|
||||
group.throughput(Throughput::Bytes(text.len() as u64));
|
||||
|
||||
group.bench_function("native_reuse_parser", |b| {
|
||||
let mut parser = native_parser(tools);
|
||||
b.iter(|| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("native_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
|| native_parser(tools),
|
||||
|mut parser| {
|
||||
let result = feed_parser(&mut *parser, black_box(&chunks));
|
||||
debug_assert_eq!(result.0, expected_normal_text);
|
||||
debug_assert_eq!(result.1, expected_native_calls_len);
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.bench_function("external_reuse_parser", |b| {
|
||||
let mut parser = ExternalQwenParser::new();
|
||||
b.iter(|| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("external_create_parser", |b| {
|
||||
b.iter_batched(
|
||||
ExternalQwenParser::new,
|
||||
|mut parser| {
|
||||
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
|
||||
black_box(result);
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_qwen3_xml(c: &mut Criterion) {
|
||||
let tools = test_tools();
|
||||
let mixed_text = mixed_fixture();
|
||||
let long_normal_text = long_normal_text_fixture();
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"qwen3_xml/mixed_text_tool_call",
|
||||
&tools,
|
||||
&mixed_text,
|
||||
CHUNK_CHARS,
|
||||
"I will check two cities before answering.\n",
|
||||
2,
|
||||
);
|
||||
|
||||
run_stream_group(
|
||||
c,
|
||||
"qwen3_xml/long_normal_text",
|
||||
&tools,
|
||||
&long_normal_text,
|
||||
CHUNK_CHARS,
|
||||
&long_normal_text,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_qwen3_xml);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,49 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use futures::FutureExt as _;
|
||||
use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool};
|
||||
use tool_parser::traits::ToolParser as ExternalToolParser;
|
||||
use vllm_parser::tool::test_utils::collect_stream;
|
||||
use vllm_parser::tool::{Tool, ToolParser};
|
||||
|
||||
pub(super) fn openai_tools(tools: &[Tool]) -> Vec<OpenAiTool> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| OpenAiTool {
|
||||
tool_type: "function".to_string(),
|
||||
function: OpenAiFunction {
|
||||
name: tool.name.clone(),
|
||||
description: tool.description.clone(),
|
||||
parameters: tool.parameters.clone(),
|
||||
strict: tool.strict,
|
||||
},
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn feed_parser(parser: &mut dyn ToolParser, chunks: &[&str]) -> (String, usize) {
|
||||
let result = collect_stream(parser, chunks);
|
||||
(result.normal_text, result.calls.len())
|
||||
}
|
||||
|
||||
pub(super) fn feed_external_parser(
|
||||
parser: &mut impl ExternalToolParser,
|
||||
tools: &[OpenAiTool],
|
||||
chunks: &[&str],
|
||||
) -> (String, usize) {
|
||||
ExternalToolParser::reset(parser);
|
||||
|
||||
let mut normal_text = String::new();
|
||||
let mut calls_len = 0;
|
||||
for chunk in chunks {
|
||||
let delta = parser
|
||||
.parse_incremental(chunk, tools)
|
||||
.now_or_never()
|
||||
.expect("external parser should not suspend")
|
||||
.expect("chunk should parse");
|
||||
normal_text.push_str(&delta.normal_text);
|
||||
calls_len += delta.calls.len();
|
||||
}
|
||||
calls_len += parser.get_unstreamed_tool_args().unwrap_or_default().len();
|
||||
(normal_text, calls_len)
|
||||
}
|
||||
Reference in New Issue
Block a user