[Rust Frontend] Introduce unified parser interface & combined parser (#46583)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-25 03:17:31 +00:00
committed by GitHub
parent efb5acffd5
commit 76c3c4ff63
74 changed files with 1204 additions and 1745 deletions
+130
View File
@@ -0,0 +1,130 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::DeepSeekParser as ExternalDeepSeekParser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{DeepSeekV3ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<|tool▁calls▁begin|>",
"<|tool▁call▁begin|>function<|tool▁sep|>get_weather\n",
"```json\n",
"{\"location\":\"Hangzhou\",\"days\":3}",
"\n```<|tool▁call▁end|>",
"<|tool▁call▁begin|>function<|tool▁sep|>get_weather\n",
"```json\n",
"{\"location\":\"San Francisco\",\"days\":2}",
"\n```<|tool▁call▁end|>",
"<|tool▁calls▁end|>",
)
.to_string()
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no DeepSeek V3 tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
DeepSeekV3ToolParser::create(tools).expect("DeepSeek V3 parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = ExternalDeepSeekParser::new();
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
ExternalDeepSeekParser::new,
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_deepseek_v3(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"deepseek_v3/mixed_text_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"deepseek_v3/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_deepseek_v3);
criterion_main!(benches);
+128
View File
@@ -0,0 +1,128 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::DeepSeek31Parser as ExternalDeepSeek31Parser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{DeepSeekV31ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<|tool▁calls▁begin|>",
"<|tool▁call▁begin|>get_weather<|tool▁sep|>",
"{\"location\":\"Hangzhou\",\"days\":3}",
"<|tool▁call▁end|>",
"<|tool▁call▁begin|>get_weather<|tool▁sep|>",
"{\"location\":\"San Francisco\",\"days\":2}",
"<|tool▁call▁end|>",
"<|tool▁calls▁end|>",
)
.to_string()
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no DeepSeek V3.1 tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
DeepSeekV31ToolParser::create(tools).expect("DeepSeek V3.1 parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = ExternalDeepSeek31Parser::new();
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
ExternalDeepSeek31Parser::new,
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_deepseek_v31(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"deepseek_v31/mixed_text_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"deepseek_v31/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_deepseek_v31);
criterion_main!(benches);
+113
View File
@@ -0,0 +1,113 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{DeepSeekV32ToolParser, Tool, ToolParser};
mod utils;
use utils::feed_parser;
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<|DSML|function_calls>\n",
"<|DSML|invoke name=\"get_weather\">\n",
"<|DSML|parameter name=\"location\" string=\"true\">Hangzhou</|DSML|parameter>\n",
"<|DSML|parameter name=\"date\" string=\"true\">2026-04-28</|DSML|parameter>\n",
"<|DSML|parameter name=\"unit\" string=\"true\">celsius</|DSML|parameter>\n",
"<|DSML|parameter name=\"days\" string=\"false\">3</|DSML|parameter>\n",
"</|DSML|invoke>\n",
"<|DSML|invoke name=\"get_weather\">\n",
"<|DSML|parameter name=\"location\" string=\"true\">San Francisco</|DSML|parameter>\n",
"<|DSML|parameter name=\"date\" string=\"true\">2026-04-28</|DSML|parameter>\n",
"<|DSML|parameter name=\"unit\" string=\"true\">fahrenheit</|DSML|parameter>\n",
"<|DSML|parameter name=\"days\" string=\"false\">2</|DSML|parameter>\n",
"</|DSML|invoke>\n",
"</|DSML|function_calls>",
)
.to_string()
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no DSML tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn parser(tools: &[Tool]) -> Box<dyn ToolParser> {
DeepSeekV32ToolParser::create(tools).expect("DeepSeek V3.2 parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("reuse_parser", |b| {
let mut parser = parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_calls_len);
black_box(result);
})
});
group.bench_function("create_parser", |b| {
b.iter_batched(
|| parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_deepseek_v32(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"deepseek_v32/mixed_text_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"deepseek_v32/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_deepseek_v32);
criterion_main!(benches);
+155
View File
@@ -0,0 +1,155 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Gemma4ToolParser, Tool, ToolParser};
mod utils;
use utils::feed_parser;
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
const LONG_TOOL_ARGUMENT_REPEATS: usize = 256;
fn mixed_fixture() -> String {
concat!(
"I will inspect the data before answering.\n",
"<|tool_call>",
"call:convert{",
"whole:114.514,",
"flag:true,",
"empty:<|\"|><|\"|>,",
"payload:{",
"name:<|\"|>demo<|\"|>,",
"count:42,",
"enabled:false,",
"missing:null,",
"nested:{level:2,label:<|\"|>deep<|\"|>},",
"tags:[<|\"|>red<|\"|>,<|\"|>blue<|\"|>,3,true,null,{kind:<|\"|>leaf<|\"|>}]",
"},",
"items:[",
"<|\"|>alpha<|\"|>,",
"{key:<|\"|>value<|\"|>,score:0.75},",
"[1,2,3]",
"]",
"}",
"<tool_call|>",
"<|tool_call>",
"call:update_record{",
"data:{id:7,active:true,notes:[<|\"|>keep<|\"|>,<|\"|>review<|\"|>]}",
"}",
"<tool_call|>",
" Finished.",
)
.to_string()
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no Gemma4 tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn long_tool_argument_fixture() -> String {
let line =
"<section><p>Literal } and <tool_call|> marker-shaped text inside content.</p></section>\n";
format!(
concat!(
"I will write the file.\n",
"<|tool_call>",
"call:write_file{{",
"path:<|\"|>index.html<|\"|>,",
"content:<|\"|>{}<|\"|>",
"}}",
"<tool_call|>",
"Done."
),
line.repeat(LONG_TOOL_ARGUMENT_REPEATS)
)
}
fn parser(tools: &[Tool]) -> Box<dyn ToolParser> {
Gemma4ToolParser::create(tools).expect("Gemma4 parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("reuse_parser", |b| {
let mut parser = parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_calls_len);
black_box(result);
})
});
group.bench_function("create_parser", |b| {
b.iter_batched(
|| parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_gemma4(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_tool_argument = long_tool_argument_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"gemma4/mixed_complex_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"I will inspect the data before answering.\n Finished.",
2,
);
run_stream_group(
c,
"gemma4/long_tool_argument",
&tools,
&long_tool_argument,
CHUNK_CHARS,
"I will write the file.\nDone.",
1,
);
run_stream_group(
c,
"gemma4/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_gemma4);
criterion_main!(benches);
+210
View File
@@ -0,0 +1,210 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::Glm4MoeParser as ExternalGlm4MoeParser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const GLM45_PARSER_NAME: &str = "glm45";
const GLM47_PARSER_NAME: &str = "glm47";
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn glm45_mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<tool_call>get_weather\n",
"<arg_key>city</arg_key>\n",
"<arg_value>Hangzhou</arg_value>\n",
"<arg_key>date</arg_key>\n",
"<arg_value>2026-05-07</arg_value>\n",
"<arg_key>unit</arg_key>\n",
"<arg_value>celsius</arg_value>\n",
"<arg_key>days</arg_key>\n",
"<arg_value>3</arg_value>\n",
"</tool_call>\n",
"<tool_call>get_weather\n",
"<arg_key>city</arg_key>\n",
"<arg_value>San Francisco</arg_value>\n",
"<arg_key>date</arg_key>\n",
"<arg_value>2026-05-07</arg_value>\n",
"<arg_key>unit</arg_key>\n",
"<arg_value>fahrenheit</arg_value>\n",
"<arg_key>days</arg_key>\n",
"<arg_value>2</arg_value>\n",
"</tool_call>",
)
.to_string()
}
fn glm47_mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<tool_call>get_weather",
"<arg_key>city</arg_key>",
"<arg_value>Hangzhou</arg_value>",
"<arg_key>date</arg_key>",
"<arg_value>2026-05-07</arg_value>",
"<arg_key>unit</arg_key>",
"<arg_value>celsius</arg_value>",
"<arg_key>days</arg_key>",
"<arg_value>3</arg_value>",
"</tool_call>",
"<tool_call>get_weather",
"<arg_key>city</arg_key>",
"<arg_value>San Francisco</arg_value>",
"<arg_key>date</arg_key>",
"<arg_value>2026-05-07</arg_value>",
"<arg_key>unit</arg_key>",
"<arg_value>fahrenheit</arg_value>",
"<arg_key>days</arg_key>",
"<arg_value>2</arg_value>",
"</tool_call>",
)
.to_string()
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no GLM MoE tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn native_parser(name: &str, tools: &[Tool]) -> Box<dyn ToolParser> {
match name {
GLM45_PARSER_NAME => Glm45MoeToolParser::create(tools),
GLM47_PARSER_NAME => Glm47MoeToolParser::create(tools),
_ => unreachable!("unexpected GLM parser name"),
}
.expect("GLM MoE parser should initialize")
}
fn external_parser(name: &str) -> ExternalGlm4MoeParser {
match name {
GLM45_PARSER_NAME => ExternalGlm4MoeParser::glm45(),
GLM47_PARSER_NAME => ExternalGlm4MoeParser::glm47(),
_ => unreachable!("unexpected GLM parser name"),
}
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
parser_name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(parser_name, tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(parser_name, tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = external_parser(parser_name);
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
|| external_parser(parser_name),
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_glm45_moe(c: &mut Criterion) {
let tools = test_tools();
let glm45_mixed_text = glm45_mixed_fixture();
let glm47_mixed_text = glm47_mixed_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"glm45/mixed_text_tool_call",
GLM45_PARSER_NAME,
&tools,
&glm45_mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"glm47/mixed_text_tool_call",
GLM47_PARSER_NAME,
&tools,
&glm47_mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"glm45/long_normal_text",
GLM45_PARSER_NAME,
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
run_stream_group(
c,
"glm47/long_normal_text",
GLM47_PARSER_NAME,
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_glm45_moe);
criterion_main!(benches);
+149
View File
@@ -0,0 +1,149 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::KimiK2Parser as ExternalKimiK2Parser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{KimiK2ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<|tool_calls_section_begin|>",
"<|tool_call_begin|>functions.get_weather:0",
"<|tool_call_argument_begin|>{\"location\":\"Hangzhou\",\"days\":3}",
"<|tool_call_end|>",
"<|tool_call_begin|>functions.get_weather:1",
"<|tool_call_argument_begin|>{\"location\":\"San Francisco\",\"days\":2}",
"<|tool_call_end|>",
"<|tool_calls_section_end|>",
)
.to_string()
}
fn mixed_chunks() -> Vec<&'static str> {
vec![
"I will check two cities before answering.\n",
"<|tool_calls_section_begin|>",
"<|tool_call_begin|>functions.get_weather:0",
"<|tool_call_argument_begin|>",
"{\"location\":",
"\"Hangzhou\",",
"\"days\":3}",
"<|tool_call_end|>",
"<|tool_call_begin|>functions.get_weather:1",
"<|tool_call_argument_begin|>",
"{\"location\":",
"\"San Francisco\",",
"\"days\":2}",
"<|tool_call_end|>",
"<|tool_calls_section_end|>",
]
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no Kimi K2 tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
KimiK2ToolParser::create(tools).expect("Kimi K2 parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunks: &[&str],
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = ExternalKimiK2Parser::new();
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(chunks));
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
ExternalKimiK2Parser::new,
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(chunks));
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_kimi_k2(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let mixed_chunks = mixed_chunks();
let long_normal_text = long_normal_text_fixture();
let long_normal_chunks = split_by_chars(&long_normal_text, CHUNK_CHARS);
run_stream_group(
c,
"kimi_k2/mixed_text_tool_call",
&tools,
&mixed_text,
&mixed_chunks,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"kimi_k2/long_normal_text",
&tools,
&long_normal_text,
&long_normal_chunks,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_kimi_k2);
criterion_main!(benches);
+128
View File
@@ -0,0 +1,128 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::LlamaParser as ExternalLlamaParser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Llama3JsonToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn tool_call(function_name: &str, parameters: &str) -> String {
format!(r#"{{"name":"{function_name}","parameters":{parameters}}}"#)
}
fn mixed_fixture() -> String {
format!(
"{}; {}",
tool_call("get_weather", r#"{"location":"Hangzhou","days":3}"#),
tool_call(
"convert",
r#"{"whole":42.5,"flag":true,"payload":{"nested":["x",null]},"items":[1,2,3],"empty":""}"#
),
)
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no Llama JSON tool call at the root.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
Llama3JsonToolParser::create(tools).expect("Llama JSON parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = ExternalLlamaParser::new();
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
ExternalLlamaParser::new,
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_llama3_json(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"llama3_json/mixed_text_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"",
2,
);
run_stream_group(
c,
"llama3_json/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_llama3_json);
criterion_main!(benches);
+136
View File
@@ -0,0 +1,136 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::MinimaxM2Parser as ExternalMinimaxM2Parser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{MinimaxM2ToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<minimax:tool_call>",
"<invoke name=\"get_weather\">",
"<parameter name=\"city\">Hangzhou</parameter>",
"<parameter name=\"date\">2026-04-30</parameter>",
"<parameter name=\"unit\">celsius</parameter>",
"<parameter name=\"days\">3</parameter>",
"</invoke>",
"<invoke name=\"get_weather\">",
"<parameter name=\"city\">San Francisco</parameter>",
"<parameter name=\"date\">2026-04-30</parameter>",
"<parameter name=\"unit\">fahrenheit</parameter>",
"<parameter name=\"days\">2</parameter>",
"</invoke>",
"</minimax:tool_call>",
)
.to_string()
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no MiniMax M2 tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
MinimaxM2ToolParser::create(tools).expect("MiniMax M2 parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = ExternalMinimaxM2Parser::new();
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
ExternalMinimaxM2Parser::new,
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_minimax_m2(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"minimax_m2/mixed_text_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"minimax_m2/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_minimax_m2);
criterion_main!(benches);
+161
View File
@@ -0,0 +1,161 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
const LONG_TOOL_BODY_REPEATS: usize = 8192;
fn mixed_fixture() -> String {
concat!(
"I will check two cities before answering.\n",
"<tool_call>\n",
"<function=get_weather>\n",
"<parameter=location>Hangzhou</parameter>\n",
"<parameter=date>2026-04-29</parameter>\n",
"<parameter=unit>celsius</parameter>\n",
"<parameter=days>3</parameter>\n",
"</function>\n",
"</tool_call>\n",
"<tool_call>\n",
"<function=get_weather>\n",
"<parameter=location>San Francisco</parameter>\n",
"<parameter=date>2026-04-29</parameter>\n",
"<parameter=unit>fahrenheit</parameter>\n",
"<parameter=days>2</parameter>\n",
"</function>\n",
"</tool_call>",
)
.to_string()
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no Qwen Coder tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn long_tool_call_fixture() -> String {
let location = "x".repeat(LONG_TOOL_BODY_REPEATS);
format!(
"<tool_call>\n\
<function=get_weather>\n\
<parameter=location>{location}</parameter>\n\
</function>\n\
</tool_call>"
)
}
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
Qwen3CoderToolParser::create(tools).expect("Qwen Coder parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = ExternalQwenCoderParser::new();
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
ExternalQwenCoderParser::new,
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_qwen3_coder(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_normal_text = long_normal_text_fixture();
let long_tool_call = long_tool_call_fixture();
run_stream_group(
c,
"qwen3_coder/mixed_text_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"qwen3_coder/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
run_stream_group(
c,
"qwen3_coder/long_tool_call_body",
&tools,
&long_tool_call,
CHUNK_CHARS,
"",
1,
);
}
criterion_group!(benches, bench_qwen3_coder);
criterion_main!(benches);
+125
View File
@@ -0,0 +1,125 @@
use std::time::Duration;
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
use tool_parser::parsers::QwenParser as ExternalQwenParser;
use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
use vllm_parser::tool::{Qwen3XmlToolParser, Tool, ToolParser};
mod utils;
use utils::{feed_external_parser, feed_parser, openai_tools};
const CHUNK_CHARS: usize = 7;
const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
fn tool_call(function_name: &str, arguments: &str) -> String {
format!("<tool_call>\n{{\"name\":\"{function_name}\",\"arguments\":{arguments}}}\n</tool_call>")
}
fn mixed_fixture() -> String {
format!(
"I will check two cities before answering.\n{}{}",
tool_call("get_weather", r#"{"location":"Hangzhou","days":3}"#),
tool_call("get_weather", r#"{"location":"San Francisco","days":2}"#),
)
}
fn long_normal_text_fixture() -> String {
let line = "This is ordinary assistant text with no Qwen XML tool markers at all.\n";
line.repeat(LONG_NORMAL_TEXT_REPEATS)
}
fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
Qwen3XmlToolParser::create(tools).expect("Qwen XML parser should initialize")
}
fn run_stream_group(
c: &mut Criterion,
name: &str,
tools: &[Tool],
text: &str,
chunk_chars: usize,
expected_normal_text: &str,
expected_native_calls_len: usize,
) {
let chunks = split_by_chars(text, chunk_chars);
let openai_tools = openai_tools(tools);
let mut group = c.benchmark_group(name);
group.sample_size(50);
group.warm_up_time(Duration::from_millis(300));
group.measurement_time(Duration::from_secs(2));
group.throughput(Throughput::Bytes(text.len() as u64));
group.bench_function("native_reuse_parser", |b| {
let mut parser = native_parser(tools);
b.iter(|| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
})
});
group.bench_function("native_create_parser", |b| {
b.iter_batched(
|| native_parser(tools),
|mut parser| {
let result = feed_parser(&mut *parser, black_box(&chunks));
debug_assert_eq!(result.0, expected_normal_text);
debug_assert_eq!(result.1, expected_native_calls_len);
black_box(result);
},
BatchSize::SmallInput,
)
});
group.bench_function("external_reuse_parser", |b| {
let mut parser = ExternalQwenParser::new();
b.iter(|| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
})
});
group.bench_function("external_create_parser", |b| {
b.iter_batched(
ExternalQwenParser::new,
|mut parser| {
let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
black_box(result);
},
BatchSize::SmallInput,
)
});
group.finish();
}
fn bench_qwen3_xml(c: &mut Criterion) {
let tools = test_tools();
let mixed_text = mixed_fixture();
let long_normal_text = long_normal_text_fixture();
run_stream_group(
c,
"qwen3_xml/mixed_text_tool_call",
&tools,
&mixed_text,
CHUNK_CHARS,
"I will check two cities before answering.\n",
2,
);
run_stream_group(
c,
"qwen3_xml/long_normal_text",
&tools,
&long_normal_text,
CHUNK_CHARS,
&long_normal_text,
0,
);
}
criterion_group!(benches, bench_qwen3_xml);
criterion_main!(benches);
+49
View File
@@ -0,0 +1,49 @@
#![allow(dead_code)]
use futures::FutureExt as _;
use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool};
use tool_parser::traits::ToolParser as ExternalToolParser;
use vllm_parser::tool::test_utils::collect_stream;
use vllm_parser::tool::{Tool, ToolParser};
pub(super) fn openai_tools(tools: &[Tool]) -> Vec<OpenAiTool> {
tools
.iter()
.map(|tool| OpenAiTool {
tool_type: "function".to_string(),
function: OpenAiFunction {
name: tool.name.clone(),
description: tool.description.clone(),
parameters: tool.parameters.clone(),
strict: tool.strict,
},
})
.collect()
}
pub(super) fn feed_parser(parser: &mut dyn ToolParser, chunks: &[&str]) -> (String, usize) {
let result = collect_stream(parser, chunks);
(result.normal_text, result.calls.len())
}
pub(super) fn feed_external_parser(
parser: &mut impl ExternalToolParser,
tools: &[OpenAiTool],
chunks: &[&str],
) -> (String, usize) {
ExternalToolParser::reset(parser);
let mut normal_text = String::new();
let mut calls_len = 0;
for chunk in chunks {
let delta = parser
.parse_incremental(chunk, tools)
.now_or_never()
.expect("external parser should not suspend")
.expect("chunk should parse");
normal_text.push_str(&delta.normal_text);
calls_len += delta.calls.len();
}
calls_len += parser.get_unstreamed_tool_args().unwrap_or_default().len();
(normal_text, calls_len)
}