[Rust Frontend] Add reasoning/tool parser & renderer roundtrip tests (#43582)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-05-27 00:49:30 +00:00
committed by GitHub
parent dede691c95
commit 0b68f21e7c
11 changed files with 671 additions and 43 deletions
+541
View File
@@ -0,0 +1,541 @@
//! Text-level roundtrip tests for the real chat-template and output-processor pairing.
//!
//! The invariant under test is that a structured assistant message rendered as history can be
//! parsed from the generated assistant completion and then rendered back to the exact same
//! assistant-completion text.
use std::pin::Pin;
use std::sync::Arc;
use anyhow::{Context as _, Result, bail, ensure};
use futures::{Stream, StreamExt as _, stream};
use serde_json_fmt::JsonFormat as JsonFmt;
use serial_test::file_serial;
use vllm_chat::{
AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall,
ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason,
GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection,
RendererSelection, load_model_backends,
};
use vllm_text::{DecodedTextEvent, Finished, Prompt};
/// One model/parser configuration used to run the fixed roundtrip fixtures.
struct RoundtripCase {
/// Hugging Face model id resolved through the production backend loader.
model_id: &'static str,
/// Final assistant-history suffix rendered by the chat template but not
/// generated by the model body consumed by the output processor.
// TODO: we should adopt `ContinueFinalAssistant` mode to naturally handle this.
assistant_stop_suffix: &'static str,
/// Tool parser selection used by the output processor.
tool_call_parser: ParserSelection,
/// Reasoning parser selection used by the output processor.
reasoning_parser: ParserSelection,
/// JSON formatting expected after this model's template has materialized
/// tool-call arguments.
json_fmt: JsonFmt,
}
impl RoundtripCase {
/// Qwen3 XML tool-call format with `qwen3` reasoning tags.
fn qwen3() -> Self {
Self {
model_id: "Qwen/Qwen3-0.6B",
assistant_stop_suffix: "<|im_end|>\n",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
json_fmt: spaced_json_fmt(),
}
}
/// Qwen3.5 coder-style JSON tool-call format with `qwen3` reasoning tags.
fn qwen35() -> Self {
Self {
model_id: "Qwen/Qwen3.5-4B",
assistant_stop_suffix: "<|im_end|>\n",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
json_fmt: compact_json_fmt(),
}
}
/// MiniMax M2.5 XML invoke format with `<think>` reasoning tags.
fn minimax_m25() -> Self {
Self {
model_id: "MiniMaxAI/MiniMax-M2.5",
assistant_stop_suffix: "[e~[\n",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
json_fmt: compact_json_fmt(),
}
}
/// DeepSeek V4 DSML tool-call format.
fn deepseek_v4() -> Self {
Self {
model_id: "deepseek-ai/DeepSeek-V4-Flash",
assistant_stop_suffix: "<|end▁of▁sentence|>",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
json_fmt: compact_json_fmt(),
}
}
/// GLM-4.7 XML-like argument format with `<think>` reasoning tags.
fn glm47() -> Self {
Self {
model_id: "zai-org/GLM-4.7-Flash",
assistant_stop_suffix: "",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
json_fmt: compact_json_fmt(),
}
}
/// Kimi K2.5 tool-call format with `<think>` reasoning tags.
#[allow(dead_code)]
fn kimi_k25() -> Self {
Self {
model_id: "moonshotai/Kimi-K2.5",
assistant_stop_suffix: "<|im_end|>",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
json_fmt: spaced_json_fmt(),
}
}
}
macro_rules! roundtrip_tests {
($($case:ident => [$($fixture:ident),* $(,)?]),+ $(,)?) => {
paste::paste! {
$(
$(
#[tokio::test]
#[file_serial([<hf_ $case>])]
async fn [<roundtrip_ $case _ $fixture>]() -> Result<()> {
[<run_roundtrip_ $fixture>](RoundtripCase::$case()).await
}
)*
)+
}
};
}
roundtrip_tests! {
qwen3 => [reasoning_and_content, tool_call_mix],
qwen35 => [reasoning_and_content, tool_call_mix],
minimax_m25 => [reasoning_and_content, tool_call_mix],
deepseek_v4 => [reasoning_and_content, tool_call_mix],
glm47 => [reasoning_and_content, tool_call_mix],
// Note: Kimi K2.5 strips the reasoning content in history.
// TODO: we don't respect model-generated tool call id now so `tool_call_mix` cannot pass.
// kimi_k25 => [tool_call_mix],
}
/// Run the fixed reasoning+content fixture for one model/parser case.
async fn run_roundtrip_reasoning_and_content(case: RoundtripCase) -> Result<()> {
let backends = load_roundtrip_backends(&case).await?;
let request = roundtrip_request(
"roundtrip-reasoning-content",
vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
Vec::new(),
);
let expected_reasoning = "Need compute 2 + 2 directly.";
let expected_text = "The answer is 4.";
let result = run_roundtrip(
&case,
&backends,
&request,
AssistantMessage {
content: vec![
AssistantContentBlock::Reasoning {
text: expected_reasoning.to_string(),
},
AssistantContentBlock::Text {
text: expected_text.to_string(),
},
],
},
)
.await?;
assert_eq!(
result.parsed_message.reasoning().as_deref().map(str::trim),
Some(expected_reasoning)
);
assert_eq!(result.parsed_message.text().trim(), expected_text);
assert_eq!(result.parsed_message.tool_calls().count(), 0);
assert_eq!(
result.rerendered_closed_completion,
result.closed_completion
);
Ok(())
}
/// Run the fixed reasoning+multiple-tools fixture for one model/parser case.
async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
let backends = load_roundtrip_backends(&case).await?;
let request = roundtrip_request(
"roundtrip-reasoning-tools",
vec![ChatMessage::text(
ChatRole::User,
"Check Shanghai weather and add 1.00 plus 2.",
)],
test_tools(),
);
let expected_reasoning = "Need call the weather and add tools.";
let expected_text = "I will call the tools.";
let result = run_roundtrip(
&case,
&backends,
&request,
AssistantMessage {
content: vec![
AssistantContentBlock::Reasoning {
text: expected_reasoning.to_string(),
},
AssistantContentBlock::Text {
text: expected_text.to_string(),
},
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "functions.get_weather:0".to_string(),
name: "get_weather".to_string(),
arguments: r#"{"location":"Shanghai"}"#.to_string(),
}),
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "functions.add:1".to_string(),
name: "add".to_string(),
// Intentionally use a non-lexical order of keys and a different number
// formatting style to verify text-level fidelity of the roundtrip.
arguments: r#"{"y":1.00,"x":2}"#.to_string(),
}),
],
},
)
.await?;
assert_eq!(
result.parsed_message.reasoning().as_deref().map(str::trim),
Some(expected_reasoning)
);
assert_eq!(result.parsed_message.text().trim(), expected_text);
let tool_calls = result.parsed_message.tool_calls().collect::<Vec<_>>();
assert_eq!(
tool_calls.len(),
2,
"parsed message: {:#?}",
result.parsed_message
);
assert_eq!(tool_calls[0].name, "get_weather");
assert_eq!(
tool_calls[0].arguments,
expected_arguments(&case, r#"{"location": "Shanghai"}"#)?,
);
assert_eq!(tool_calls[1].name, "add");
assert_eq!(
tool_calls[1].arguments,
expected_arguments(&case, r#"{"y": 1.00, "x": 2}"#)?,
);
assert_eq!(
result.rerendered_closed_completion,
result.closed_completion
);
Ok(())
}
/// Compact JSON argument formatting used by JSON-native parsers/renderers.
fn compact_json_fmt() -> JsonFmt {
JsonFmt::new()
}
/// Python `json.dumps`-style compact formatting with a space after commas and
/// colons.
fn spaced_json_fmt() -> JsonFmt {
JsonFmt::new()
.comma(", ")
.expect("literal comma separator is valid JSON")
.colon(": ")
.expect("literal colon separator is valid JSON")
}
/// Parse and format expected tool-call arguments from raw JSON text.
/// Pass in a raw JSON string instead of a structured value to ensure the exact precision and
/// formatting of numbers are preserved.
fn expected_arguments(case: &RoundtripCase, raw_json: &str) -> Result<String> {
let value: serde_json::Value =
serde_json::from_str(raw_json).context("invalid expected tool-call arguments")?;
case.json_fmt
.format_to_string(&value)
.context("failed to format expected tool-call arguments")
}
/// Load the real model chat/text backend for one roundtrip case.
async fn load_roundtrip_backends(case: &RoundtripCase) -> Result<vllm_chat::LoadedModelBackends> {
load_model_backends(
case.model_id,
LoadModelBackendsOptions {
renderer: RendererSelection::Auto,
..Default::default()
},
)
.await
.with_context(|| format!("failed to load HF model files for {}", case.model_id))
}
/// Roundtrip artifacts needed for semantic and exact-text assertions.
struct RoundtripResult {
/// Final assistant message reconstructed by the output processor.
parsed_message: AssistantMessage,
/// Assistant-completion suffix cut from rendering the expected assistant as
/// history.
closed_completion: String,
/// Assistant-completion suffix cut after rendering the parsed assistant
/// back as history.
rerendered_closed_completion: String,
}
/// Render, parse, and rerender one assistant turn through the production
/// renderer/output-processor boundary.
async fn run_roundtrip(
case: &RoundtripCase,
backends: &vllm_chat::LoadedModelBackends,
request: &ChatRequest,
assistant: AssistantMessage,
) -> Result<RoundtripResult> {
let renderer = backends.chat_backend.chat_renderer();
let (prompt, closed_completion_text) =
render_closed_completion(renderer.as_ref(), request, &assistant)?;
let completion_body = closed_completion_text
.strip_suffix(case.assistant_stop_suffix)
.with_context(|| {
format!(
"closed assistant completion did not end with {:?}: {:?}",
case.assistant_stop_suffix, closed_completion_text
)
})?;
let parsed_message =
parse_completion(case, backends, request, &prompt, completion_body).await?;
let (_, rerendered_closed_completion) =
render_closed_completion(renderer.as_ref(), request, &parsed_message)?;
Ok(RoundtripResult {
parsed_message,
closed_completion: closed_completion_text,
rerendered_closed_completion,
})
}
/// Render `history` as a production prompt and `history + assistant` as closed
/// history, then return the production prompt and assistant-completion suffix.
fn render_closed_completion(
renderer: &dyn vllm_chat::ChatRenderer,
base_request: &ChatRequest,
assistant: &AssistantMessage,
) -> Result<(String, String)> {
let mut prompt_request = base_request.clone();
prompt_request.chat_options.generation_prompt_mode = GenerationPromptMode::StartNewAssistant;
let prompt = render_text(renderer, &prompt_request).context("failed to render prompt")?;
let mut full_request = base_request.clone();
full_request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
full_request.messages.push(ChatMessage::from(assistant.clone()));
let full = render_text(renderer, &full_request).context("failed to render full prompt")?;
ensure!(
full.starts_with(&prompt),
"full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
);
let completion = full[prompt.len()..].to_string();
Ok((prompt, completion))
}
/// Render one chat request and require a text prompt.
fn render_text(renderer: &dyn vllm_chat::ChatRenderer, request: &ChatRequest) -> Result<String> {
match renderer.render(request)?.prompt {
Prompt::Text(text) => Ok(text),
other => bail!("roundtrip tests expect text prompts, got {other:?}"),
}
}
/// Feed one rendered assistant completion body into the real output processor
/// and collect its terminal assistant message.
async fn parse_completion(
case: &RoundtripCase,
backends: &vllm_chat::LoadedModelBackends,
base_request: &ChatRequest,
prompt: &str,
completion_body: &str,
) -> Result<AssistantMessage> {
let tokenizer = backends.text_backend.tokenizer();
let prompt_token_ids = tokenizer
.encode(prompt, base_request.add_special_tokens)
.context("failed to encode rendered prompt")?;
let mut request = base_request.clone();
let processor = backends.chat_backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_call_parser: &case.tool_call_parser,
reasoning_parser: &case.reasoning_parser,
},
)?;
let decoded = decoded_completion_stream(prompt_token_ids, completion_body);
let mut events = processor.process(decoded)?;
while let Some(event) = events.next().await {
if let ChatEvent::Done { message, .. } = event? {
// TODO: currently our parsers are not very strict about preserving or trimming
// whitespace, so we trim here to avoid roundtrip failures due to
// insignificant whitespace differences. However, this may hurt token-level
// fidelity so we should consider improving them.
return Ok(message.trim());
}
}
bail!("output processor finished without a Done event")
}
/// Build a decoded-text stream from an already-rendered completion body.
///
/// The first event carries real prompt token ids so reasoning parsers can
/// initialize from the same prompt boundary production uses. Completion text is
/// split into small chunks to exercise streaming parser state across marker
/// and JSON boundaries.
fn decoded_completion_stream(
prompt_token_ids: Vec<u32>,
completion_body: &str,
) -> Pin<Box<dyn Stream<Item = vllm_chat::Result<DecodedTextEvent>> + Send>> {
let prompt_token_count = prompt_token_ids.len();
let mut events = vec![DecodedTextEvent::Start {
prompt_token_ids: Arc::from(prompt_token_ids.into_boxed_slice()),
prompt_logprobs: None,
}];
let chunks = split_by_chars(completion_body, 7);
if chunks.is_empty() {
events.push({
DecodedTextEvent::TextDelta {
delta: String::new(),
token_ids: Vec::new(),
logprobs: None,
finished: Some(Finished {
prompt_token_count: 0,
output_token_count: 0,
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
}
});
} else {
let last_index = chunks.len() - 1;
for (index, chunk) in chunks.into_iter().enumerate() {
let finished = (index == last_index).then(|| Finished {
prompt_token_count,
output_token_count: completion_body.chars().count(),
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
});
events.push(DecodedTextEvent::TextDelta {
delta: chunk,
token_ids: Vec::new(),
logprobs: None,
finished,
});
}
}
stream::iter(events).map(Ok).boxed()
}
/// Split text into chunks containing at most `chunk_chars` Unicode scalar
/// values.
fn split_by_chars(text: &str, chunk_chars: usize) -> Vec<String> {
let mut chunks = Vec::new();
let mut start = 0;
let mut count = 0;
for (index, _) in text.char_indices() {
if count == chunk_chars {
chunks.push(text[start..index].to_string());
start = index;
count = 0;
}
count += 1;
}
if start < text.len() {
chunks.push(text[start..].to_string());
}
chunks
}
/// Build a chat request fixture with parser-enabling tool-choice semantics.
fn roundtrip_request(
request_id: impl Into<String>,
messages: Vec<ChatMessage>,
tools: Vec<ChatTool>,
) -> ChatRequest {
let mut request = ChatRequest {
request_id: request_id.into(),
messages,
tool_choice: if tools.is_empty() {
ChatToolChoice::None
} else {
ChatToolChoice::Auto
},
tools,
..ChatRequest::for_test()
};
// Enable thinking for some models so that rendering and parsing the reasoning block is
// exercised in the roundtrip.
for key in ["thinking", "enable_thinking"] {
request.chat_options.template_kwargs.insert(key.to_string(), true.into());
}
request
}
/// Return the function tools used by the multiple-tool-call fixture.
fn test_tools() -> Vec<ChatTool> {
vec![
ChatTool {
name: "get_weather".to_string(),
description: Some("Get weather for a location".to_string()),
parameters: serde_json::json!({
"type": "object",
"properties": {
"location": { "type": "string" }
},
"required": ["location"]
}),
strict: None,
},
ChatTool {
name: "add".to_string(),
description: Some("Add two integers".to_string()),
parameters: serde_json::json!({
"type": "object",
"properties": {
"y": { "type": "number" },
"x": { "type": "number" }
},
"required": ["y", "x"]
}),
strict: None,
},
]
}