iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Add HY3 unified parser and local XGrammar structural-tag builder (#53054)

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-08-20 19:13:54 -07:00
committed by GitHub
parent c6e19b3be2
commit df3b3422b4
17 changed files with 849 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/chat/src/lib.rs
+1 -1
View File
@@ -415,6 +415,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
rust/src/chat/src/parser/reasoning/mod.rs
+3
View File
@@ -25,6 +25,7 @@ pub mod names {
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
pub const GLM45: &str = "glm45";
pub const HY_V3: &str = "hy_v3";
pub const KIMI: &str = "kimi";
pub const KIMI_K2: &str = "kimi_k2";
pub const KIMI_K3: &str = "kimi_k3";
@@ -67,6 +68,7 @@ impl ReasoningParserFactory {
.register_unified_dummy(names::GEMMA4)
.register_unified_dummy(names::INKLING)
.register_parser::<Glm45ReasoningParser>(names::GLM45)
.register_unified_dummy(names::HY_V3)
.register_parser::<KimiReasoningParser>(names::KIMI)
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
.register_unified_dummy(names::KIMI_K3)
@@ -91,6 +93,7 @@ impl ReasoningParserFactory {
.register_pattern("glm-4.7", names::GLM45)
.register_pattern("glm-4.6", names::GLM45)
.register_pattern("glm-4.5", names::GLM45)
.register_pattern("hy3", names::HY_V3)
.register_pattern("kimi-k2", names::KIMI_K2)
.register_pattern("kimi", names::KIMI)
// step3p5 patterns must precede `step3`: substring matching would
rust/src/chat/src/parser/tool/mod.rs
+2 -3
View File
@@ -7,7 +7,7 @@ use std::sync::{Arc, LazyLock};
pub use vllm_parser::tool::{
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, HyV3ToolParser,
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
@@ -76,7 +76,7 @@ impl ToolParserFactory {
.register_unified_dummy(names::INKLING)
.register_parser::<Granite4ToolParser>(names::GRANITE4)
.register_parser::<HermesToolParser>(names::HERMES)
.register_parser::<HyV3ToolParser>(names::HY_V3)
.register_unified_dummy(names::HY_V3)
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
.register_unified_dummy(names::KIMI_K3)
@@ -100,7 +100,6 @@ impl ToolParserFactory {
.register_pattern("qwen3", names::QWEN3_XML)
.register_pattern("hermes", names::HERMES)
.register_pattern("hy3", names::HY_V3)
.register_pattern("hy_v3", names::HY_V3)
// Narrow to `internlm2` substring so it matches `internlm2-chat-7b`
// and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1,
// routes to Llama), `internlm3-*` (also Llama-architecture per
rust/src/chat/src/parser/unified.rs
+5 -1
View File
@@ -6,7 +6,8 @@
use std::sync::LazyLock;
pub use vllm_parser::unified::{
Gemma4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, UnifiedParser,
Gemma4UnifiedParser, HyV3UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser,
UnifiedParser,
};
use vllm_tokenizer::DynTokenizer;
@@ -16,6 +17,7 @@ use crate::request::ChatTool;
/// Canonical public names for registered unified parsers.
pub mod names {
pub const GEMMA4: &str = "gemma4";
pub const HY_V3: &str = "hy_v3";
pub const INKLING: &str = "inkling";
pub const KIMI_K3: &str = "kimi_k3";
}
@@ -41,12 +43,14 @@ impl UnifiedParserFactory {
let mut factory = Self::default();
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
factory.register_parser::<HyV3UnifiedParser>(names::HY_V3);
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
factory
.register_pattern("gemma-4", names::GEMMA4)
.register_pattern("gemma4", names::GEMMA4)
.register_pattern("hy3", names::HY_V3)
.register_pattern("inkling", names::INKLING)
.register_pattern("kimi-k3", names::KIMI_K3)
.register_pattern("kimi_k3", names::KIMI_K3);
rust/src/chat/tests/roundtrip.rs
+60 -7
View File
@@ -55,6 +55,12 @@ enum ThinkingBehavior {
Toggleable { default: bool },
/// The chat template always behaves as `value` for this fixture.
Always { value: bool },
/// The chat template selects thinking mode through `reasoning_effort`.
ReasoningEffort {
default: bool,
enabled: &'static str,
disabled: &'static str,
},
}
impl ThinkingBehavior {
@@ -62,6 +68,7 @@ impl ThinkingBehavior {
match self {
Self::Toggleable { default } => default,
Self::Always { value } => value,
Self::ReasoningEffort { default, .. } => default,
}
}
@@ -76,6 +83,33 @@ impl ThinkingBehavior {
Some(value), // explicitly request the supported thinking behavior
None, // use default template behavior
],
Self::ReasoningEffort { .. } => vec![
Some(true), // explicitly enable thinking
Some(false), // explicitly disable thinking
None, // use default template behavior
],
}
}
fn apply(self, request: &mut ChatRequest, thinking: Option<bool>) {
let Some(thinking) = thinking else {
return;
};
match self {
Self::Toggleable { .. } | Self::Always { .. } => {
for key in ["thinking", "enable_thinking"] {
request.chat_options.template_kwargs.insert(key.to_string(), thinking.into());
}
}
Self::ReasoningEffort {
enabled, disabled, ..
} => {
request.chat_options.template_kwargs.insert(
"reasoning_effort".to_string(),
if thinking { enabled } else { disabled }.into(),
);
}
}
}
}
@@ -242,6 +276,23 @@ impl RoundtripCase {
}
}
/// HY3 suffixed reasoning/tool markers discovered from tokenizer added vocab.
fn hy_v3() -> Self {
Self {
model_id: "tencent/Hy3",
assistant_stop_suffix: "<｜hy_eos:opensource｜>",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
thinking_behavior: ThinkingBehavior::ReasoningEffort {
default: false,
enabled: "high",
disabled: "no_think",
},
json_fmt: compact_json_fmt(),
sort_json_keys: false,
}
}
/// SeedOSS with `<seed:think>` / `</seed:think>` reasoning tags.
fn seed_oss() -> Self {
Self {
@@ -345,6 +396,9 @@ roundtrip_tests! {
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
// K3 drops plain-assistant reasoning in history; tool-call turns keep it.
kimi_k3 => [tool_call_mix],
// HY3's final plain-assistant history omits EOS; tool-call history keeps it
// and exercises both stages of the tokenizer-derived unified parser.
hy_v3 => [tool_call_mix],
gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
inkling => [reasoning_and_content, tool_call_mix],
}
@@ -370,6 +424,7 @@ async fn run_roundtrip_reasoning_and_content_inner(
vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
Vec::new(),
thinking,
case.thinking_behavior,
);
let expected_reasoning = "Need compute 2 + 2 directly.";
let expected_text = "The answer is 4.";
@@ -417,6 +472,7 @@ async fn run_roundtrip_tool_call_mix(
)],
test_tools(),
Some(true), // always enable thinking in this fixture
case.thinking_behavior,
);
let expected_reasoning = "Need call the weather and add tools.";
let expected_text = "I will call the tools.";
@@ -838,6 +894,7 @@ fn roundtrip_request(
messages: Vec<ChatMessage>,
tools: Vec<ChatTool>,
thinking: Option<bool>,
thinking_behavior: ThinkingBehavior,
) -> ChatRequest {
let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true)
.expect("tool context should resolve");
@@ -848,13 +905,9 @@ fn roundtrip_request(
..ChatRequest::for_test()
};
// Explicitly enable or disable thinking so that rendering and parsing the reasoning block is
// exercised or skipped in the roundtrip. If unspecified, use the default template behavior.
if let Some(thinking) = thinking {
for key in ["thinking", "enable_thinking"] {
request.chat_options.template_kwargs.insert(key.to_string(), thinking.into());
}
}
// Explicitly enable or disable thinking using the controls understood by
// this model's template. If unspecified, use the template default.
thinking_behavior.apply(&mut request, thinking);
request
}
rust/src/parser/src/reasoning/delimited.rs
+13 -9
View File
@@ -35,25 +35,29 @@ impl DelimitedReasoningParser {
/// start or end delimiter, that prompt boundary always wins.
pub(crate) fn new(
tokenizer: DynTokenizer,
start_token: &'static str,
end_token: &'static str,
start_token: impl Into<String>,
end_token: impl Into<String>,
default_in_reasoning: bool,
) -> Result<Self> {
let start_token = start_token.into();
let end_token = end_token.into();
let start_token_id =
tokenizer.token_to_id(start_token).ok_or_else(|| ReasoningError::MissingToken {
token: start_token.to_string(),
})?;
tokenizer
.token_to_id(&start_token)
.ok_or_else(|| ReasoningError::MissingToken {
token: start_token.clone(),
})?;
let end_token_id =
tokenizer.token_to_id(end_token).ok_or_else(|| ReasoningError::MissingToken {
token: end_token.to_string(),
tokenizer.token_to_id(&end_token).ok_or_else(|| ReasoningError::MissingToken {
token: end_token.clone(),
})?;
Ok(Self {
tokenizer,
current_in_reasoning: default_in_reasoning,
buffer: String::new(),
start_token: start_token.to_string(),
end_token: end_token.to_string(),
start_token,
end_token,
start_token_id,
end_token_id,
default_in_reasoning,
rust/src/parser/src/reasoning/hy_v3.rs
+51
View File
@@ -0,0 +1,51 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_tokenizer::DynTokenizer;
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result};
/// Internal HY3 reasoning stage used by the unified HY3 parser.
pub(crate) struct HyV3ReasoningParser {
inner: DelimitedReasoningParser,
}
impl HyV3ReasoningParser {
/// Create a HY3 reasoning parser for the tokenizer-specific marker suffix.
pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(
tokenizer,
format!("<think{suffix}>"),
format!("</think{suffix}>"),
false,
)?,
})
}
}
impl ReasoningParser for HyV3ReasoningParser {
// Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
// tool delimiters always use the same tokenizer-derived value.
fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
where
Self: Sized + 'static,
{
Err(ReasoningError::DummyUnifiedParser {
name: "hy_v3".to_string(),
})
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids);
Ok(())
}
fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
Ok(self.inner.push(delta))
}
fn finish(&mut self) -> Result<ReasoningDelta> {
Ok(self.inner.finish())
}
}
rust/src/parser/src/reasoning/mod.rs
+2
View File
@@ -20,6 +20,7 @@
mod cohere_cmd;
mod deepseek_r1;
mod delimited;
mod hy_v3;
mod kimi;
mod minimax_m3;
mod qwen3;
@@ -32,6 +33,7 @@ use vllm_tokenizer::DynTokenizer;
pub use self::cohere_cmd::CohereCmdReasoningParser;
pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary};
pub(crate) use self::hy_v3::HyV3ReasoningParser;
pub use self::kimi::KimiReasoningParser;
pub use self::minimax_m3::MiniMaxM3ReasoningParser;
pub use self::qwen3::Qwen3ReasoningParser;
rust/src/parser/src/tool/hy_v3.rs
+142 -62
View File
@@ -1,26 +1,65 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
mod structural_tag;
use std::sync::Arc;
use winnow::ascii::multispace0 as ws0;
use winnow::combinator::{alt, delimited, eof, repeat, seq, terminated};
use winnow::prelude::*;
use winnow::stream::Partial;
use winnow::token::{literal, rest, take_until};
use self::structural_tag::HyV3StructuralTagBuilder;
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
use super::{Result, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput};
use crate::tool::{StructuralTagBuilder, Tool};
const TOOL_CALLS_START: &str = "<tool_calls>";
const TOOL_CALLS_END: &str = "</tool_calls>";
const TOOL_CALL_START: &str = "<tool_call>";
const TOOL_CALL_END: &str = "</tool_call>";
const TOOL_SEP: &str = "<tool_sep>";
const ARG_KEY_START: &str = "<arg_key>";
const ARG_KEY_END: &str = "</arg_key>";
const ARG_VALUE_START: &str = "<arg_value>";
const ARG_VALUE_END: &str = "</arg_value>";
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct HyV3ToolMarkers {
tool_calls_start: String,
tool_calls_end: String,
tool_call_start: String,
tool_call_end: String,
tool_sep: String,
arg_key_start: String,
arg_key_end: String,
arg_value_start: String,
arg_value_end: String,
}
impl HyV3ToolMarkers {
pub(crate) fn new(suffix: &str) -> Self {
Self {
tool_calls_start: format!("<tool_calls{suffix}>"),
tool_calls_end: format!("</tool_calls{suffix}>"),
tool_call_start: format!("<tool_call{suffix}>"),
tool_call_end: format!("</tool_call{suffix}>"),
tool_sep: format!("<tool_sep{suffix}>"),
arg_key_start: format!("<arg_key{suffix}>"),
arg_key_end: format!("</arg_key{suffix}>"),
arg_value_start: format!("<arg_value{suffix}>"),
arg_value_end: format!("</arg_value{suffix}>"),
}
}
pub(crate) fn iter(&self) -> impl Iterator<Item = &str> {
[
self.tool_calls_start.as_str(),
self.tool_calls_end.as_str(),
self.tool_call_start.as_str(),
self.tool_call_end.as_str(),
self.tool_sep.as_str(),
self.arg_key_start.as_str(),
self.arg_key_end.as_str(),
self.arg_value_start.as_str(),
self.arg_value_end.as_str(),
]
.into_iter()
}
}
type HyV3Input<'i> = Partial<&'i str>;
@@ -60,21 +99,26 @@ enum HyV3Event {
/// Arguments are emitted only after a full `<tool_call>` block is parsed.
/// HY3 marker tokens are added-vocabulary tokens rather than tokenizer special
/// tokens, so the default `preserve_special_tokens() == false` is sufficient.
pub struct HyV3ToolParser {
pub(crate) struct HyV3ToolParser {
buffer: String,
mode: HyV3Mode,
emitted_tool_count: usize,
tool_parameters: ToolSchemas,
markers: Arc<HyV3ToolMarkers>,
structural_tag_builder: HyV3StructuralTagBuilder,
}
impl HyV3ToolParser {
/// Create a HY3 tool parser.
fn new(tools: &[Tool]) -> Self {
pub(crate) fn new(tools: &[Tool], suffix: &str) -> Self {
let markers = Arc::new(HyV3ToolMarkers::new(suffix));
Self {
buffer: String::new(),
mode: HyV3Mode::Text,
emitted_tool_count: 0,
tool_parameters: ToolSchemas::from_tools(tools),
markers: Arc::clone(&markers),
structural_tag_builder: HyV3StructuralTagBuilder::new(markers),
}
}
@@ -109,22 +153,26 @@ impl HyV3ToolParser {
}
impl ToolParser for HyV3ToolParser {
fn create(tools: &[Tool]) -> Result<Box<dyn ToolParser>>
// Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
// tool delimiters always use the same tokenizer-derived value.
fn create(_tools: &[Tool]) -> Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self::new(tools)))
Err(ToolParserError::DummyUnifiedParser {
name: "hy_v3".to_string(),
})
}
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
Some(xgrammar_structural_tag::Model::HyV3.builder())
Some(&self.structural_tag_builder)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.buffer.push_str(chunk);
while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
parse_next_hy_v3_event(input, &mut self.mode)
parse_next_hy_v3_event(input, &mut self.mode, &self.markers)
})? {
self.apply_event(event, output)?;
self.buffer.drain(..consumed_len);
@@ -155,62 +203,83 @@ impl ToolParser for HyV3ToolParser {
fn parse_next_hy_v3_event(
input: &mut HyV3Input<'_>,
mode: &mut HyV3Mode,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
match mode {
HyV3Mode::Text => parse_text_event(input),
HyV3Mode::Text => parse_text_event(input, markers),
HyV3Mode::ToolBlock { tool_call_end_scan } => {
parse_tool_block_event(input, tool_call_end_scan)
parse_tool_block_event(input, tool_call_end_scan, markers)
}
HyV3Mode::Done => ignored_rest_event(input),
}
}
/// Parse a text-mode HY3 event.
fn parse_text_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
alt((tool_block_start_event, safe_text_event)).parse_next(input)
fn parse_text_event(
input: &mut HyV3Input<'_>,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
alt((
|input: &mut HyV3Input<'_>| tool_block_start_event(input, markers),
|input: &mut HyV3Input<'_>| safe_text_event(input, markers),
))
.parse_next(input)
}
/// Parse a HY3 tool-block start marker.
fn tool_block_start_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
literal(TOOL_CALLS_START).value(HyV3Event::ToolBlockStart).parse_next(input)
fn tool_block_start_event(
input: &mut HyV3Input<'_>,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
literal(markers.tool_calls_start.as_str())
.value(HyV3Event::ToolBlockStart)
.parse_next(input)
}
/// Parse a safe text run before the next HY3 marker.
fn safe_text_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
safe_text_len(input, TOOL_CALLS_START).map(|len| HyV3Event::Text { len })
fn safe_text_event(input: &mut HyV3Input<'_>, markers: &HyV3ToolMarkers) -> ModalResult<HyV3Event> {
safe_text_len(input, &markers.tool_calls_start).map(|len| HyV3Event::Text { len })
}
/// Parse one event inside a HY3 tool block.
fn parse_tool_block_event(
input: &mut HyV3Input<'_>,
tool_call_end_scan: &mut MarkerScanState,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
alt((tool_block_end_event, |input: &mut HyV3Input<'_>| {
tool_call_event(input, tool_call_end_scan)
}))
alt((
|input: &mut HyV3Input<'_>| tool_block_end_event(input, markers),
|input: &mut HyV3Input<'_>| tool_call_event(input, tool_call_end_scan, markers),
))
.parse_next(input)
}
/// Parse a HY3 tool-block end marker.
fn tool_block_end_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
(ws0, literal(TOOL_CALLS_END)).value(HyV3Event::ToolBlockEnd).parse_next(input)
fn tool_block_end_event(
input: &mut HyV3Input<'_>,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
(ws0, literal(markers.tool_calls_end.as_str()))
.value(HyV3Event::ToolBlockEnd)
.parse_next(input)
}
/// Parse a complete HY3 tool-call block.
fn tool_call_event(
input: &mut HyV3Input<'_>,
tool_call_end_scan: &mut MarkerScanState,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
let (name, body) = seq!(
_: ws0,
_: literal(TOOL_CALL_START),
take_until(0.., TOOL_SEP),
_: literal(TOOL_SEP),
take_until_marker(TOOL_CALL_END, tool_call_end_scan),
_: literal(TOOL_CALL_END),
_: literal(markers.tool_call_start.as_str()),
take_until(0.., markers.tool_sep.as_str()),
_: literal(markers.tool_sep.as_str()),
take_until_marker(markers.tool_call_end.as_str(), tool_call_end_scan),
_: literal(markers.tool_call_end.as_str()),
)
.parse_next(input)?;
let raw_params = parse_tool_call_params(body)?;
let raw_params = parse_tool_call_params(body, markers)?;
Ok(HyV3Event::ToolCall {
name: name.trim().to_string(),
@@ -219,21 +288,32 @@ fn tool_call_event(
}
/// Parse all parameter blocks inside a complete HY3 tool call.
fn parse_tool_call_params(tool_call_body: &str) -> ModalResult<Vec<(String, String)>> {
fn parse_tool_call_params(
tool_call_body: &str,
markers: &HyV3ToolMarkers,
) -> ModalResult<Vec<(String, String)>> {
let mut input = tool_call_body;
delimited(ws0, repeat(0.., terminated(parameter, ws0)), eof).parse_next(&mut input)
delimited(
ws0,
repeat(
0..,
terminated(|input: &mut &str| parameter(input, markers), ws0),
),
eof,
)
.parse_next(&mut input)
}
/// Parse a HY3 argument key/value block.
fn parameter(input: &mut &str) -> ModalResult<(String, String)> {
fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String, String)> {
let (name, value) = seq!(
_: literal(ARG_KEY_START),
take_until(0.., ARG_KEY_END),
_: literal(ARG_KEY_END),
_: literal(markers.arg_key_start.as_str()),
take_until(0.., markers.arg_key_end.as_str()),
_: literal(markers.arg_key_end.as_str()),
_: ws0,
_: literal(ARG_VALUE_START),
take_until(0.., ARG_VALUE_END),
_: literal(ARG_VALUE_END),
_: literal(markers.arg_value_start.as_str()),
take_until(0.., markers.arg_value_end.as_str()),
_: literal(markers.arg_value_end.as_str()),
)
.parse_next(input)?;
@@ -274,14 +354,14 @@ mod tests {
#[test]
fn hy_v3_does_not_preserve_special_tokens() {
let parser = HyV3ToolParser::new(&test_tools());
let parser = HyV3ToolParser::new(&test_tools(), "");
assert!(!parser.preserve_special_tokens());
}
#[test]
fn hy_v3_parse_complete_without_tool_call_keeps_text() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser.parse_complete("This is a plain response.").unwrap();
assert_eq!(output.normal_text(), "This is a plain response.");
@@ -290,7 +370,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_zero_arg_inline_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser
.parse_complete(
"<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>",
@@ -305,7 +385,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_zero_arg_newline_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser
.parse_complete(
"<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>",
@@ -318,7 +398,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_arguments_on_same_line() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser
.parse_complete(
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call></tool_calls>",
@@ -333,7 +413,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_arguments_with_newlines() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser
.parse_complete(&build_tool_calls(&[build_tool_call(
"get_weather",
@@ -349,7 +429,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_preserves_prefix_and_ignores_trailing_text() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser
.parse_complete(&format!(
"Checking.{} trailing text",
@@ -363,7 +443,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_multiple_tool_calls_in_one_block() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser
.parse_complete(&build_tool_calls(&[
build_tool_call(
@@ -406,7 +486,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_converts_schema_types() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = parser
.parse_complete(&build_tool_calls(&[build_tool_call(
"convert",
@@ -432,7 +512,7 @@ mod tests {
#[test]
fn hy_v3_streaming_without_tool_call_emits_text_incrementally() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut output = ToolParserOutput::default();
output.append(parser.parse_chunk("This is ").unwrap());
@@ -446,7 +526,7 @@ mod tests {
#[test]
fn hy_v3_streaming_extracts_zero_arg_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let chunks = [
"<tool_calls>",
"\n<tool_call>",
@@ -465,7 +545,7 @@ mod tests {
#[test]
fn hy_v3_streaming_extracts_arguments() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let chunks = [
"<tool_calls>",
"\n<tool_call>",
@@ -491,7 +571,7 @@ mod tests {
#[test]
fn hy_v3_streaming_preserves_prefix_text() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let chunks = [
"Checking.",
"<tool_calls>",
@@ -521,7 +601,7 @@ mod tests {
),
]);
let chunks = split_by_chars(&input, 9);
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = collect_stream(&mut parser, &chunks);
@@ -537,7 +617,7 @@ mod tests {
build_tool_calls(&[build_tool_call("get_weather", &[("city", "Beijing")])])
);
let chunks = split_by_chars(&input, 5);
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let output = collect_stream(&mut parser, &chunks);
@@ -548,7 +628,7 @@ mod tests {
#[test]
fn hy_v3_streaming_does_not_emit_incomplete_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut output = ToolParserOutput::default();
parser
@@ -564,7 +644,7 @@ mod tests {
#[test]
fn hy_v3_finish_fails_incomplete_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
parser.parse_chunk("<tool_calls><tool_call>get_weather<tool_sep>").unwrap();
let error = parser.finish().unwrap_err();
@@ -575,7 +655,7 @@ mod tests {
#[test]
fn hy_v3_malformed_tool_call_fails_fast() {
let mut parser = HyV3ToolParser::new(&test_tools());
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let error = parser
.parse_complete(
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</tool_call></tool_calls>",
rust/src/parser/src/tool/hy_v3/structural_tag.rs
+261
View File
@@ -0,0 +1,261 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Structural-tag grammar for HY3 XML-style tool calls.
use std::collections::HashSet;
use std::sync::Arc;
use serde_json::Value;
use xgrammar_structural_tag::Result;
use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagContext};
use xgrammar_structural_tag::format::{Format, StructuralTag, TagFormat};
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
use super::HyV3ToolMarkers;
/// HY3 structural-tag builder using tokenizer-specific structural markers.
#[derive(Debug, Clone)]
pub(super) struct HyV3StructuralTagBuilder {
markers: Arc<HyV3ToolMarkers>,
}
impl HyV3StructuralTagBuilder {
pub(super) fn new(markers: Arc<HyV3ToolMarkers>) -> Self {
Self { markers }
}
fn argument_pair(&self, key: &str) -> Format {
let excludes = self.markers.iter().collect::<Vec<_>>();
Format::sequence(vec![
Format::const_string(&self.markers.arg_key_start),
Format::const_string(key),
Format::const_string(&self.markers.arg_key_end),
Format::const_string("\n"),
Format::const_string(&self.markers.arg_value_start),
Format::any_text_excluding(&excludes),
Format::const_string(&self.markers.arg_value_end),
Format::const_string("\n"),
])
}
fn tool_call(&self, tool: &FunctionToolParam) -> TagFormat {
let (required_keys, optional_keys) = argument_keys(tool.function.parameters.as_ref());
let mut elements =
required_keys.into_iter().map(|key| self.argument_pair(key)).collect::<Vec<_>>();
if !optional_keys.is_empty() {
let mut pairs =
optional_keys.into_iter().map(|key| self.argument_pair(key)).collect::<Vec<_>>();
let optional = if pairs.len() == 1 {
pairs.pop().unwrap()
} else {
Format::or(pairs)
};
elements.push(Format::star(optional));
}
let content = if elements.is_empty() {
Format::any_text()
} else {
Format::sequence(elements)
};
TagFormat::new(
format!(
"{}{}{}\n",
self.markers.tool_call_start, tool.function.name, self.markers.tool_sep
),
content,
self.markers.tool_call_end.clone(),
)
}
fn tool_calls(&self, tools: &[FunctionToolParam], choice: BuilderToolChoice) -> Format {
let mut calls = tools.iter().map(|tool| self.tool_call(tool)).collect::<Vec<_>>();
let begin = format!("{}\n", self.markers.tool_calls_start);
let end = format!("\n{}", self.markers.tool_calls_end);
match choice {
BuilderToolChoice::Auto if calls.is_empty() => Format::any_text(),
BuilderToolChoice::Auto => {
let outer = TagFormat::new(
begin,
Format::tags_with_separator(calls, "\n", true, false),
end,
);
Format::triggered_tags(&[&self.markers.tool_calls_start], vec![outer])
}
BuilderToolChoice::Forced => Format::sequence(vec![
Format::const_string(begin),
Format::Tag(calls.pop().unwrap()),
Format::const_string(end),
]),
BuilderToolChoice::Required => Format::sequence(vec![
Format::const_string(begin),
Format::tags_with_separator(calls, "\n", true, false),
Format::const_string(end),
]),
}
}
}
impl StructuralTagBuilder for HyV3StructuralTagBuilder {
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
Ok(StructuralTag::new(
self.tool_calls(ctx.function_tools, ctx.tool_choice),
))
}
}
/// Split argument keys into required and optional declaration-order groups.
fn argument_keys(parameters: Option<&Value>) -> (Vec<&str>, Vec<&str>) {
let Some(parameters) = parameters.and_then(Value::as_object) else {
return (Vec::new(), Vec::new());
};
let properties = parameters.get("properties").and_then(Value::as_object);
let required = parameters
.get("required")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.collect::<Vec<_>>();
let required_set = required.iter().copied().collect::<HashSet<_>>();
let mut required_keys = properties
.into_iter()
.flat_map(|properties| properties.keys())
.map(String::as_str)
.filter(|key| required_set.contains(key))
.collect::<Vec<_>>();
required_keys.extend(
required
.into_iter()
.filter(|key| properties.is_none_or(|properties| !properties.contains_key(*key))),
);
let optional_keys = properties
.into_iter()
.flat_map(|properties| properties.keys())
.map(String::as_str)
.filter(|key| !required_set.contains(key))
.collect();
(required_keys, optional_keys)
}
#[cfg(test)]
mod tests {
use expect_test::expect;
use serde_json::{Value, json};
use std::sync::Arc;
use xgrammar_structural_tag::builders::StructuralTagOptions;
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
};
use super::HyV3StructuralTagBuilder;
use crate::tool::HyV3ToolMarkers;
fn tool(name: &str, parameters: Value) -> ToolParam {
ToolParam::Function(FunctionToolParam::new(
FunctionDefinition::new(name).with_parameters(parameters),
))
}
fn build(
suffix: &str,
tools: &[ToolParam],
choice: ToolChoice,
) -> xgrammar_structural_tag::format::StructuralTag {
build_structural_tag(
HyV3StructuralTagBuilder::new(Arc::new(HyV3ToolMarkers::new(suffix))),
tools,
choice,
StructuralTagOptions::default().with_reasoning(false),
)
.unwrap()
}
#[test]
fn required_uses_suffixed_hy3_skeleton_and_bounded_values() {
let tag = build(
":opensource",
&[tool(
"get_weather",
json!({
"type": "object",
"properties": {
"city": { "type": "string" },
"days": { "type": "integer" }
},
"required": ["city"]
}),
)],
ToolChoice::required(),
);
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
}
#[test]
fn optional_only_schema_keeps_declared_key_alternatives() {
let tag = build(
"",
&[tool(
"lookup",
json!({
"type": "object",
"properties": {
"query": { "type": "string" },
"limit": { "type": "integer" }
}
}),
)],
ToolChoice::required(),
);
let value = serde_json::to_value(tag).unwrap();
expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
&serde_json::to_string(
&value["format"]["elements"][1]["tags"][0]["content"],
)
.unwrap(),
);
}
#[test]
fn auto_and_forced_preserve_tool_choice_shape() {
let tools = [
tool("search", json!({ "type": "object" })),
tool("lookup", json!({ "type": "object" })),
];
let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap();
let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap();
expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
}
#[test]
fn missing_required_property_is_still_emitted() {
let tag = build(
"",
&[tool(
"search",
json!({
"type": "object",
"properties": { "query": { "type": "string" } },
"required": ["query", "tenant"]
}),
)],
ToolChoice::required(),
);
let value = serde_json::to_value(tag).unwrap();
expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
&serde_json::to_string(
&value["format"]["elements"][1]["tags"][0]["content"],
)
.unwrap(),
);
}
}
rust/src/parser/src/tool/mod.rs
+1 -1
View File
@@ -24,7 +24,7 @@ pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
pub use error::{Result, ToolParserError};
pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
pub use hy_v3::HyV3ToolParser;
pub(crate) use hy_v3::{HyV3ToolMarkers, HyV3ToolParser};
pub use json::{
Granite4ToolParser, HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser,
MistralToolParser, Phi4MiniJsonToolParser, Qwen3XmlToolParser,
rust/src/parser/src/unified/hy_v3.rs
+207
View File
@@ -0,0 +1,207 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_tokenizer::{DynTokenizer, Tokenizer};
use super::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
use crate::reasoning::HyV3ReasoningParser;
use crate::tool::{HyV3ToolMarkers, HyV3ToolParser, StructuralTagBuilder, Tool};
const HY_V3_MARKER_STEMS: &[&str] = &[
"think",
"tool_calls",
"tool_call",
"tool_sep",
"arg_key",
"arg_value",
];
/// Unified reasoning and tool parser for HY3 output.
pub struct HyV3UnifiedParser {
inner: CombinedParser,
}
impl HyV3UnifiedParser {
/// Create a HY3 parser using the suffix encoded in tokenizer added tokens.
pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
let suffix = detect_token_suffix(tokenizer.as_ref());
let markers = HyV3ToolMarkers::new(&suffix);
for marker in markers.iter() {
token_id(tokenizer.as_ref(), marker)?;
}
let reasoning = HyV3ReasoningParser::new(tokenizer, &suffix)?;
let tool = HyV3ToolParser::new(tools, &suffix);
Ok(Self {
inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))),
})
}
}
impl UnifiedParser for HyV3UnifiedParser {
fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
where
Self: Sized + 'static,
{
Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids)
}
fn preserve_special_tokens(&self) -> bool {
self.inner.preserve_special_tokens()
}
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
self.inner.structural_tag_builder()
}
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
self.inner.tool_call_id(tool_index)
}
fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> {
self.inner.parse_into(delta, output)
}
fn finish(&mut self) -> Result<UnifiedParserOutput> {
self.inner.finish()
}
fn reset(&mut self) -> String {
self.inner.reset()
}
}
/// Detect the HY3 structural-token suffix from tokenizer added vocabulary.
fn detect_token_suffix(tokenizer: &dyn Tokenizer) -> String {
tokenizer
.added_vocab()
.iter()
.filter_map(|(token, id)| marker_suffix(token).map(|suffix| (*id, suffix)))
.min_by_key(|(id, _)| *id)
.map(|(_, suffix)| suffix.to_string())
.unwrap_or_default()
}
/// Extract the suffix from one opening or closing HY3 structural token.
fn marker_suffix(token: &str) -> Option<&str> {
let body = token.strip_prefix('<')?.strip_suffix('>')?;
let body = body.strip_prefix('/').unwrap_or(body);
HY_V3_MARKER_STEMS.iter().find_map(|stem| {
let suffix = body.strip_prefix(stem)?;
(suffix.is_empty() || suffix.starts_with(':')).then_some(suffix)
})
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use serde_json::json;
use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
use xgrammar_structural_tag::builders::StructuralTagOptions;
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
};
use super::{HyV3UnifiedParser, UnifiedParser};
use crate::tool::Tool;
use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
fn tokenizer() -> TestTokenizer {
[
"<think:opensource>",
"</think:opensource>",
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>",
]
.into_iter()
.enumerate()
.fold(TestTokenizer::new(), |tokenizer, (index, token)| {
tokenizer.with_regular_token(token, 1000 + index as u32)
})
}
fn tools() -> Vec<Tool> {
vec![Tool {
name: "get_weather".to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": { "city": { "type": "string" } },
}),
strict: None,
}]
}
#[test]
fn parses_suffixed_reasoning_and_tool_call_as_one_stream() {
let tokenizer = Arc::new(tokenizer());
let think_start_id = tokenizer.token_to_id("<think:opensource>").unwrap();
let mut parser = HyV3UnifiedParser::new(&tools(), tokenizer).unwrap();
parser.initialize(&[think_start_id]).unwrap();
let chunks = [
"reasoning</think:open",
"source>answer<tool_calls:opensource><tool_call:opensource>get_weather",
"<tool_sep:opensource><arg_key:opensource>city</arg_key:opensource>",
"<arg_value:opensource>Beijing</arg_value:opensource>",
"</tool_call:opensource></tool_calls:opensource>",
];
let mut output = UnifiedParserOutput::default();
for chunk in chunks {
parser.parse_into(chunk, &mut output).unwrap();
}
output.append(parser.finish().unwrap());
assert_eq!(
output.events,
vec![
UnifiedParserEvent::Reasoning("reasoning".to_string()),
UnifiedParserEvent::Text("answer".to_string()),
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: r#"{"city":"Beijing"}"#.to_string(),
}),
]
);
}
#[test]
fn structural_tag_uses_tokenizer_detected_suffix() {
let parser = HyV3UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
let structural_tools = [ToolParam::Function(FunctionToolParam::new(
FunctionDefinition::new("get_weather").with_parameters(json!({
"type": "object",
"properties": { "city": { "type": "string" } },
"required": ["city"]
})),
))];
let tag = build_structural_tag(
parser.structural_tag_builder().unwrap(),
&structural_tools,
ToolChoice::required(),
StructuralTagOptions::default().with_reasoning(false),
)
.unwrap()
.to_json_string()
.unwrap();
assert!(tag.contains("<tool_calls:opensource>"));
assert!(tag.contains("<arg_value:opensource>"));
assert!(!tag.contains("glm_xml"));
}
}
rust/src/parser/src/unified/mod.rs
+2
View File
@@ -5,11 +5,13 @@
mod combined;
mod gemma4;
mod hy_v3;
mod inkling;
mod kimi_k3;
pub use combined::CombinedParser;
pub use gemma4::Gemma4UnifiedParser;
pub use hy_v3::HyV3UnifiedParser;
pub use inkling::InklingUnifiedParser;
pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
use thiserror::Error;
rust/src/text/src/backend/hf/config.rs
+38 -1
View File
@@ -111,12 +111,32 @@ pub(super) struct GenerationConfig {
pub eos_token_id: Option<OneOrManyTokenIds>,
pub temperature: Option<f32>,
pub top_p: Option<f32>,
#[serde(deserialize_with = "deserialize_top_k")]
pub top_k: Option<u32>,
pub min_p: Option<f32>,
pub repetition_penalty: Option<f32>,
pub max_new_tokens: Option<u32>,
}
/// Deserialize vLLM-compatible `top_k` values from generation configs.
///
/// Both `-1` and `0` disable top-k sampling and become `None`; positive values
/// are preserved.
fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result<Option<u32>, D::Error>
where
D: serde::Deserializer<'de>,
{
match Option::<i64>::deserialize(deserializer)? {
None | Some(-1) | Some(0) => Ok(None),
Some(value) if value > 0 => {
u32::try_from(value).map(Some).map_err(serde::de::Error::custom)
}
Some(value) => Err(serde::de::Error::custom(format!(
"top_k must be -1, 0, or a positive integer, got {value}"
))),
}
}
/// HF generation configs allow either one EOS id or a list of EOS ids.
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
@@ -294,7 +314,24 @@ where
#[cfg(test)]
mod tests {
use super::ModelConfig;
use super::{GenerationConfig, ModelConfig};
#[test]
fn generation_config_normalizes_vllm_top_k_values() {
for (input, expected) in [
(r#"{"top_k":null}"#, None),
(r#"{"top_k":-1}"#, None),
(r#"{"top_k":0}"#, None),
(r#"{"top_k":20}"#, Some(20)),
] {
let config: GenerationConfig = serde_json::from_str(input).unwrap();
assert_eq!(config.top_k, expected, "input={input}");
}
for input in [r#"{"top_k":-2}"#, r#"{"top_k":4294967296}"#] {
assert!(serde_json::from_str::<GenerationConfig>(input).is_err());
}
}
#[test]
fn model_config_detects_moe_from_named_expert_fields() {
rust/src/tokenizer/src/hf.rs
+33
View File
@@ -149,10 +149,20 @@ fn encode_fastokens_ordinary(
pub struct HuggingFaceTokenizer {
backend: Backend,
special_token_ids: Arc<[u32]>,
added_vocab: Box<[(String, u32)]>,
}
impl HuggingFaceTokenizer {
fn from_hf_backend(tokenizer: HfTokenizer) -> Self {
let added_vocab = {
let mut vocab: Vec<_> = tokenizer
.get_added_tokens_decoder()
.iter()
.map(|(&id, token)| (token.content.clone(), id))
.collect();
vocab.sort_unstable_by_key(|(_, id)| *id);
vocab.into_boxed_slice()
};
let special_token_ids = {
let mut ids: Vec<u32> = tokenizer
.get_added_tokens_decoder()
@@ -167,10 +177,21 @@ impl HuggingFaceTokenizer {
Self {
backend: Backend::Hf(Box::new(tokenizer)),
special_token_ids,
added_vocab,
}
}
fn from_fastokens_backend(tokenizer: FastokensTokenizer) -> Self {
let added_vocab = {
let mut vocab: Vec<_> = tokenizer
.added_tokens()
.into_iter()
.flat_map(|added_tokens| added_tokens.iter())
.map(|token| (token.content.to_string(), token.id))
.collect();
vocab.sort_unstable_by_key(|(_, id)| *id);
vocab.into_boxed_slice()
};
let special_token_ids = {
let mut ids: Vec<u32> = tokenizer
.added_tokens()
@@ -192,6 +213,7 @@ impl HuggingFaceTokenizer {
Self {
backend,
special_token_ids,
added_vocab,
}
}
@@ -292,6 +314,10 @@ impl Tokenizer for HuggingFaceTokenizer {
}
}
fn added_vocab(&self) -> &[(String, u32)] {
&self.added_vocab
}
fn is_special_id(&self, token_id: u32) -> bool {
self.special_token_ids.binary_search(&token_id).is_ok()
}
@@ -477,6 +503,13 @@ mod tests {
tokenizer.encode(SPECIAL_TOKEN, false).unwrap(),
vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()]
);
assert_eq!(
tokenizer.added_vocab(),
vec![
(REGULAR_TOKEN.to_string(), 256),
(SPECIAL_TOKEN.to_string(), 257),
]
);
for text in [
"",
rust/src/tokenizer/src/lib.rs
+9
View File
@@ -39,6 +39,15 @@ pub trait Tokenizer: Send + Sync {
/// Convert one token ID into the tokenizer's raw token string.
fn id_to_token(&self, id: u32) -> Option<String>;
/// Borrow tokenizer added-vocabulary entries as `(token, id)` pairs.
///
/// Backends that do not expose the distinction between model vocabulary
/// and added vocabulary return an empty list.
// TODO: add support to all tokenizer backends
fn added_vocab(&self) -> &[(String, u32)] {
&[]
}
/// Return the vocabulary size. Backends that cannot report it fall back to
/// `usize::MAX`, an effectively unbounded value used only by test stubs.
fn vocab_size(&self) -> usize {
rust/src/tokenizer/src/test_utils.rs
+19 -1
View File
@@ -69,6 +69,7 @@ struct TestToken {
pub struct TestTokenizer {
token_to_id: BTreeMap<String, u32>,
id_to_token: BTreeMap<u32, TestToken>,
added_vocab: Vec<(String, u32)>,
unknown_decode: UnknownDecode,
vocab_size: Option<usize>,
bos_token_id: Option<u32>,
@@ -86,6 +87,7 @@ impl TestTokenizer {
Self {
token_to_id: BTreeMap::new(),
id_to_token: BTreeMap::new(),
added_vocab: Vec::new(),
unknown_decode: UnknownDecode::Error,
vocab_size: None,
bos_token_id: None,
@@ -154,9 +156,21 @@ impl TestTokenizer {
if self.token_to_id.insert(token.clone(), id).is_some() {
panic!("configured test token text {token:?} was registered more than once");
}
if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() {
if self
.id_to_token
.insert(
id,
TestToken {
text: token.clone(),
kind,
},
)
.is_some()
{
panic!("configured test token id {id} was registered more than once");
}
self.added_vocab.push((token, id));
self.added_vocab.sort_unstable_by_key(|(_, id)| *id);
}
fn byte_to_token(id: u32) -> Option<String> {
@@ -254,6 +268,10 @@ impl Tokenizer for TestTokenizer {
.or_else(|| Self::byte_to_token(id))
}
fn added_vocab(&self) -> &[(String, u32)] {
&self.added_vocab
}
fn vocab_size(&self) -> usize {
self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size())
}