From df3b3422b495a33fd10d0b2f06c051144d6bb2c2 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 21 Aug 2026 12:13:54 +1000 Subject: [PATCH] [Rust Frontend] Add HY3 unified parser and local XGrammar structural-tag builder (#53054) Signed-off-by: Bugen Zhao --- rust/src/chat/src/lib.rs | 2 +- rust/src/chat/src/parser/reasoning/mod.rs | 3 + rust/src/chat/src/parser/tool/mod.rs | 5 +- rust/src/chat/src/parser/unified.rs | 6 +- rust/src/chat/tests/roundtrip.rs | 67 ++++- rust/src/parser/src/reasoning/delimited.rs | 22 +- rust/src/parser/src/reasoning/hy_v3.rs | 51 ++++ rust/src/parser/src/reasoning/mod.rs | 2 + rust/src/parser/src/tool/hy_v3.rs | 204 +++++++++----- .../parser/src/tool/hy_v3/structural_tag.rs | 261 ++++++++++++++++++ rust/src/parser/src/tool/mod.rs | 2 +- rust/src/parser/src/unified/hy_v3.rs | 207 ++++++++++++++ rust/src/parser/src/unified/mod.rs | 2 + rust/src/text/src/backend/hf/config.rs | 39 ++- rust/src/tokenizer/src/hf.rs | 33 +++ rust/src/tokenizer/src/lib.rs | 9 + rust/src/tokenizer/src/test_utils.rs | 20 +- 17 files changed, 849 insertions(+), 86 deletions(-) create mode 100644 rust/src/parser/src/reasoning/hy_v3.rs create mode 100644 rust/src/parser/src/tool/hy_v3/structural_tag.rs create mode 100644 rust/src/parser/src/unified/hy_v3.rs diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index c6b91679e0..53caf31c3a 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -415,6 +415,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/parser/reasoning/mod.rs b/rust/src/chat/src/parser/reasoning/mod.rs index f85c4a930d..c8e1c2df39 100644 --- a/rust/src/chat/src/parser/reasoning/mod.rs +++ b/rust/src/chat/src/parser/reasoning/mod.rs @@ -25,6 +25,7 @@ pub mod names { pub const GEMMA4: &str = "gemma4"; pub const INKLING: &str = "inkling"; pub const GLM45: &str = "glm45"; + pub const HY_V3: &str = "hy_v3"; pub const KIMI: &str = "kimi"; pub const KIMI_K2: &str = "kimi_k2"; pub const KIMI_K3: &str = "kimi_k3"; @@ -67,6 +68,7 @@ impl ReasoningParserFactory { .register_unified_dummy(names::GEMMA4) .register_unified_dummy(names::INKLING) .register_parser::(names::GLM45) + .register_unified_dummy(names::HY_V3) .register_parser::(names::KIMI) .register_parser::(names::KIMI_K2) .register_unified_dummy(names::KIMI_K3) @@ -91,6 +93,7 @@ impl ReasoningParserFactory { .register_pattern("glm-4.7", names::GLM45) .register_pattern("glm-4.6", names::GLM45) .register_pattern("glm-4.5", names::GLM45) + .register_pattern("hy3", names::HY_V3) .register_pattern("kimi-k2", names::KIMI_K2) .register_pattern("kimi", names::KIMI) // step3p5 patterns must precede `step3`: substring matching would diff --git a/rust/src/chat/src/parser/tool/mod.rs b/rust/src/chat/src/parser/tool/mod.rs index cc8b160d93..580ede0019 100644 --- a/rust/src/chat/src/parser/tool/mod.rs +++ b/rust/src/chat/src/parser/tool/mod.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, LazyLock}; pub use vllm_parser::tool::{ DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser, - Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, HyV3ToolParser, + Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser, Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError, @@ -76,7 +76,7 @@ impl ToolParserFactory { .register_unified_dummy(names::INKLING) .register_parser::(names::GRANITE4) .register_parser::(names::HERMES) - .register_parser::(names::HY_V3) + .register_unified_dummy(names::HY_V3) .register_parser::(names::INTERNLM) .register_parser::(names::KIMI_K2) .register_unified_dummy(names::KIMI_K3) @@ -100,7 +100,6 @@ impl ToolParserFactory { .register_pattern("qwen3", names::QWEN3_XML) .register_pattern("hermes", names::HERMES) .register_pattern("hy3", names::HY_V3) - .register_pattern("hy_v3", names::HY_V3) // Narrow to `internlm2` substring so it matches `internlm2-chat-7b` // and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1, // routes to Llama), `internlm3-*` (also Llama-architecture per diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index 6246733db2..ccd0fff637 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -6,7 +6,8 @@ use std::sync::LazyLock; pub use vllm_parser::unified::{ - Gemma4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, UnifiedParser, + Gemma4UnifiedParser, HyV3UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, + UnifiedParser, }; use vllm_tokenizer::DynTokenizer; @@ -16,6 +17,7 @@ use crate::request::ChatTool; /// Canonical public names for registered unified parsers. pub mod names { pub const GEMMA4: &str = "gemma4"; + pub const HY_V3: &str = "hy_v3"; pub const INKLING: &str = "inkling"; pub const KIMI_K3: &str = "kimi_k3"; } @@ -41,12 +43,14 @@ impl UnifiedParserFactory { let mut factory = Self::default(); factory.register_parser::(names::GEMMA4); + factory.register_parser::(names::HY_V3); factory.register_parser::(names::INKLING); factory.register_parser::(names::KIMI_K3); factory .register_pattern("gemma-4", names::GEMMA4) .register_pattern("gemma4", names::GEMMA4) + .register_pattern("hy3", names::HY_V3) .register_pattern("inkling", names::INKLING) .register_pattern("kimi-k3", names::KIMI_K3) .register_pattern("kimi_k3", names::KIMI_K3); diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 049dc6a403..e9bc14a5e4 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -55,6 +55,12 @@ enum ThinkingBehavior { Toggleable { default: bool }, /// The chat template always behaves as `value` for this fixture. Always { value: bool }, + /// The chat template selects thinking mode through `reasoning_effort`. + ReasoningEffort { + default: bool, + enabled: &'static str, + disabled: &'static str, + }, } impl ThinkingBehavior { @@ -62,6 +68,7 @@ impl ThinkingBehavior { match self { Self::Toggleable { default } => default, Self::Always { value } => value, + Self::ReasoningEffort { default, .. } => default, } } @@ -76,6 +83,33 @@ impl ThinkingBehavior { Some(value), // explicitly request the supported thinking behavior None, // use default template behavior ], + Self::ReasoningEffort { .. } => vec![ + Some(true), // explicitly enable thinking + Some(false), // explicitly disable thinking + None, // use default template behavior + ], + } + } + + fn apply(self, request: &mut ChatRequest, thinking: Option) { + let Some(thinking) = thinking else { + return; + }; + + match self { + Self::Toggleable { .. } | Self::Always { .. } => { + for key in ["thinking", "enable_thinking"] { + request.chat_options.template_kwargs.insert(key.to_string(), thinking.into()); + } + } + Self::ReasoningEffort { + enabled, disabled, .. + } => { + request.chat_options.template_kwargs.insert( + "reasoning_effort".to_string(), + if thinking { enabled } else { disabled }.into(), + ); + } } } } @@ -242,6 +276,23 @@ impl RoundtripCase { } } + /// HY3 suffixed reasoning/tool markers discovered from tokenizer added vocab. + fn hy_v3() -> Self { + Self { + model_id: "tencent/Hy3", + assistant_stop_suffix: "<｜hy_eos:opensource｜>", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + thinking_behavior: ThinkingBehavior::ReasoningEffort { + default: false, + enabled: "high", + disabled: "no_think", + }, + json_fmt: compact_json_fmt(), + sort_json_keys: false, + } + } + /// SeedOSS with `` / `` reasoning tags. fn seed_oss() -> Self { Self { @@ -345,6 +396,9 @@ roundtrip_tests! { kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history // K3 drops plain-assistant reasoning in history; tool-call turns keep it. kimi_k3 => [tool_call_mix], + // HY3's final plain-assistant history omits EOS; tool-call history keeps it + // and exercises both stages of the tokenizer-derived unified parser. + hy_v3 => [tool_call_mix], gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call inkling => [reasoning_and_content, tool_call_mix], } @@ -370,6 +424,7 @@ async fn run_roundtrip_reasoning_and_content_inner( vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")], Vec::new(), thinking, + case.thinking_behavior, ); let expected_reasoning = "Need compute 2 + 2 directly."; let expected_text = "The answer is 4."; @@ -417,6 +472,7 @@ async fn run_roundtrip_tool_call_mix( )], test_tools(), Some(true), // always enable thinking in this fixture + case.thinking_behavior, ); let expected_reasoning = "Need call the weather and add tools."; let expected_text = "I will call the tools."; @@ -838,6 +894,7 @@ fn roundtrip_request( messages: Vec, tools: Vec, thinking: Option, + thinking_behavior: ThinkingBehavior, ) -> ChatRequest { let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true) .expect("tool context should resolve"); @@ -848,13 +905,9 @@ fn roundtrip_request( ..ChatRequest::for_test() }; - // Explicitly enable or disable thinking so that rendering and parsing the reasoning block is - // exercised or skipped in the roundtrip. If unspecified, use the default template behavior. - if let Some(thinking) = thinking { - for key in ["thinking", "enable_thinking"] { - request.chat_options.template_kwargs.insert(key.to_string(), thinking.into()); - } - } + // Explicitly enable or disable thinking using the controls understood by + // this model's template. If unspecified, use the template default. + thinking_behavior.apply(&mut request, thinking); request } diff --git a/rust/src/parser/src/reasoning/delimited.rs b/rust/src/parser/src/reasoning/delimited.rs index 51cb4eaf91..78213d291a 100644 --- a/rust/src/parser/src/reasoning/delimited.rs +++ b/rust/src/parser/src/reasoning/delimited.rs @@ -35,25 +35,29 @@ impl DelimitedReasoningParser { /// start or end delimiter, that prompt boundary always wins. pub(crate) fn new( tokenizer: DynTokenizer, - start_token: &'static str, - end_token: &'static str, + start_token: impl Into, + end_token: impl Into, default_in_reasoning: bool, ) -> Result { + let start_token = start_token.into(); + let end_token = end_token.into(); let start_token_id = - tokenizer.token_to_id(start_token).ok_or_else(|| ReasoningError::MissingToken { - token: start_token.to_string(), - })?; + tokenizer + .token_to_id(&start_token) + .ok_or_else(|| ReasoningError::MissingToken { + token: start_token.clone(), + })?; let end_token_id = - tokenizer.token_to_id(end_token).ok_or_else(|| ReasoningError::MissingToken { - token: end_token.to_string(), + tokenizer.token_to_id(&end_token).ok_or_else(|| ReasoningError::MissingToken { + token: end_token.clone(), })?; Ok(Self { tokenizer, current_in_reasoning: default_in_reasoning, buffer: String::new(), - start_token: start_token.to_string(), - end_token: end_token.to_string(), + start_token, + end_token, start_token_id, end_token_id, default_in_reasoning, diff --git a/rust/src/parser/src/reasoning/hy_v3.rs b/rust/src/parser/src/reasoning/hy_v3.rs new file mode 100644 index 0000000000..f25f4a22e0 --- /dev/null +++ b/rust/src/parser/src/reasoning/hy_v3.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use vllm_tokenizer::DynTokenizer; + +use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result}; + +/// Internal HY3 reasoning stage used by the unified HY3 parser. +pub(crate) struct HyV3ReasoningParser { + inner: DelimitedReasoningParser, +} + +impl HyV3ReasoningParser { + /// Create a HY3 reasoning parser for the tokenizer-specific marker suffix. + pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result { + Ok(Self { + inner: DelimitedReasoningParser::new( + tokenizer, + format!(""), + format!(""), + false, + )?, + }) + } +} + +impl ReasoningParser for HyV3ReasoningParser { + // Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and + // tool delimiters always use the same tokenizer-derived value. + fn create(_tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Err(ReasoningError::DummyUnifiedParser { + name: "hy_v3".to_string(), + }) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.inner.initialize(prompt_token_ids); + Ok(()) + } + + fn push(&mut self, delta: &str) -> Result { + Ok(self.inner.push(delta)) + } + + fn finish(&mut self) -> Result { + Ok(self.inner.finish()) + } +} diff --git a/rust/src/parser/src/reasoning/mod.rs b/rust/src/parser/src/reasoning/mod.rs index e8dda6631a..7588fe8f58 100644 --- a/rust/src/parser/src/reasoning/mod.rs +++ b/rust/src/parser/src/reasoning/mod.rs @@ -20,6 +20,7 @@ mod cohere_cmd; mod deepseek_r1; mod delimited; +mod hy_v3; mod kimi; mod minimax_m3; mod qwen3; @@ -32,6 +33,7 @@ use vllm_tokenizer::DynTokenizer; pub use self::cohere_cmd::CohereCmdReasoningParser; pub use self::deepseek_r1::DeepSeekR1ReasoningParser; pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary}; +pub(crate) use self::hy_v3::HyV3ReasoningParser; pub use self::kimi::KimiReasoningParser; pub use self::minimax_m3::MiniMaxM3ReasoningParser; pub use self::qwen3::Qwen3ReasoningParser; diff --git a/rust/src/parser/src/tool/hy_v3.rs b/rust/src/parser/src/tool/hy_v3.rs index a1426d6a3b..e289ab66ff 100644 --- a/rust/src/parser/src/tool/hy_v3.rs +++ b/rust/src/parser/src/tool/hy_v3.rs @@ -1,26 +1,65 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +mod structural_tag; + +use std::sync::Arc; + use winnow::ascii::multispace0 as ws0; use winnow::combinator::{alt, delimited, eof, repeat, seq, terminated}; use winnow::prelude::*; use winnow::stream::Partial; use winnow::token::{literal, rest, take_until}; +use self::structural_tag::HyV3StructuralTagBuilder; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; -use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; +use super::{Result, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput}; use crate::tool::{StructuralTagBuilder, Tool}; -const TOOL_CALLS_START: &str = ""; -const TOOL_CALLS_END: &str = ""; -const TOOL_CALL_START: &str = ""; -const TOOL_CALL_END: &str = ""; -const TOOL_SEP: &str = ""; -const ARG_KEY_START: &str = ""; -const ARG_KEY_END: &str = ""; -const ARG_VALUE_START: &str = ""; -const ARG_VALUE_END: &str = ""; +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HyV3ToolMarkers { + tool_calls_start: String, + tool_calls_end: String, + tool_call_start: String, + tool_call_end: String, + tool_sep: String, + arg_key_start: String, + arg_key_end: String, + arg_value_start: String, + arg_value_end: String, +} + +impl HyV3ToolMarkers { + pub(crate) fn new(suffix: &str) -> Self { + Self { + tool_calls_start: format!(""), + tool_calls_end: format!(""), + tool_call_start: format!(""), + tool_call_end: format!(""), + tool_sep: format!(""), + arg_key_start: format!(""), + arg_key_end: format!(""), + arg_value_start: format!(""), + arg_value_end: format!(""), + } + } + + pub(crate) fn iter(&self) -> impl Iterator { + [ + self.tool_calls_start.as_str(), + self.tool_calls_end.as_str(), + self.tool_call_start.as_str(), + self.tool_call_end.as_str(), + self.tool_sep.as_str(), + self.arg_key_start.as_str(), + self.arg_key_end.as_str(), + self.arg_value_start.as_str(), + self.arg_value_end.as_str(), + ] + .into_iter() + } +} type HyV3Input<'i> = Partial<&'i str>; @@ -60,21 +99,26 @@ enum HyV3Event { /// Arguments are emitted only after a full `` block is parsed. /// HY3 marker tokens are added-vocabulary tokens rather than tokenizer special /// tokens, so the default `preserve_special_tokens() == false` is sufficient. -pub struct HyV3ToolParser { +pub(crate) struct HyV3ToolParser { buffer: String, mode: HyV3Mode, emitted_tool_count: usize, tool_parameters: ToolSchemas, + markers: Arc, + structural_tag_builder: HyV3StructuralTagBuilder, } impl HyV3ToolParser { /// Create a HY3 tool parser. - fn new(tools: &[Tool]) -> Self { + pub(crate) fn new(tools: &[Tool], suffix: &str) -> Self { + let markers = Arc::new(HyV3ToolMarkers::new(suffix)); Self { buffer: String::new(), mode: HyV3Mode::Text, emitted_tool_count: 0, tool_parameters: ToolSchemas::from_tools(tools), + markers: Arc::clone(&markers), + structural_tag_builder: HyV3StructuralTagBuilder::new(markers), } } @@ -109,22 +153,26 @@ impl HyV3ToolParser { } impl ToolParser for HyV3ToolParser { - fn create(tools: &[Tool]) -> Result> + // Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and + // tool delimiters always use the same tokenizer-derived value. + fn create(_tools: &[Tool]) -> Result> where Self: Sized + 'static, { - Ok(Box::new(Self::new(tools))) + Err(ToolParserError::DummyUnifiedParser { + name: "hy_v3".to_string(), + }) } fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { - Some(xgrammar_structural_tag::Model::HyV3.builder()) + Some(&self.structural_tag_builder) } fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| { - parse_next_hy_v3_event(input, &mut self.mode) + parse_next_hy_v3_event(input, &mut self.mode, &self.markers) })? { self.apply_event(event, output)?; self.buffer.drain(..consumed_len); @@ -155,62 +203,83 @@ impl ToolParser for HyV3ToolParser { fn parse_next_hy_v3_event( input: &mut HyV3Input<'_>, mode: &mut HyV3Mode, + markers: &HyV3ToolMarkers, ) -> ModalResult { match mode { - HyV3Mode::Text => parse_text_event(input), + HyV3Mode::Text => parse_text_event(input, markers), HyV3Mode::ToolBlock { tool_call_end_scan } => { - parse_tool_block_event(input, tool_call_end_scan) + parse_tool_block_event(input, tool_call_end_scan, markers) } HyV3Mode::Done => ignored_rest_event(input), } } /// Parse a text-mode HY3 event. -fn parse_text_event(input: &mut HyV3Input<'_>) -> ModalResult { - alt((tool_block_start_event, safe_text_event)).parse_next(input) +fn parse_text_event( + input: &mut HyV3Input<'_>, + markers: &HyV3ToolMarkers, +) -> ModalResult { + alt(( + |input: &mut HyV3Input<'_>| tool_block_start_event(input, markers), + |input: &mut HyV3Input<'_>| safe_text_event(input, markers), + )) + .parse_next(input) } /// Parse a HY3 tool-block start marker. -fn tool_block_start_event(input: &mut HyV3Input<'_>) -> ModalResult { - literal(TOOL_CALLS_START).value(HyV3Event::ToolBlockStart).parse_next(input) +fn tool_block_start_event( + input: &mut HyV3Input<'_>, + markers: &HyV3ToolMarkers, +) -> ModalResult { + literal(markers.tool_calls_start.as_str()) + .value(HyV3Event::ToolBlockStart) + .parse_next(input) } /// Parse a safe text run before the next HY3 marker. -fn safe_text_event(input: &mut HyV3Input<'_>) -> ModalResult { - safe_text_len(input, TOOL_CALLS_START).map(|len| HyV3Event::Text { len }) +fn safe_text_event(input: &mut HyV3Input<'_>, markers: &HyV3ToolMarkers) -> ModalResult { + safe_text_len(input, &markers.tool_calls_start).map(|len| HyV3Event::Text { len }) } /// Parse one event inside a HY3 tool block. fn parse_tool_block_event( input: &mut HyV3Input<'_>, tool_call_end_scan: &mut MarkerScanState, + markers: &HyV3ToolMarkers, ) -> ModalResult { - alt((tool_block_end_event, |input: &mut HyV3Input<'_>| { - tool_call_event(input, tool_call_end_scan) - })) + alt(( + |input: &mut HyV3Input<'_>| tool_block_end_event(input, markers), + |input: &mut HyV3Input<'_>| tool_call_event(input, tool_call_end_scan, markers), + )) .parse_next(input) } /// Parse a HY3 tool-block end marker. -fn tool_block_end_event(input: &mut HyV3Input<'_>) -> ModalResult { - (ws0, literal(TOOL_CALLS_END)).value(HyV3Event::ToolBlockEnd).parse_next(input) +fn tool_block_end_event( + input: &mut HyV3Input<'_>, + markers: &HyV3ToolMarkers, +) -> ModalResult { + (ws0, literal(markers.tool_calls_end.as_str())) + .value(HyV3Event::ToolBlockEnd) + .parse_next(input) } /// Parse a complete HY3 tool-call block. fn tool_call_event( input: &mut HyV3Input<'_>, tool_call_end_scan: &mut MarkerScanState, + markers: &HyV3ToolMarkers, ) -> ModalResult { let (name, body) = seq!( _: ws0, - _: literal(TOOL_CALL_START), - take_until(0.., TOOL_SEP), - _: literal(TOOL_SEP), - take_until_marker(TOOL_CALL_END, tool_call_end_scan), - _: literal(TOOL_CALL_END), + _: literal(markers.tool_call_start.as_str()), + take_until(0.., markers.tool_sep.as_str()), + _: literal(markers.tool_sep.as_str()), + take_until_marker(markers.tool_call_end.as_str(), tool_call_end_scan), + _: literal(markers.tool_call_end.as_str()), ) .parse_next(input)?; - let raw_params = parse_tool_call_params(body)?; + let raw_params = parse_tool_call_params(body, markers)?; Ok(HyV3Event::ToolCall { name: name.trim().to_string(), @@ -219,21 +288,32 @@ fn tool_call_event( } /// Parse all parameter blocks inside a complete HY3 tool call. -fn parse_tool_call_params(tool_call_body: &str) -> ModalResult> { +fn parse_tool_call_params( + tool_call_body: &str, + markers: &HyV3ToolMarkers, +) -> ModalResult> { let mut input = tool_call_body; - delimited(ws0, repeat(0.., terminated(parameter, ws0)), eof).parse_next(&mut input) + delimited( + ws0, + repeat( + 0.., + terminated(|input: &mut &str| parameter(input, markers), ws0), + ), + eof, + ) + .parse_next(&mut input) } /// Parse a HY3 argument key/value block. -fn parameter(input: &mut &str) -> ModalResult<(String, String)> { +fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String, String)> { let (name, value) = seq!( - _: literal(ARG_KEY_START), - take_until(0.., ARG_KEY_END), - _: literal(ARG_KEY_END), + _: literal(markers.arg_key_start.as_str()), + take_until(0.., markers.arg_key_end.as_str()), + _: literal(markers.arg_key_end.as_str()), _: ws0, - _: literal(ARG_VALUE_START), - take_until(0.., ARG_VALUE_END), - _: literal(ARG_VALUE_END), + _: literal(markers.arg_value_start.as_str()), + take_until(0.., markers.arg_value_end.as_str()), + _: literal(markers.arg_value_end.as_str()), ) .parse_next(input)?; @@ -274,14 +354,14 @@ mod tests { #[test] fn hy_v3_does_not_preserve_special_tokens() { - let parser = HyV3ToolParser::new(&test_tools()); + let parser = HyV3ToolParser::new(&test_tools(), ""); assert!(!parser.preserve_special_tokens()); } #[test] fn hy_v3_parse_complete_without_tool_call_keeps_text() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser.parse_complete("This is a plain response.").unwrap(); assert_eq!(output.normal_text(), "This is a plain response."); @@ -290,7 +370,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_zero_arg_inline_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser .parse_complete( "get_current_date", @@ -305,7 +385,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_zero_arg_newline_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser .parse_complete( "\nget_current_date\n\n", @@ -318,7 +398,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_arguments_on_same_line() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser .parse_complete( "get_weathercityBeijingdate2026-03-30", @@ -333,7 +413,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_arguments_with_newlines() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser .parse_complete(&build_tool_calls(&[build_tool_call( "get_weather", @@ -349,7 +429,7 @@ mod tests { #[test] fn hy_v3_parse_complete_preserves_prefix_and_ignores_trailing_text() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser .parse_complete(&format!( "Checking.{} trailing text", @@ -363,7 +443,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_multiple_tool_calls_in_one_block() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser .parse_complete(&build_tool_calls(&[ build_tool_call( @@ -406,7 +486,7 @@ mod tests { #[test] fn hy_v3_parse_complete_converts_schema_types() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = parser .parse_complete(&build_tool_calls(&[build_tool_call( "convert", @@ -432,7 +512,7 @@ mod tests { #[test] fn hy_v3_streaming_without_tool_call_emits_text_incrementally() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let mut output = ToolParserOutput::default(); output.append(parser.parse_chunk("This is ").unwrap()); @@ -446,7 +526,7 @@ mod tests { #[test] fn hy_v3_streaming_extracts_zero_arg_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let chunks = [ "", "\n", @@ -465,7 +545,7 @@ mod tests { #[test] fn hy_v3_streaming_extracts_arguments() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let chunks = [ "", "\n", @@ -491,7 +571,7 @@ mod tests { #[test] fn hy_v3_streaming_preserves_prefix_text() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let chunks = [ "Checking.", "", @@ -521,7 +601,7 @@ mod tests { ), ]); let chunks = split_by_chars(&input, 9); - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = collect_stream(&mut parser, &chunks); @@ -537,7 +617,7 @@ mod tests { build_tool_calls(&[build_tool_call("get_weather", &[("city", "Beijing")])]) ); let chunks = split_by_chars(&input, 5); - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let output = collect_stream(&mut parser, &chunks); @@ -548,7 +628,7 @@ mod tests { #[test] fn hy_v3_streaming_does_not_emit_incomplete_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let mut output = ToolParserOutput::default(); parser @@ -564,7 +644,7 @@ mod tests { #[test] fn hy_v3_finish_fails_incomplete_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); parser.parse_chunk("get_weather").unwrap(); let error = parser.finish().unwrap_err(); @@ -575,7 +655,7 @@ mod tests { #[test] fn hy_v3_malformed_tool_call_fails_fast() { - let mut parser = HyV3ToolParser::new(&test_tools()); + let mut parser = HyV3ToolParser::new(&test_tools(), ""); let error = parser .parse_complete( "get_weathercityBeijing", diff --git a/rust/src/parser/src/tool/hy_v3/structural_tag.rs b/rust/src/parser/src/tool/hy_v3/structural_tag.rs new file mode 100644 index 0000000000..ef182da267 --- /dev/null +++ b/rust/src/parser/src/tool/hy_v3/structural_tag.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Structural-tag grammar for HY3 XML-style tool calls. + +use std::collections::HashSet; +use std::sync::Arc; + +use serde_json::Value; +use xgrammar_structural_tag::Result; +use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagContext}; +use xgrammar_structural_tag::format::{Format, StructuralTag, TagFormat}; +use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam}; + +use super::HyV3ToolMarkers; + +/// HY3 structural-tag builder using tokenizer-specific structural markers. +#[derive(Debug, Clone)] +pub(super) struct HyV3StructuralTagBuilder { + markers: Arc, +} + +impl HyV3StructuralTagBuilder { + pub(super) fn new(markers: Arc) -> Self { + Self { markers } + } + + fn argument_pair(&self, key: &str) -> Format { + let excludes = self.markers.iter().collect::>(); + Format::sequence(vec![ + Format::const_string(&self.markers.arg_key_start), + Format::const_string(key), + Format::const_string(&self.markers.arg_key_end), + Format::const_string("\n"), + Format::const_string(&self.markers.arg_value_start), + Format::any_text_excluding(&excludes), + Format::const_string(&self.markers.arg_value_end), + Format::const_string("\n"), + ]) + } + + fn tool_call(&self, tool: &FunctionToolParam) -> TagFormat { + let (required_keys, optional_keys) = argument_keys(tool.function.parameters.as_ref()); + let mut elements = + required_keys.into_iter().map(|key| self.argument_pair(key)).collect::>(); + + if !optional_keys.is_empty() { + let mut pairs = + optional_keys.into_iter().map(|key| self.argument_pair(key)).collect::>(); + let optional = if pairs.len() == 1 { + pairs.pop().unwrap() + } else { + Format::or(pairs) + }; + elements.push(Format::star(optional)); + } + + let content = if elements.is_empty() { + Format::any_text() + } else { + Format::sequence(elements) + }; + TagFormat::new( + format!( + "{}{}{}\n", + self.markers.tool_call_start, tool.function.name, self.markers.tool_sep + ), + content, + self.markers.tool_call_end.clone(), + ) + } + + fn tool_calls(&self, tools: &[FunctionToolParam], choice: BuilderToolChoice) -> Format { + let mut calls = tools.iter().map(|tool| self.tool_call(tool)).collect::>(); + let begin = format!("{}\n", self.markers.tool_calls_start); + let end = format!("\n{}", self.markers.tool_calls_end); + + match choice { + BuilderToolChoice::Auto if calls.is_empty() => Format::any_text(), + BuilderToolChoice::Auto => { + let outer = TagFormat::new( + begin, + Format::tags_with_separator(calls, "\n", true, false), + end, + ); + Format::triggered_tags(&[&self.markers.tool_calls_start], vec![outer]) + } + BuilderToolChoice::Forced => Format::sequence(vec![ + Format::const_string(begin), + Format::Tag(calls.pop().unwrap()), + Format::const_string(end), + ]), + BuilderToolChoice::Required => Format::sequence(vec![ + Format::const_string(begin), + Format::tags_with_separator(calls, "\n", true, false), + Format::const_string(end), + ]), + } + } +} + +impl StructuralTagBuilder for HyV3StructuralTagBuilder { + fn build(&self, ctx: StructuralTagContext<'_>) -> Result { + Ok(StructuralTag::new( + self.tool_calls(ctx.function_tools, ctx.tool_choice), + )) + } +} + +/// Split argument keys into required and optional declaration-order groups. +fn argument_keys(parameters: Option<&Value>) -> (Vec<&str>, Vec<&str>) { + let Some(parameters) = parameters.and_then(Value::as_object) else { + return (Vec::new(), Vec::new()); + }; + let properties = parameters.get("properties").and_then(Value::as_object); + let required = parameters + .get("required") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect::>(); + let required_set = required.iter().copied().collect::>(); + + let mut required_keys = properties + .into_iter() + .flat_map(|properties| properties.keys()) + .map(String::as_str) + .filter(|key| required_set.contains(key)) + .collect::>(); + required_keys.extend( + required + .into_iter() + .filter(|key| properties.is_none_or(|properties| !properties.contains_key(*key))), + ); + let optional_keys = properties + .into_iter() + .flat_map(|properties| properties.keys()) + .map(String::as_str) + .filter(|key| !required_set.contains(key)) + .collect(); + (required_keys, optional_keys) +} + +#[cfg(test)] +mod tests { + use expect_test::expect; + use serde_json::{Value, json}; + use std::sync::Arc; + use xgrammar_structural_tag::builders::StructuralTagOptions; + use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, + }; + + use super::HyV3StructuralTagBuilder; + use crate::tool::HyV3ToolMarkers; + + fn tool(name: &str, parameters: Value) -> ToolParam { + ToolParam::Function(FunctionToolParam::new( + FunctionDefinition::new(name).with_parameters(parameters), + )) + } + + fn build( + suffix: &str, + tools: &[ToolParam], + choice: ToolChoice, + ) -> xgrammar_structural_tag::format::StructuralTag { + build_structural_tag( + HyV3StructuralTagBuilder::new(Arc::new(HyV3ToolMarkers::new(suffix))), + tools, + choice, + StructuralTagOptions::default().with_reasoning(false), + ) + .unwrap() + } + + #[test] + fn required_uses_suffixed_hy3_skeleton_and_bounded_values() { + let tag = build( + ":opensource", + &[tool( + "get_weather", + json!({ + "type": "object", + "properties": { + "city": { "type": "string" }, + "days": { "type": "integer" } + }, + "required": ["city"] + }), + )], + ToolChoice::required(), + ); + + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + } + + #[test] + fn optional_only_schema_keeps_declared_key_alternatives() { + let tag = build( + "", + &[tool( + "lookup", + json!({ + "type": "object", + "properties": { + "query": { "type": "string" }, + "limit": { "type": "integer" } + } + }), + )], + ToolChoice::required(), + ); + let value = serde_json::to_value(tag).unwrap(); + + expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( + &serde_json::to_string( + &value["format"]["elements"][1]["tags"][0]["content"], + ) + .unwrap(), + ); + } + + #[test] + fn auto_and_forced_preserve_tool_choice_shape() { + let tools = [ + tool("search", json!({ "type": "object" })), + tool("lookup", json!({ "type": "object" })), + ]; + let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap(); + let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap(); + + expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"begin":"\n","content":{"type":"tags_with_separator","tags":[{"begin":"search\n","content":{"type":"any_text","excludes":[]},"end":""},{"begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); + } + + #[test] + fn missing_required_property_is_still_emitted() { + let tag = build( + "", + &[tool( + "search", + json!({ + "type": "object", + "properties": { "query": { "type": "string" } }, + "required": ["query", "tenant"] + }), + )], + ToolChoice::required(), + ); + let value = serde_json::to_value(tag).unwrap(); + + expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( + &serde_json::to_string( + &value["format"]["elements"][1]["tags"][0]["content"], + ) + .unwrap(), + ); + } +} diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index e25eae882f..940072bdac 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -24,7 +24,7 @@ pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser}; pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser}; pub use error::{Result, ToolParserError}; pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser}; -pub use hy_v3::HyV3ToolParser; +pub(crate) use hy_v3::{HyV3ToolMarkers, HyV3ToolParser}; pub use json::{ Granite4ToolParser, HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3XmlToolParser, diff --git a/rust/src/parser/src/unified/hy_v3.rs b/rust/src/parser/src/unified/hy_v3.rs new file mode 100644 index 0000000000..ede3b51dd2 --- /dev/null +++ b/rust/src/parser/src/unified/hy_v3.rs @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use vllm_tokenizer::{DynTokenizer, Tokenizer}; + +use super::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id}; +use crate::reasoning::HyV3ReasoningParser; +use crate::tool::{HyV3ToolMarkers, HyV3ToolParser, StructuralTagBuilder, Tool}; + +const HY_V3_MARKER_STEMS: &[&str] = &[ + "think", + "tool_calls", + "tool_call", + "tool_sep", + "arg_key", + "arg_value", +]; + +/// Unified reasoning and tool parser for HY3 output. +pub struct HyV3UnifiedParser { + inner: CombinedParser, +} + +impl HyV3UnifiedParser { + /// Create a HY3 parser using the suffix encoded in tokenizer added tokens. + pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result { + let suffix = detect_token_suffix(tokenizer.as_ref()); + let markers = HyV3ToolMarkers::new(&suffix); + for marker in markers.iter() { + token_id(tokenizer.as_ref(), marker)?; + } + + let reasoning = HyV3ReasoningParser::new(tokenizer, &suffix)?; + let tool = HyV3ToolParser::new(tools, &suffix); + Ok(Self { + inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))), + }) + } +} + +impl UnifiedParser for HyV3UnifiedParser { + fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.inner.initialize(prompt_token_ids) + } + + fn preserve_special_tokens(&self) -> bool { + self.inner.preserve_special_tokens() + } + + fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { + self.inner.structural_tag_builder() + } + + fn tool_call_id(&self, tool_index: usize) -> Option<&str> { + self.inner.tool_call_id(tool_index) + } + + fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> { + self.inner.parse_into(delta, output) + } + + fn finish(&mut self) -> Result { + self.inner.finish() + } + + fn reset(&mut self) -> String { + self.inner.reset() + } +} + +/// Detect the HY3 structural-token suffix from tokenizer added vocabulary. +fn detect_token_suffix(tokenizer: &dyn Tokenizer) -> String { + tokenizer + .added_vocab() + .iter() + .filter_map(|(token, id)| marker_suffix(token).map(|suffix| (*id, suffix))) + .min_by_key(|(id, _)| *id) + .map(|(_, suffix)| suffix.to_string()) + .unwrap_or_default() +} + +/// Extract the suffix from one opening or closing HY3 structural token. +fn marker_suffix(token: &str) -> Option<&str> { + let body = token.strip_prefix('<')?.strip_suffix('>')?; + let body = body.strip_prefix('/').unwrap_or(body); + + HY_V3_MARKER_STEMS.iter().find_map(|stem| { + let suffix = body.strip_prefix(stem)?; + (suffix.is_empty() || suffix.starts_with(':')).then_some(suffix) + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use serde_json::json; + use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer}; + use xgrammar_structural_tag::builders::StructuralTagOptions; + use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, + }; + + use super::{HyV3UnifiedParser, UnifiedParser}; + use crate::tool::Tool; + use crate::unified::{UnifiedParserEvent, UnifiedParserOutput}; + + fn tokenizer() -> TestTokenizer { + [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ] + .into_iter() + .enumerate() + .fold(TestTokenizer::new(), |tokenizer, (index, token)| { + tokenizer.with_regular_token(token, 1000 + index as u32) + }) + } + + fn tools() -> Vec { + vec![Tool { + name: "get_weather".to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { "city": { "type": "string" } }, + }), + strict: None, + }] + } + + #[test] + fn parses_suffixed_reasoning_and_tool_call_as_one_stream() { + let tokenizer = Arc::new(tokenizer()); + let think_start_id = tokenizer.token_to_id("").unwrap(); + let mut parser = HyV3UnifiedParser::new(&tools(), tokenizer).unwrap(); + parser.initialize(&[think_start_id]).unwrap(); + + let chunks = [ + "reasoninganswerget_weather", + "city", + "Beijing", + "", + ]; + let mut output = UnifiedParserOutput::default(); + for chunk in chunks { + parser.parse_into(chunk, &mut output).unwrap(); + } + output.append(parser.finish().unwrap()); + + assert_eq!( + output.events, + vec![ + UnifiedParserEvent::Reasoning("reasoning".to_string()), + UnifiedParserEvent::Text("answer".to_string()), + UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta { + tool_index: 0, + name: Some("get_weather".to_string()), + arguments: r#"{"city":"Beijing"}"#.to_string(), + }), + ] + ); + } + + #[test] + fn structural_tag_uses_tokenizer_detected_suffix() { + let parser = HyV3UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap(); + let structural_tools = [ToolParam::Function(FunctionToolParam::new( + FunctionDefinition::new("get_weather").with_parameters(json!({ + "type": "object", + "properties": { "city": { "type": "string" } }, + "required": ["city"] + })), + ))]; + + let tag = build_structural_tag( + parser.structural_tag_builder().unwrap(), + &structural_tools, + ToolChoice::required(), + StructuralTagOptions::default().with_reasoning(false), + ) + .unwrap() + .to_json_string() + .unwrap(); + + assert!(tag.contains("")); + assert!(tag.contains("")); + assert!(!tag.contains("glm_xml")); + } +} diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index ae5524759d..9deac1853f 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -5,11 +5,13 @@ mod combined; mod gemma4; +mod hy_v3; mod inkling; mod kimi_k3; pub use combined::CombinedParser; pub use gemma4::Gemma4UnifiedParser; +pub use hy_v3::HyV3UnifiedParser; pub use inkling::InklingUnifiedParser; pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser}; use thiserror::Error; diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 2a69632c87..7f393014b7 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -111,12 +111,32 @@ pub(super) struct GenerationConfig { pub eos_token_id: Option, pub temperature: Option, pub top_p: Option, + #[serde(deserialize_with = "deserialize_top_k")] pub top_k: Option, pub min_p: Option, pub repetition_penalty: Option, pub max_new_tokens: Option, } +/// Deserialize vLLM-compatible `top_k` values from generation configs. +/// +/// Both `-1` and `0` disable top-k sampling and become `None`; positive values +/// are preserved. +fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + match Option::::deserialize(deserializer)? { + None | Some(-1) | Some(0) => Ok(None), + Some(value) if value > 0 => { + u32::try_from(value).map(Some).map_err(serde::de::Error::custom) + } + Some(value) => Err(serde::de::Error::custom(format!( + "top_k must be -1, 0, or a positive integer, got {value}" + ))), + } +} + /// HF generation configs allow either one EOS id or a list of EOS ids. #[derive(Debug, Clone, Deserialize)] #[serde(untagged)] @@ -294,7 +314,24 @@ where #[cfg(test)] mod tests { - use super::ModelConfig; + use super::{GenerationConfig, ModelConfig}; + + #[test] + fn generation_config_normalizes_vllm_top_k_values() { + for (input, expected) in [ + (r#"{"top_k":null}"#, None), + (r#"{"top_k":-1}"#, None), + (r#"{"top_k":0}"#, None), + (r#"{"top_k":20}"#, Some(20)), + ] { + let config: GenerationConfig = serde_json::from_str(input).unwrap(); + assert_eq!(config.top_k, expected, "input={input}"); + } + + for input in [r#"{"top_k":-2}"#, r#"{"top_k":4294967296}"#] { + assert!(serde_json::from_str::(input).is_err()); + } + } #[test] fn model_config_detects_moe_from_named_expert_fields() { diff --git a/rust/src/tokenizer/src/hf.rs b/rust/src/tokenizer/src/hf.rs index eb527294a5..d936ac80c6 100644 --- a/rust/src/tokenizer/src/hf.rs +++ b/rust/src/tokenizer/src/hf.rs @@ -149,10 +149,20 @@ fn encode_fastokens_ordinary( pub struct HuggingFaceTokenizer { backend: Backend, special_token_ids: Arc<[u32]>, + added_vocab: Box<[(String, u32)]>, } impl HuggingFaceTokenizer { fn from_hf_backend(tokenizer: HfTokenizer) -> Self { + let added_vocab = { + let mut vocab: Vec<_> = tokenizer + .get_added_tokens_decoder() + .iter() + .map(|(&id, token)| (token.content.clone(), id)) + .collect(); + vocab.sort_unstable_by_key(|(_, id)| *id); + vocab.into_boxed_slice() + }; let special_token_ids = { let mut ids: Vec = tokenizer .get_added_tokens_decoder() @@ -167,10 +177,21 @@ impl HuggingFaceTokenizer { Self { backend: Backend::Hf(Box::new(tokenizer)), special_token_ids, + added_vocab, } } fn from_fastokens_backend(tokenizer: FastokensTokenizer) -> Self { + let added_vocab = { + let mut vocab: Vec<_> = tokenizer + .added_tokens() + .into_iter() + .flat_map(|added_tokens| added_tokens.iter()) + .map(|token| (token.content.to_string(), token.id)) + .collect(); + vocab.sort_unstable_by_key(|(_, id)| *id); + vocab.into_boxed_slice() + }; let special_token_ids = { let mut ids: Vec = tokenizer .added_tokens() @@ -192,6 +213,7 @@ impl HuggingFaceTokenizer { Self { backend, special_token_ids, + added_vocab, } } @@ -292,6 +314,10 @@ impl Tokenizer for HuggingFaceTokenizer { } } + fn added_vocab(&self) -> &[(String, u32)] { + &self.added_vocab + } + fn is_special_id(&self, token_id: u32) -> bool { self.special_token_ids.binary_search(&token_id).is_ok() } @@ -477,6 +503,13 @@ mod tests { tokenizer.encode(SPECIAL_TOKEN, false).unwrap(), vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()] ); + assert_eq!( + tokenizer.added_vocab(), + vec![ + (REGULAR_TOKEN.to_string(), 256), + (SPECIAL_TOKEN.to_string(), 257), + ] + ); for text in [ "", diff --git a/rust/src/tokenizer/src/lib.rs b/rust/src/tokenizer/src/lib.rs index 0f8c7dc16e..2bc1cfca3d 100644 --- a/rust/src/tokenizer/src/lib.rs +++ b/rust/src/tokenizer/src/lib.rs @@ -39,6 +39,15 @@ pub trait Tokenizer: Send + Sync { /// Convert one token ID into the tokenizer's raw token string. fn id_to_token(&self, id: u32) -> Option; + /// Borrow tokenizer added-vocabulary entries as `(token, id)` pairs. + /// + /// Backends that do not expose the distinction between model vocabulary + /// and added vocabulary return an empty list. + // TODO: add support to all tokenizer backends + fn added_vocab(&self) -> &[(String, u32)] { + &[] + } + /// Return the vocabulary size. Backends that cannot report it fall back to /// `usize::MAX`, an effectively unbounded value used only by test stubs. fn vocab_size(&self) -> usize { diff --git a/rust/src/tokenizer/src/test_utils.rs b/rust/src/tokenizer/src/test_utils.rs index 6fdb6e0272..924f70c98a 100644 --- a/rust/src/tokenizer/src/test_utils.rs +++ b/rust/src/tokenizer/src/test_utils.rs @@ -69,6 +69,7 @@ struct TestToken { pub struct TestTokenizer { token_to_id: BTreeMap, id_to_token: BTreeMap, + added_vocab: Vec<(String, u32)>, unknown_decode: UnknownDecode, vocab_size: Option, bos_token_id: Option, @@ -86,6 +87,7 @@ impl TestTokenizer { Self { token_to_id: BTreeMap::new(), id_to_token: BTreeMap::new(), + added_vocab: Vec::new(), unknown_decode: UnknownDecode::Error, vocab_size: None, bos_token_id: None, @@ -154,9 +156,21 @@ impl TestTokenizer { if self.token_to_id.insert(token.clone(), id).is_some() { panic!("configured test token text {token:?} was registered more than once"); } - if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() { + if self + .id_to_token + .insert( + id, + TestToken { + text: token.clone(), + kind, + }, + ) + .is_some() + { panic!("configured test token id {id} was registered more than once"); } + self.added_vocab.push((token, id)); + self.added_vocab.sort_unstable_by_key(|(_, id)| *id); } fn byte_to_token(id: u32) -> Option { @@ -254,6 +268,10 @@ impl Tokenizer for TestTokenizer { .or_else(|| Self::byte_to_token(id)) } + fn added_vocab(&self) -> &[(String, u32)] { + &self.added_vocab + } + fn vocab_size(&self) -> usize { self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size()) }