diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 6c8d85b54d..0669af8daf 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -233,7 +233,7 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, hermes, hy_v3, kimi_k2, llama3_json, llama4_json, minimax_m2, mistral, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string()); + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, mistral, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string()); } #[test] diff --git a/rust/src/chat/src/parser/tool/mod.rs b/rust/src/chat/src/parser/tool/mod.rs index 7dc7267229..ad220b5a78 100644 --- a/rust/src/chat/src/parser/tool/mod.rs +++ b/rust/src/chat/src/parser/tool/mod.rs @@ -5,9 +5,9 @@ use std::sync::LazyLock; pub use vllm_tool_parser::{ DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser, Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, HermesToolParser, HyV3ToolParser, - KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser, MistralToolParser, - Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError, - ToolParserOutput, + Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser, + MistralToolParser, Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, + ToolParserError, ToolParserOutput, }; use crate::parser::ParserFactory; @@ -24,6 +24,9 @@ pub mod names { pub const GEMMA4: &str = "gemma4"; pub const HERMES: &str = "hermes"; pub const HY_V3: &str = "hy_v3"; + // Matches the Python CLI name `--tool-call-parser internlm`, which Python + // also routes to `Internlm2ToolParser` despite the version-agnostic name. + pub const INTERNLM: &str = "internlm"; pub const KIMI_K2: &str = "kimi_k2"; pub const LLAMA3_JSON: &str = "llama3_json"; pub const LLAMA4_JSON: &str = "llama4_json"; @@ -62,6 +65,7 @@ impl ToolParserFactory { .register_parser::(names::GEMMA4) .register_parser::(names::HERMES) .register_parser::(names::HY_V3) + .register_parser::(names::INTERNLM) .register_parser::(names::KIMI_K2) .register_parser::(names::LLAMA3_JSON) .register_parser::(names::LLAMA4_JSON) @@ -80,6 +84,12 @@ impl ToolParserFactory { .register_pattern("hermes", names::HERMES) .register_pattern("hy3", names::HY_V3) .register_pattern("hy_v3", names::HY_V3) + // Narrow to `internlm2` substring so it matches `internlm2-chat-7b` + // and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1, + // routes to Llama), `internlm3-*` (also Llama-architecture per + // vllm/model_executor/models/registry.py:146), or `Intern-S1` / + // `Intern-S1-Pro` (separate intern-s1 parser, see PR #40115). + .register_pattern("internlm2", names::INTERNLM) .register_pattern("llama-4", names::LLAMA4_JSON) .register_pattern("llama-3.2", names::LLAMA3_JSON) .register_pattern("llama-3.1", names::LLAMA3_JSON) diff --git a/rust/src/chat/src/parser/tool/tests.rs b/rust/src/chat/src/parser/tool/tests.rs index f18aa69f95..65e9f4e075 100644 --- a/rust/src/chat/src/parser/tool/tests.rs +++ b/rust/src/chat/src/parser/tool/tests.rs @@ -161,4 +161,33 @@ fn factory_new_resolves_default_patterns() { factory.resolve_name_for_model("org/mm-m2-base"), Some(names::MINIMAX_M2) ); + + // InternLM2 positive: both dashed and underscored versioned names route. + assert_eq!( + factory.resolve_name_for_model("internlm/internlm2-chat-7b"), + Some(names::INTERNLM) + ); + assert_eq!( + factory.resolve_name_for_model("internlm/internlm2_5-7b-chat"), + Some(names::INTERNLM) + ); + + // Negative: other internlm-org models do NOT route to the InternLM2 parser, + // since they use unrelated prompt formats. + // - InternLM v1 (`internlm-chat-7b`) routes to Llama + // - InternLM3 (`internlm3-8b-instruct`) routes to Llama + // - Intern-S1 / Intern-S1-Pro have their own parser (Python PR #40115) + assert_eq!( + factory.resolve_name_for_model("internlm/internlm-chat-7b"), + None + ); + assert_eq!( + factory.resolve_name_for_model("internlm/internlm3-8b-instruct"), + None + ); + assert_eq!(factory.resolve_name_for_model("internlm/Intern-S1"), None); + assert_eq!( + factory.resolve_name_for_model("internlm/Intern-S1-Pro"), + None + ); } diff --git a/rust/src/tool-parser/src/json/hermes.rs b/rust/src/tool-parser/src/json/hermes.rs index c8e6a49a7c..f6b130ec47 100644 --- a/rust/src/tool-parser/src/json/hermes.rs +++ b/rust/src/tool-parser/src/json/hermes.rs @@ -8,7 +8,7 @@ const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig { marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: None, name_key: "name", - arguments_key: "arguments", + arguments_key: &["arguments"], }; /// Tool parser for Hermes XML-wrapped JSON tool calls. diff --git a/rust/src/tool-parser/src/json/internlm2.rs b/rust/src/tool-parser/src/json/internlm2.rs new file mode 100644 index 0000000000..5c3b024a93 --- /dev/null +++ b/rust/src/tool-parser/src/json/internlm2.rs @@ -0,0 +1,351 @@ +use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; +use crate::{Result, Tool, ToolParser, ToolParserOutput}; + +const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig { + parser_name: "InternLM2", + start_marker: "<|action_start|><|plugin|>", + end_marker: "<|action_end|>", + marker_whitespace: JsonToolCallWhitespace::Optional, + delimiter: None, + name_key: "name", + // The Python parser's `get_arguments()` accepts either `parameters` or + // `arguments` and prefers `parameters` when both are present. This Rust + // parser uses first-encountered semantics because the header parser only + // permits one args key per tool-call object; if a future model emits + // both keys in the same object, the Rust port will accept the first one + // and reject the trailing one as a syntax error rather than silently + // shadowing it. + arguments_key: &["parameters", "arguments"], +}; + +/// Tool parser for InternLM2 special-token wrapped JSON tool calls. +/// +/// Example tool call content: +/// +/// ```text +/// <|action_start|><|plugin|>{"name": "get_weather", "parameters": {"location":"Tokyo"}}<|action_end|> +/// ``` +/// +/// Arguments are already OpenAI-style JSON text, so they are streamed as raw +/// argument deltas without schema conversion or JSON normalization. +/// +/// # Divergences from the Python reference +/// +/// This Rust port intentionally diverges from +/// `vllm/tool_parsers/internlm2_tool_parser.py` in two user-visible ways: +/// +/// - **Parallel tool calls are supported.** Python silently drops every +/// `<|action_start|>` block after the first (`current_tool_id > 0` returns +/// an empty delta); this parser emits every well-formed block with +/// incrementing `tool_index`. Models that legitimately emit multiple action +/// blocks therefore produce more tool calls under Rust than under Python. +/// - **End-marker bytes inside JSON string values are preserved.** Python +/// does `action.split("<|action_end|>")[0]` which truncates regardless of +/// JSON context; this parser scans matched braces and quotes so a literal +/// `<|action_end|>` inside an arguments string is forwarded intact. +/// - **Only whitespace is allowed before the `{`.** Python's non-streaming +/// `action[action.find("{"):]` drops any bytes before the first `{`, but +/// its streaming path has no equivalent and the model format always emits +/// `<|plugin|>{...`; this parser allows only whitespace there, matching the +/// other JSON parsers in this crate. +/// - **Truncated tool calls error rather than silently dropping.** Python's +/// streaming wrapper swallows mid-stream errors with `except Exception: +/// return None` (logging a traceback) while its non-streaming path raises +/// `JSONDecodeError`; this parser returns an `incomplete InternLM2 tool +/// call` error from `finish()`, matching the other JSON parsers and Python's +/// non-streaming behavior. +/// +/// # Known unaddressed divergences (TODO) +/// +/// The following Python behaviors are NOT yet matched. They are deferred to +/// follow-up work because they require non-local changes to the shared +/// `JsonToolCallParser` core that would affect Hermes / Llama / Mistral / +/// Qwen as well. If a real-world InternLM2 deployment hits one of these, +/// prioritize the corresponding fix. +/// +/// - **Arguments value type.** The shared core requires the arguments value +/// to be a JSON object (`take_json_object` rejects anything not starting +/// with `{`). Python's `json.dumps(action_dict.get("parameters", ...))` +/// accepts `null`, arrays, strings, and numbers and round-trips them +/// verbatim. Models that legitimately emit `"parameters":null` will hard- +/// fail under Rust. +/// - **Unknown arguments key.** Python falls back to `{}` via +/// `action_dict.get("parameters", action_dict.get("arguments", {}))` when +/// neither key is present; the Rust header parser raises +/// `parsing failed: invalid InternLM2` for any unrecognized key. A model +/// that emits a typo (e.g. `"params"`) breaks the whole response. +/// - **Field order independence.** The header parser requires the JSON keys +/// to appear in the order `name` then arguments key. Python's +/// `json.loads` + `dict.get` is order-independent, so a model emitting +/// `{"parameters":{...},"name":"foo"}` parses in Python but fails in Rust. +pub struct Internlm2ToolParser { + inner: JsonToolCallParser, +} + +impl Internlm2ToolParser { + /// Create an InternLM2 tool parser. + fn new(_tools: &[Tool]) -> Self { + Self { + inner: JsonToolCallParser::new(INTERNLM2_CONFIG), + } + } +} + +impl ToolParser for Internlm2ToolParser { + /// Create a boxed InternLM2 tool parser. + fn create(tools: &[Tool]) -> Result> + where + Self: Sized + 'static, + { + Ok(Box::new(Self::new(tools))) + } + + /// Preserve special-token markers while decoding, since + /// `<|action_start|>`, `<|plugin|>`, and `<|action_end|>` are tokenizer + /// special tokens in InternLM2 models. + fn preserve_special_tokens(&self) -> bool { + true + } + + /// Feed one decoded text chunk through the InternLM2 parser. + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { + self.inner.parse_into(chunk, output) + } + + /// Flush any buffered partial state at end of stream. + fn finish(&mut self) -> Result { + self.inner.finish() + } + + /// Clear parser state and return currently uncommitted buffered text. + fn reset(&mut self) -> String { + self.inner.reset() + } +} + +#[cfg(test)] +mod tests { + use expect_test::expect; + use thiserror_ext::AsReport; + + use super::Internlm2ToolParser; + use crate::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + + const ACTION_START: &str = "<|action_start|><|plugin|>"; + const ACTION_END: &str = "<|action_end|>"; + + fn build_tool_call(function_name: &str, args_key: &str, arguments: &str) -> String { + format!( + r#"{ACTION_START}{{"name":"{function_name}","{args_key}":{arguments}}}{ACTION_END}"# + ) + } + + #[test] + fn internlm2_parse_complete_without_tool_call_keeps_text() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let result = parser.parse_complete("Hello, world!").unwrap(); + + assert_eq!(result.normal_text, "Hello, world!"); + assert!(result.calls.is_empty()); + } + + #[test] + fn internlm2_parse_complete_extracts_parameters_key() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let arguments = r#"{"location":"Tokyo","days":"3"}"#; + let result = parser + .parse_complete(&format!( + "Let me check.\n{}", + build_tool_call("get_weather", "parameters", arguments) + )) + .unwrap(); + + assert_eq!(result.normal_text, "Let me check.\n"); + assert_eq!(result.calls.len(), 1); + assert_eq!(result.calls[0].tool_index, 0); + assert_eq!(result.calls[0].name.as_deref(), Some("get_weather")); + assert_eq!(result.calls[0].arguments, arguments); + } + + #[test] + fn internlm2_parse_complete_extracts_arguments_key_fallback() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let arguments = r#"{"location":"Tokyo"}"#; + let result = parser + .parse_complete(&build_tool_call("get_weather", "arguments", arguments)) + .unwrap(); + + assert_eq!(result.calls.len(), 1); + assert_eq!(result.calls[0].name.as_deref(), Some("get_weather")); + assert_eq!(result.calls[0].arguments, arguments); + } + + #[test] + fn internlm2_accepts_whitespace_after_plugin_marker() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let result = parser + .parse_complete(&format!( + r#"{ACTION_START} +{{"name":"get_weather","parameters":{{}}}}{ACTION_END}"# + )) + .unwrap(); + + assert_eq!(result.calls.len(), 1); + assert_eq!(result.calls[0].name.as_deref(), Some("get_weather")); + } + + #[test] + fn internlm2_does_not_validate_or_normalize_arguments() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let arguments = r#"{"location":"Tokyo",}"#; + let result = parser + .parse_complete(&build_tool_call("get_weather", "parameters", arguments)) + .unwrap(); + + assert_eq!(result.calls[0].arguments, arguments); + } + + #[test] + fn internlm2_streaming_emits_argument_deltas() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let chunks = [ + "preface <|action", + "_start|><|plugin|>", + r#"{"name":"get_weather","parameters":"#, + r#"{"location":"#, + r#""Beijing""#, + r#"}"#, + r#"}<|action_end|> suffix"#, + ]; + + let mut result = ToolParserOutput::default(); + let mut observed_arguments = Vec::new(); + for chunk in chunks { + let next = parser.parse_chunk(chunk).unwrap(); + observed_arguments.extend( + next.calls + .iter() + .filter(|call| call.name.is_none()) + .map(|call| call.arguments.clone()), + ); + result.append(next); + } + result.append(parser.finish().unwrap()); + + assert_eq!( + observed_arguments, + [r#"{"location":"#, r#""Beijing""#, r#"}"#] + ); + assert_eq!(result.normal_text, "preface suffix"); + assert_eq!( + result.coalesce_calls().calls[0].arguments, + r#"{"location":"Beijing"}"# + ); + } + + #[test] + fn internlm2_streaming_handles_split_markers() { + let input = format!( + "hello {}", + build_tool_call("get_weather", "parameters", r#"{"location":"Tokyo"}"#) + ); + let chunks = split_by_chars(&input, 5); + let mut parser = Internlm2ToolParser::new(&test_tools()); + + let result = collect_stream(&mut parser, &chunks); + + assert_eq!(result.normal_text, "hello "); + assert_eq!(result.calls.len(), 1); + assert_eq!(result.calls[0].arguments, r#"{"location":"Tokyo"}"#); + } + + #[test] + fn internlm2_streaming_extracts_multiple_blocks() { + let input = format!( + "{}{}", + build_tool_call("get_weather", "parameters", r#"{"location":"Shanghai"}"#), + build_tool_call("add", "arguments", r#"{"x":1,"y":2}"#), + ); + let chunks = split_by_chars(&input, 7); + let mut parser = Internlm2ToolParser::new(&test_tools()); + + let result = collect_stream(&mut parser, &chunks); + + expect![[r#" + ToolParserOutput { + normal_text: "", + calls: [ + ToolCallDelta { + tool_index: 0, + name: Some( + "get_weather", + ), + arguments: "{\"location\":\"Shanghai\"}", + }, + ToolCallDelta { + tool_index: 1, + name: Some( + "add", + ), + arguments: "{\"x\":1,\"y\":2}", + }, + ], + } + "#]] + .assert_debug_eq(&result); + } + + #[test] + fn internlm2_keeps_end_marker_literal_inside_json_string() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let arguments = format!(r#"{{"text":"literal {ACTION_END} inside"}}"#); + let input = build_tool_call("echo", "parameters", &arguments); + + let result = parser.parse_complete(&input).unwrap(); + + assert_eq!(result.calls.len(), 1); + assert_eq!(result.calls[0].arguments, arguments); + } + + #[test] + fn internlm2_finish_errors_on_truncated_tool_call() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let pre_finish = parser + .parse_chunk(&format!( + r#"{ACTION_START}{{"name":"get_weather","parameters":{{"location""# + )) + .unwrap(); + let error = parser.finish().unwrap_err(); + + assert_eq!( + pre_finish.calls[0].name.as_deref(), + Some("get_weather"), + "name delta is still emitted from parse_chunk() before truncation", + ); + assert!( + error.to_report_string().contains("incomplete InternLM2 tool call"), + "finish() reports the truncated tool call as incomplete: {}", + error.to_report_string(), + ); + } + + #[test] + fn internlm2_unknown_arguments_key_fails() { + let mut parser = Internlm2ToolParser::new(&test_tools()); + let input = build_tool_call("get_weather", "params", r#"{"location":"Tokyo"}"#); + + let error = parser.parse_chunk(&input).unwrap_err(); + + expect![[r#" + tool parser parsing failed: invalid InternLM2 + expected `parameters`, `arguments`"#]] + .assert_eq(&error.to_report_string()); + } + + #[test] + fn internlm2_preserve_special_tokens_is_true() { + let parser = Internlm2ToolParser::new(&test_tools()); + assert!(parser.preserve_special_tokens()); + } +} diff --git a/rust/src/tool-parser/src/json/llama.rs b/rust/src/tool-parser/src/json/llama.rs index 5ad6918b61..36bc8a8347 100644 --- a/rust/src/tool-parser/src/json/llama.rs +++ b/rust/src/tool-parser/src/json/llama.rs @@ -203,7 +203,7 @@ fn llama_tool_call_header_event(input: &mut JsonToolInput<'_>) -> ModalResult, name_key: &'static str, - arguments_key: &'static str, + /// Candidate JSON keys naming the arguments payload, tried in order. + /// Most parsers use a single key like `["arguments"]`, but some accept + /// multiple (e.g. InternLM2 accepts `parameters` or `arguments`). + arguments_key: &'static [&'static str], } #[derive(Debug, Clone, Copy)] @@ -224,7 +229,7 @@ fn tool_call_header_event( _: ws0, _: literal(","), _: ws0, - _: |input: &mut JsonToolInput<'_>| json_key(input, config.arguments_key), + _: |input: &mut JsonToolInput<'_>| json_arguments_key(input, config.arguments_key), _: ws0, _: literal(":"), _: ws0, @@ -246,6 +251,39 @@ fn json_key(input: &mut JsonToolInput<'_>, key: &'static str) -> ModalResult<()> .parse_next(input) } +/// Parse a JSON object key accepting any of `candidates`. +/// +/// The full quoted key is consumed and compared against the candidate list, +/// so this works correctly under partial input regardless of key lengths. +/// +/// On mismatch, each candidate is attached as its own `Expected` context so the +/// error enumerates every valid key ("expected `a`, expected `b`"). Because +/// `StrContextValue::StringLiteral` carries a single `&'static str`, the +/// contexts are added in a loop over `candidates` rather than through chained +/// `.context(...)` calls, which keeps the diagnostics complete for any number +/// of candidates. +fn json_arguments_key( + input: &mut JsonToolInput<'_>, + candidates: &'static [&'static str], +) -> ModalResult<()> { + let start = input.checkpoint(); + json_str + .verify(|key: &String| candidates.contains(&key.as_str())) + .void() + .parse_next(input) + .map_err(|err| { + err.map(|context_error| { + candidates.iter().fold(context_error, |context_error, candidate| { + context_error.add_context( + &*input, + &start, + StrContext::Expected(StrContextValue::StringLiteral(candidate)), + ) + }) + }) + }) +} + /// Parse one event inside a marker-wrapped JSON tool-call arguments payload. fn parse_arguments_event( input: &mut JsonToolInput<'_>, @@ -341,7 +379,7 @@ mod tests { marker_whitespace: JsonToolCallWhitespace::Optional, delimiter: Some("<"), name_key: "function", - arguments_key: "parameters", + arguments_key: &["parameters"], }; fn build_tool_call(function_name: &str, arguments: &str) -> String { diff --git a/rust/src/tool-parser/src/json/qwen.rs b/rust/src/tool-parser/src/json/qwen.rs index f58ca6e0fa..b8caff0fef 100644 --- a/rust/src/tool-parser/src/json/qwen.rs +++ b/rust/src/tool-parser/src/json/qwen.rs @@ -8,7 +8,7 @@ const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig { marker_whitespace: JsonToolCallWhitespace::Exact("\n"), delimiter: None, name_key: "name", - arguments_key: "arguments", + arguments_key: &["arguments"], }; /// Tool parser for Qwen XML-wrapped JSON tool calls. diff --git a/rust/src/tool-parser/src/lib.rs b/rust/src/tool-parser/src/lib.rs index d8f648b81d..f1dc045584 100644 --- a/rust/src/tool-parser/src/lib.rs +++ b/rust/src/tool-parser/src/lib.rs @@ -24,7 +24,10 @@ pub use error::{Result, ToolParserError}; pub use gemma4::Gemma4ToolParser; pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser}; pub use hy_v3::HyV3ToolParser; -pub use json::{HermesToolParser, Llama3JsonToolParser, MistralToolParser, Qwen3XmlToolParser}; +pub use json::{ + HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser, MistralToolParser, + Qwen3XmlToolParser, +}; pub use kimi_k2::KimiK2ToolParser; pub use minimax_m2::MinimaxM2ToolParser; pub use qwen_coder::Qwen3CoderToolParser;