[Rust Frontend] Fix several hf chat template rendering issues (#44311)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -34,26 +34,23 @@ const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||
/// This Rust port intentionally diverges from
|
||||
/// `vllm/tool_parsers/internlm2_tool_parser.py` in two user-visible ways:
|
||||
///
|
||||
/// - **Parallel tool calls are supported.** Python silently drops every
|
||||
/// `<|action_start|>` block after the first (`current_tool_id > 0` returns
|
||||
/// an empty delta); this parser emits every well-formed block with
|
||||
/// incrementing `tool_index`. Models that legitimately emit multiple action
|
||||
/// blocks therefore produce more tool calls under Rust than under Python.
|
||||
/// - **End-marker bytes inside JSON string values are preserved.** Python
|
||||
/// does `action.split("<|action_end|>")[0]` which truncates regardless of
|
||||
/// JSON context; this parser scans matched braces and quotes so a literal
|
||||
/// `<|action_end|>` inside an arguments string is forwarded intact.
|
||||
/// - **Parallel tool calls are supported.** Python silently drops every `<|action_start|>` block
|
||||
/// after the first (`current_tool_id > 0` returns an empty delta); this parser emits every
|
||||
/// well-formed block with incrementing `tool_index`. Models that legitimately emit multiple
|
||||
/// action blocks therefore produce more tool calls under Rust than under Python.
|
||||
/// - **End-marker bytes inside JSON string values are preserved.** Python does
|
||||
/// `action.split("<|action_end|>")[0]` which truncates regardless of JSON context; this parser
|
||||
/// scans matched braces and quotes so a literal `<|action_end|>` inside an arguments string is
|
||||
/// forwarded intact.
|
||||
/// - **Only whitespace is allowed before the `{`.** Python's non-streaming
|
||||
/// `action[action.find("{"):]` drops any bytes before the first `{`, but
|
||||
/// its streaming path has no equivalent and the model format always emits
|
||||
/// `<|plugin|>{...`; this parser allows only whitespace there, matching the
|
||||
/// other JSON parsers in this crate.
|
||||
/// - **Truncated tool calls error rather than silently dropping.** Python's
|
||||
/// streaming wrapper swallows mid-stream errors with `except Exception:
|
||||
/// return None` (logging a traceback) while its non-streaming path raises
|
||||
/// `JSONDecodeError`; this parser returns an `incomplete InternLM2 tool
|
||||
/// call` error from `finish()`, matching the other JSON parsers and Python's
|
||||
/// non-streaming behavior.
|
||||
/// `action[action.find("{"):]` drops any bytes before the first `{`, but its streaming path has
|
||||
/// no equivalent and the model format always emits `<|plugin|>{...`; this parser allows only
|
||||
/// whitespace there, matching the other JSON parsers in this crate.
|
||||
/// - **Truncated tool calls error rather than silently dropping.** Python's streaming wrapper
|
||||
/// swallows mid-stream errors with `except Exception: return None` (logging a traceback) while
|
||||
/// its non-streaming path raises `JSONDecodeError`; this parser returns an `incomplete InternLM2
|
||||
/// tool call` error from `finish()`, matching the other JSON parsers and Python's non-streaming
|
||||
/// behavior.
|
||||
///
|
||||
/// # Known unaddressed divergences (TODO)
|
||||
///
|
||||
@@ -63,21 +60,18 @@ const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||
/// Qwen as well. If a real-world InternLM2 deployment hits one of these,
|
||||
/// prioritize the corresponding fix.
|
||||
///
|
||||
/// - **Arguments value type.** The shared core requires the arguments value
|
||||
/// to be a JSON object (`take_json_object` rejects anything not starting
|
||||
/// with `{`). Python's `json.dumps(action_dict.get("parameters", ...))`
|
||||
/// accepts `null`, arrays, strings, and numbers and round-trips them
|
||||
/// verbatim. Models that legitimately emit `"parameters":null` will hard-
|
||||
/// - **Arguments value type.** The shared core requires the arguments value to be a JSON object
|
||||
/// (`take_json_object` rejects anything not starting with `{`). Python's
|
||||
/// `json.dumps(action_dict.get("parameters", ...))` accepts `null`, arrays, strings, and numbers
|
||||
/// and round-trips them verbatim. Models that legitimately emit `"parameters":null` will hard-
|
||||
/// fail under Rust.
|
||||
/// - **Unknown arguments key.** Python falls back to `{}` via
|
||||
/// `action_dict.get("parameters", action_dict.get("arguments", {}))` when
|
||||
/// neither key is present; the Rust header parser raises
|
||||
/// `parsing failed: invalid InternLM2` for any unrecognized key. A model
|
||||
/// that emits a typo (e.g. `"params"`) breaks the whole response.
|
||||
/// - **Field order independence.** The header parser requires the JSON keys
|
||||
/// to appear in the order `name` then arguments key. Python's
|
||||
/// `json.loads` + `dict.get` is order-independent, so a model emitting
|
||||
/// `{"parameters":{...},"name":"foo"}` parses in Python but fails in Rust.
|
||||
/// - **Unknown arguments key.** Python falls back to `{}` via `action_dict.get("parameters",
|
||||
/// action_dict.get("arguments", {}))` when neither key is present; the Rust header parser raises
|
||||
/// `parsing failed: invalid InternLM2` for any unrecognized key. A model that emits a typo (e.g.
|
||||
/// `"params"`) breaks the whole response.
|
||||
/// - **Field order independence.** The header parser requires the JSON keys to appear in the order
|
||||
/// `name` then arguments key. Python's `json.loads` + `dict.get` is order-independent, so a model
|
||||
/// emitting `{"parameters":{...},"name":"foo"}` parses in Python but fails in Rust.
|
||||
pub struct Internlm2ToolParser {
|
||||
inner: JsonToolCallParser,
|
||||
}
|
||||
|
||||
@@ -501,15 +501,21 @@ mod tests {
|
||||
|
||||
assert_eq!(converted_number_text(¶ms, "5"), "5");
|
||||
assert_eq!(converted_number_text(¶ms, "5.0"), "5.0");
|
||||
assert_eq!(converted_number_text(¶ms, "5.00"), "5.00");
|
||||
assert_eq!(converted_number_text(¶ms, "1e0"), "1e+0");
|
||||
assert_eq!(converted_number_text(¶ms, "5."), "5.0");
|
||||
assert_eq!(converted_number_text(¶ms, "+1"), "1");
|
||||
assert_eq!(converted_number_text(¶ms, "+1.0"), "1.0");
|
||||
assert_eq!(
|
||||
converted_number_text(¶ms, "9223372036854775807.5"),
|
||||
"9223372036854775807.5"
|
||||
);
|
||||
|
||||
// TODO: we cannot preserve the original number precision by enabling `serde_json`'s
|
||||
// `arbitrary_precision` feature, otherwise the test
|
||||
// `serialized_json_numbers_do_not_leak_serde_private_representation` will fail.
|
||||
// See issue: https://github.com/mitsuhiko/minijinja/issues/641
|
||||
|
||||
// assert_eq!(converted_number_text(¶ms, "5.00"), "5.00");
|
||||
// assert_eq!(converted_number_text(¶ms, "1e0"), "1e+0");
|
||||
// assert_eq!(
|
||||
// converted_number_text(¶ms, "9223372036854775807.5"),
|
||||
// "9223372036854775807.5"
|
||||
// );
|
||||
}
|
||||
|
||||
fn converted_number_text(params: &ToolSchema, value: &str) -> String {
|
||||
|
||||
Reference in New Issue
Block a user