[Model] Add Kimi K3 support: Rust frontend [1/2] (#50104)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-28 14:31:13 -07:00
committed by GitHub
parent 0b6aa3c47c
commit 2899dca843
35 changed files with 3003 additions and 39 deletions
+2 -1
View File
@@ -20,7 +20,7 @@ use crate::output::{
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
use crate::renderer::{
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
InklingChatRenderer,
InklingChatRenderer, KimiK3ChatRenderer,
};
use crate::request::ChatRequest;
use crate::{DynChatOutputProcessor, RendererSelection};
@@ -73,6 +73,7 @@ impl HfChatBackend {
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
};
info!(
+10 -3
View File
@@ -33,7 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection,
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
RendererSelection,
};
pub use request::{
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
@@ -353,6 +354,12 @@ mod tests {
.unwrap();
}
#[test]
fn validate_parser_overrides_accepts_explicit_kimi_k3() {
let selection = ParserSelection::Explicit("kimi_k3".to_string());
validate_parser_overrides(&selection, &selection).unwrap();
}
#[test]
fn validate_parser_overrides_accepts_auto_and_none() {
validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap();
@@ -366,7 +373,7 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
}
#[test]
@@ -377,6 +384,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
@@ -27,6 +27,7 @@ pub mod names {
pub const GLM45: &str = "glm45";
pub const KIMI: &str = "kimi";
pub const KIMI_K2: &str = "kimi_k2";
pub const KIMI_K3: &str = "kimi_k3";
pub const MINIMAX_M2: &str = "minimax_m2";
pub const MINIMAX_M3: &str = "minimax_m3";
pub const NEMOTRON_V3: &str = "nemotron_v3";
@@ -68,6 +69,7 @@ impl ReasoningParserFactory {
.register_parser::<Glm45ReasoningParser>(names::GLM45)
.register_parser::<KimiReasoningParser>(names::KIMI)
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
.register_unified_dummy(names::KIMI_K3)
.register_parser::<MiniMaxM2ReasoningParser>(names::MINIMAX_M2)
.register_parser::<MiniMaxM3ReasoningParser>(names::MINIMAX_M3)
.register_parser::<NemotronV3ReasoningParser>(names::NEMOTRON_V3)
+2
View File
@@ -33,6 +33,7 @@ pub mod names {
// also routes to `Internlm2ToolParser` despite the version-agnostic name.
pub const INTERNLM: &str = "internlm";
pub const KIMI_K2: &str = "kimi_k2";
pub const KIMI_K3: &str = "kimi_k3";
pub const LLAMA3_JSON: &str = "llama3_json";
pub const LLAMA4_JSON: &str = "llama4_json";
pub const MINIMAX_M2: &str = "minimax_m2";
@@ -78,6 +79,7 @@ impl ToolParserFactory {
.register_parser::<HyV3ToolParser>(names::HY_V3)
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
.register_unified_dummy(names::KIMI_K3)
.register_parser::<Llama3JsonToolParser>(names::LLAMA3_JSON)
.register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
.register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
+24 -2
View File
@@ -5,7 +5,9 @@
use std::sync::LazyLock;
pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser};
pub use vllm_parser::unified::{
Gemma4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, UnifiedParser,
};
use vllm_tokenizer::DynTokenizer;
use crate::parser::ParserFactory;
@@ -15,6 +17,7 @@ use crate::request::ChatTool;
pub mod names {
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
pub const KIMI_K3: &str = "kimi_k3";
}
/// Constructor signature for one registered unified parser implementation.
@@ -39,11 +42,14 @@ impl UnifiedParserFactory {
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
factory
.register_pattern("gemma-4", names::GEMMA4)
.register_pattern("gemma4", names::GEMMA4)
.register_pattern("inkling", names::INKLING);
.register_pattern("inkling", names::INKLING)
.register_pattern("kimi-k3", names::KIMI_K3)
.register_pattern("kimi_k3", names::KIMI_K3);
factory
}
@@ -121,4 +127,20 @@ mod tests {
);
factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap();
}
#[test]
fn factory_registers_kimi_k3() {
let factory = UnifiedParserFactory::new();
let tokenizer = TestTokenizer::new()
.with_regular_token("<|open|>", 1001)
.with_regular_token("<|close|>", 1002)
.with_regular_token("<|sep|>", 1003);
assert!(factory.contains(names::KIMI_K3));
assert_eq!(
factory.resolve_name_for_model("moonshotai/Kimi-K3"),
Some(names::KIMI_K3)
);
factory.create(names::KIMI_K3, &[], Arc::new(tokenizer)).unwrap();
}
}
@@ -59,7 +59,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
fn deepseek_fixture_options() -> FixtureRequestOptions {
FixtureRequestOptions {
enable_thinking: true,
enable_thinking: Some(true),
no_generation_prompt_when_last_assistant: true,
}
}
@@ -27,7 +27,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
fn deepseek_fixture_options() -> FixtureRequestOptions {
FixtureRequestOptions {
enable_thinking: true,
enable_thinking: Some(true),
no_generation_prompt_when_last_assistant: true,
}
}
+1 -1
View File
@@ -22,7 +22,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
fixture_chat_request(
&fixture_path(input_name),
FixtureRequestOptions {
enable_thinking: false,
enable_thinking: Some(false),
no_generation_prompt_when_last_assistant: false,
},
)
+1 -1
View File
@@ -105,7 +105,7 @@ fn fixture_request(name: &str) -> ChatRequest {
fn inkling_fixture_options() -> FixtureRequestOptions {
FixtureRequestOptions {
enable_thinking: false,
enable_thinking: Some(false),
no_generation_prompt_when_last_assistant: false,
}
}
@@ -0,0 +1,608 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Kimi K3 XTML prompt renderer.
//!
//! Port of Moonshot remote-code `encoding_k3.py::build_chat_segments()`.
use std::collections::HashMap;
use serde_json::{Map, Value, json};
use vllm_tokenizer::Tokenizer;
use crate::error::{Error, Result};
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
};
use crate::{AssistantContentBlock, AssistantToolCall};
pub(super) const OPEN: &str = "<|open|>";
pub(super) const CLOSE: &str = "<|close|>";
pub(super) const SEP: &str = "<|sep|>";
pub(super) const END_OF_MSG: &str = "<|end_of_msg|>";
pub(super) const IMAGE_PLACEHOLDER: &str = "<|media_pad|>";
const DEFAULT_THINKING_EFFORT: &str = "max";
const VALID_THINKING_EFFORTS: &[&str] = &["low", "high", "max"];
/// K3 prompt encoder preserving Python's per-segment tokenization boundaries.
pub(super) struct K3TokenWriter<'a> {
tokenizer: &'a dyn Tokenizer,
token_ids: Vec<u32>,
}
impl<'a> K3TokenWriter<'a> {
pub(super) fn new(tokenizer: &'a dyn Tokenizer) -> Self {
Self {
tokenizer,
token_ids: Vec::new(),
}
}
/// Encode one trusted segment with normal added-token recognition.
pub(super) fn control(&mut self, text: &str) -> Result<()> {
if !text.is_empty() {
self.token_ids.extend(self.tokenizer.encode(text, false)?);
}
Ok(())
}
/// Encode one literal segment while bypassing every added-token matcher.
pub(super) fn ordinary(&mut self, text: &str) -> Result<()> {
if !text.is_empty() {
self.token_ids.extend(self.tokenizer.encode_ordinary(text)?);
}
Ok(())
}
pub(super) fn finish(self) -> Vec<u32> {
self.token_ids
}
}
/// Render and tokenize one chat request using K3's segment-aware contract.
pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -> Result<Vec<u32>> {
let thinking = thinking_enabled(request)?;
let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?;
let tools = request_tools(request);
let mut out = K3TokenWriter::new(tokenizer);
if !tools.is_empty() {
write_tool_declare(&mut out, tools, false)?;
}
if let Some(effort) = thinking_effort {
// Preserve the checkpoint's literal guidance text: it still names
// `medium`, although the validator above no longer accepts it.
write_internal_system(
&mut out,
"thinking-effort",
&format!(
"`thinking_effort` guides on how much to think in your \
thinking channel (not including the response channel), \
supported values include `low`, `medium`, `high`, and `max`.\n\
Now the system is invoked with `thinking_effort={effort}`."
),
)?;
}
// Track prior assistant tool-call ids for tool-result reordering / naming.
let mut tool_call_id_index: HashMap<String, (usize, String)> = HashMap::new();
let mut pending_tool_run: Vec<(usize, ChatMessage)> = Vec::new();
let flush_tool_run = |out: &mut K3TokenWriter<'_>,
run: &mut Vec<(usize, ChatMessage)>,
id_index: &HashMap<String, (usize, String)>|
-> Result<()> {
if run.is_empty() {
return Ok(());
}
let mut resolved = Vec::with_capacity(run.len());
let mut unresolved = false;
for (offset, message) in run.drain(..) {
let ChatMessage::ToolResponse {
content,
tool_call_id,
} = message
else {
unreachable!("pending tool run only holds tool responses");
};
match id_index.get(&tool_call_id) {
Some(&(position, ref name)) => {
resolved.push((position, offset, content, Some(name.clone())));
}
None => {
unresolved = true;
resolved.push((usize::MAX, offset, content, None));
}
}
}
if unresolved {
// Preserve original order when the run cannot be fully matched.
resolved.sort_by_key(|item| item.1);
} else {
resolved.sort_by_key(|item| (item.0, item.1));
}
for (xtml_index, (_, _, content, name)) in resolved.into_iter().enumerate() {
let tool_name = name.as_deref().ok_or_else(|| {
Error::ChatTemplate(
"Kimi K3 tool messages need a resolvable tool name: \
carry a matching tool_call_id against a preceding \
assistant tool_call"
.to_string(),
)
})?;
write_tool_message(out, tool_name, xtml_index + 1, &content)?;
}
Ok(())
};
for (message_index, message) in request.messages.iter().enumerate() {
match message {
ChatMessage::ToolResponse { .. } => {
pending_tool_run.push((message_index, message.clone()));
continue;
}
_ => {
flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
}
}
match message {
// Python: role=system with a `tools` field renders a dynamic
// tool-declare (`## New Tools Available`). Map that to Developer
// messages that carry tools (OpenAI "developer" / system-tools).
ChatMessage::Developer {
content,
tools: Some(local_tools),
} if !local_tools.is_empty() => {
write_tool_declare(&mut out, local_tools, true)?;
if !content_is_empty(content) {
write_role_message(&mut out, "system", None, content)?;
}
}
ChatMessage::System { content } | ChatMessage::Developer { content, .. } => {
write_role_message(&mut out, "system", None, content)?;
}
ChatMessage::User { content } => {
write_role_message(&mut out, "user", None, content)?;
}
ChatMessage::Assistant { content } => {
tool_call_id_index.clear();
let mut call_position = 0usize;
for block in content {
if let AssistantContentBlock::ToolCall(call) = block {
call_position += 1;
if !call.id.is_empty() {
tool_call_id_index
.entry(call.id.clone())
.or_insert((call_position, call.name.clone()));
}
}
}
write_assistant_message(&mut out, content, thinking)?;
}
ChatMessage::ToolResponse { .. } => unreachable!("handled above"),
}
}
flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
match &request.tool_choice {
ChatToolChoice::Required => {
write_internal_system(
&mut out,
"tool-choice",
"The system is invoked with `tool_choice=required`.\n\
You MUST call tools in the next message.",
)?;
}
// Emit only when tools are present: Rust defaults tool_choice to None
// for tool-free requests, which must not inject a tool-choice message.
ChatToolChoice::None if !request.tools.is_empty() => {
write_internal_system(
&mut out,
"tool-choice",
"The system is invoked with `tool_choice=none`.\n\
You MUST NOT call any tools in the next message.",
)?;
}
ChatToolChoice::None | ChatToolChoice::Auto | ChatToolChoice::Function { .. } => {}
}
write_response_format(&mut out, request)?;
if request.chat_options.add_generation_prompt() {
write_open_tag(&mut out, "message", &[("role", "assistant")])?;
write_open_tag(&mut out, if thinking { "think" } else { "response" }, &[])?;
}
Ok(out.finish())
}
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
// Declare tools whenever the request carries them. K3 tool-declare is
// independent of tool_choice; tool_choice only injects control messages.
request.tools.as_slice()
}
fn thinking_enabled(request: &ChatRequest) -> Result<bool> {
if let Some(thinking) = request.parse_template_bool("thinking")? {
return Ok(thinking);
}
if let Some(enable_thinking) = request.parse_template_bool("enable_thinking")? {
return Ok(enable_thinking);
}
Ok(request
.chat_options
.reasoning_effort
.map(|effort| effort != crate::request::ReasoningEffort::None)
.unwrap_or(true))
}
fn thinking_effort(request: &ChatRequest) -> Result<String> {
let effort = if let Some(value) = request.chat_options.template_kwargs.get("thinking_effort") {
value.as_str().ok_or_else(|| {
Error::ChatTemplate(format!(
"template kwarg `thinking_effort` must be a string, got {value}"
))
})?
} else if let Some(effort) = request.chat_options.reasoning_effort {
effort.as_str()
} else if let Some(value) = request.chat_options.template_kwargs.get("reasoning_effort") {
value.as_str().ok_or_else(|| {
Error::ChatTemplate(format!(
"template kwarg `reasoning_effort` must be a string, got {value}"
))
})?
} else {
DEFAULT_THINKING_EFFORT
};
if !VALID_THINKING_EFFORTS.contains(&effort) {
return Err(Error::ChatTemplate(format!(
"unsupported thinking_effort={effort:?}; supported values are `low`, `high`, and `max`"
)));
}
Ok(effort.to_string())
}
fn content_is_empty(content: &ChatContent) -> bool {
match content {
ChatContent::Text(text) => text.is_empty(),
ChatContent::Parts(parts) => parts.iter().all(|part| match part {
ChatContentPart::Text { text } => text.is_empty(),
ChatContentPart::ImageUrl { .. }
| ChatContentPart::VideoUrl { .. }
| ChatContentPart::InputAudio { .. }
| ChatContentPart::AudioUrl { .. } => false,
}),
}
}
fn write_tool_declare(
out: &mut K3TokenWriter<'_>,
tools: &[ChatTool],
dynamic: bool,
) -> Result<()> {
let mut specs = Vec::with_capacity(tools.len());
for tool in tools {
let mut function = Map::new();
function.insert(
"description".to_string(),
Value::String(tool.description.clone().unwrap_or_default()),
);
function.insert("name".to_string(), Value::String(tool.name.clone()));
function.insert("parameters".to_string(), sort_json(&tool.parameters));
specs.push(json!({
"function": Value::Object(function),
"type": "function",
}));
}
let payload = compact_json(&sort_json(&Value::Array(specs)))?;
let body = if dynamic {
format!(
"## New Tools Available\n\
The system dynamically extends the toolset via lazy-loading.\n\
You have access to all existing and extended tools.\n\
Here are the specs for the extended tools.\n\n\
```json\n\
{payload}\n\
```"
)
} else {
format!(
"# Tools\n\
Here are the available tools, described in JSONSchema.\n\n\
```json\n\
{payload}\n\
```"
)
};
write_internal_system(out, "tool-declare", &body)
}
fn write_internal_system(
out: &mut K3TokenWriter<'_>,
message_type: &str,
body: &str,
) -> Result<()> {
write_open_tag(
out,
"message",
&[("role", "system"), ("type", message_type)],
)?;
out.ordinary(body.trim())?;
write_close_tag(out, "message")?;
out.control(END_OF_MSG)
}
fn write_role_message(
out: &mut K3TokenWriter<'_>,
role: &str,
name: Option<&str>,
content: &ChatContent,
) -> Result<()> {
let mut attrs = vec![("role", role.to_string())];
if let Some(name) = name.filter(|name| !name.is_empty()) {
attrs.push(("name", name.to_string()));
}
let attr_refs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect();
write_open_tag(out, "message", &attr_refs)?;
write_content(out, content)?;
write_close_tag(out, "message")?;
out.control(END_OF_MSG)
}
fn write_tool_message(
out: &mut K3TokenWriter<'_>,
tool_name: &str,
index: usize,
content: &ChatContent,
) -> Result<()> {
let index_str = index.to_string();
write_open_tag(
out,
"message",
&[("role", "tool"), ("tool", tool_name), ("index", &index_str)],
)?;
write_content(out, content)?;
write_close_tag(out, "message")?;
out.control(END_OF_MSG)
}
fn write_assistant_message(
out: &mut K3TokenWriter<'_>,
content: &[AssistantContentBlock],
thinking: bool,
) -> Result<()> {
write_open_tag(out, "message", &[("role", "assistant")])?;
let mut reasoning = String::new();
let mut response = String::new();
let mut tool_calls = Vec::new();
for block in content {
match block {
AssistantContentBlock::Reasoning { text } => reasoning.push_str(text),
AssistantContentBlock::Text { text } => response.push_str(text),
AssistantContentBlock::ToolCall(call) => tool_calls.push(call),
}
}
// The think channel is structural: in thinking mode every assistant
// message carries open/close tags even when there is no reasoning content.
// In non-thinking mode the channel is dropped entirely.
if thinking {
write_open_tag(out, "think", &[])?;
if !reasoning.trim().is_empty() {
out.ordinary(&reasoning)?;
}
write_close_tag(out, "think")?;
}
write_open_tag(out, "response", &[])?;
out.ordinary(&response)?;
write_close_tag(out, "response")?;
if !tool_calls.is_empty() {
write_open_tag(out, "tools", &[])?;
for (index, tool_call) in tool_calls.into_iter().enumerate() {
write_assistant_tool_call(out, tool_call, index + 1)?;
}
write_close_tag(out, "tools")?;
}
write_close_tag(out, "message")?;
out.control(END_OF_MSG)
}
fn write_assistant_tool_call(
out: &mut K3TokenWriter<'_>,
tool_call: &AssistantToolCall,
index: usize,
) -> Result<()> {
let index_str = index.to_string();
write_open_tag(
out,
"call",
&[
("tool", tool_call.name.as_str()),
("index", index_str.as_str()),
],
)?;
let (args, json_block) = normalize_tool_arguments(&tool_call.arguments)?;
if let Some(raw) = json_block {
write_open_tag(out, "json", &[("type", "object")])?;
out.ordinary(&raw)?;
write_close_tag(out, "json")?;
} else {
for (key, value) in args {
let typ = xtml_type(&value);
write_open_tag(out, "argument", &[("key", key.as_str()), ("type", typ)])?;
out.ordinary(&xtml_value(&value))?;
write_close_tag(out, "argument")?;
}
}
write_close_tag(out, "call")
}
fn write_content(out: &mut K3TokenWriter<'_>, content: &ChatContent) -> Result<()> {
match content {
ChatContent::Text(text) => write_text_with_images(out, text),
ChatContent::Parts(parts) => {
for part in parts {
match part {
ChatContentPart::Text { text } => write_text_with_images(out, text)?,
ChatContentPart::ImageUrl { .. } => out.control(IMAGE_PLACEHOLDER)?,
ChatContentPart::VideoUrl { .. } => {
return Err(Error::UnsupportedMultimodalContent("video_url"));
}
ChatContentPart::InputAudio { .. } => {
return Err(Error::UnsupportedMultimodalContent("input_audio"));
}
ChatContentPart::AudioUrl { .. } => {
return Err(Error::UnsupportedMultimodalContent("audio_url"));
}
}
}
Ok(())
}
}
}
fn write_text_with_images(out: &mut K3TokenWriter<'_>, text: &str) -> Result<()> {
// Placeholder expansion is left as the literal K3 image token; multimodal
// preprocessing can replace it once image prompts are known.
out.ordinary(text)
}
fn write_response_format(out: &mut K3TokenWriter<'_>, request: &ChatRequest) -> Result<()> {
let Some(rf) = request.chat_options.response_format.as_ref() else {
return Ok(());
};
let rf_type = rf.get("type").and_then(Value::as_str).or_else(|| rf.as_str()).unwrap_or("");
match rf_type {
"json_object" => {
write_internal_system(
out,
"response-format",
"The system is invoked with `response_format=json_object`.\n\
Your response must be raw JSON data without markdown code \
blocks (```json) or any additional formatting.",
)?;
}
"json_schema" => {
let schema = extract_response_schema(rf);
let schema_json = compact_json(&sort_json(&schema.unwrap_or(Value::Null)))?;
write_internal_system(
out,
"response-format",
&format!(
"The system is invoked with `response_format=json_schema`.\n\
Your response must be raw JSON data without markdown code \
blocks (```json) or any additional formatting.\n\
The JSON data must match the following schema:\n\
```json\n\
{schema_json}\n\
```"
),
)?;
}
_ => {}
}
Ok(())
}
fn extract_response_schema(response_format: &Value) -> Option<Value> {
let json_schema = response_format.get("json_schema")?;
if let Some(schema) = json_schema.get("schema") {
return Some(schema.clone());
}
if let Some(schema) = json_schema.get("json_schema") {
return Some(schema.clone());
}
Some(json_schema.clone())
}
fn normalize_tool_arguments(arguments: &str) -> Result<(Map<String, Value>, Option<String>)> {
let trimmed = arguments.trim();
if trimmed.is_empty() {
return Ok((Map::new(), None));
}
match serde_json::from_str::<Value>(trimmed) {
Ok(Value::Object(map)) => Ok((map, None)),
Ok(_) => Err(Error::ChatTemplate(
"Kimi K3 tool call arguments must be a JSON object".to_string(),
)),
Err(_) => Ok((Map::new(), Some(arguments.to_string()))),
}
}
fn xtml_type(value: &Value) -> &'static str {
match value {
Value::Bool(_) => "boolean",
Value::Null => "null",
Value::Number(_) => "number",
Value::String(_) => "string",
Value::Object(_) => "object",
Value::Array(_) => "array",
}
}
fn xtml_value(value: &Value) -> String {
match value {
Value::String(text) => text.clone(),
other => compact_json(other).unwrap_or_else(|_| other.to_string()),
}
}
fn write_open_tag(out: &mut K3TokenWriter<'_>, tag: &str, attrs: &[(&str, &str)]) -> Result<()> {
out.control(OPEN)?;
out.ordinary(tag)?;
for (key, value) in attrs {
out.ordinary(&format!(" {key}"))?;
out.ordinary("=\"")?;
out.ordinary(&escape_attr_value(value))?;
out.ordinary("\"")?;
}
out.control(SEP)
}
fn write_close_tag(out: &mut K3TokenWriter<'_>, tag: &str) -> Result<()> {
out.control(CLOSE)?;
out.ordinary(tag)?;
out.control(SEP)
}
fn escape_attr_value(value: &str) -> String {
value.replace('&', "&amp;").replace('"', "&quot;")
}
fn compact_json(value: &Value) -> Result<String> {
serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.to_string()))
}
fn sort_json(value: &Value) -> Value {
match value {
Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()),
Value::Object(map) => {
let mut sorted = Map::new();
let mut keys = map.keys().collect::<Vec<_>>();
keys.sort();
for key in keys {
sorted.insert(key.clone(), sort_json(&map[key]));
}
Value::Object(sorted)
}
_ => value.clone(),
}
}
@@ -0,0 +1,15 @@
{
"messages": [
{
"role": "user",
"content": "json pls"
}
],
"add_generation_prompt": true,
"response_format": {
"type": "json_object"
},
"template_kwargs": {
"thinking": false
}
}
@@ -0,0 +1,2 @@
<|open|>message role="user"<|sep|>json pls<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="response-format"<|sep|>The system is invoked with `response_format=json_object`.
Your response must be raw JSON data without markdown code blocks (```json) or any additional formatting.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>response<|sep|>
@@ -0,0 +1,62 @@
{
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "weather",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string"
},
"days": {
"type": "number"
}
},
"required": [
"city"
]
}
}
}
],
"messages": [
{
"role": "user",
"content": "hi"
},
{
"role": "developer",
"tools": [
{
"type": "function",
"function": {
"name": "calc",
"description": "calculator",
"parameters": {
"type": "object",
"properties": {
"x": {
"type": "number"
}
},
"required": [
"x"
]
}
}
}
]
},
{
"role": "user",
"content": "use calc"
}
],
"add_generation_prompt": true,
"template_kwargs": {
"thinking": true
}
}
@@ -0,0 +1,14 @@
<|open|>message role="system" type="tool-declare"<|sep|># Tools
Here are the available tools, described in JSONSchema.
```json
[{"function":{"description":"weather","name":"get_weather","parameters":{"properties":{"city":{"type":"string"},"days":{"type":"number"}},"required":["city"],"type":"object"}},"type":"function"}]
```<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>hi<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-declare"<|sep|>## New Tools Available
The system dynamically extends the toolset via lazy-loading.
You have access to all existing and extended tools.
Here are the specs for the extended tools.
```json
[{"function":{"description":"calculator","name":"calc","parameters":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}},"type":"function"}]
```<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>use calc<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
@@ -0,0 +1,31 @@
{
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "see this"
},
{
"type": "image_url",
"image_url": "https://example.com/a.png"
}
]
},
{
"role": "assistant",
"content": "old answer",
"reasoning_content": "old reasoning"
},
{
"role": "user",
"content": "continue"
}
],
"add_generation_prompt": true,
"template_kwargs": {
"thinking": true,
"thinking_effort": "high"
}
}
@@ -0,0 +1,2 @@
<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
Now the system is invoked with `thinking_effort=high`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>see this<|media_pad|><|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>old reasoning<|close|>think<|sep|><|open|>response<|sep|>old answer<|close|>response<|sep|><|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>continue<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
@@ -0,0 +1,87 @@
{
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "weather",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string"
},
"days": {
"type": "number"
}
},
"required": [
"city"
]
}
}
},
{
"type": "function",
"function": {
"name": "calc",
"description": "calculator",
"parameters": {
"type": "object",
"properties": {
"x": {
"type": "number"
}
},
"required": [
"x"
]
}
}
}
],
"messages": [
{
"role": "user",
"content": "Hangzhou weather and calc"
},
{
"role": "assistant",
"content": "I'll check.",
"reasoning_content": "Need tools.",
"tool_calls": [
{
"id": "get_weather:0",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"city\":\"Hangzhou\",\"days\":1}"
}
},
{
"id": "calc:1",
"type": "function",
"function": {
"name": "calc",
"arguments": "{\"x\":2}"
}
}
]
},
{
"role": "tool",
"tool_call_id": "calc:1",
"content": "2"
},
{
"role": "tool",
"tool_call_id": "get_weather:0",
"content": "{\"city\":\"Hangzhou\",\"condition\":\"rain\"}"
}
],
"add_generation_prompt": true,
"tool_choice": "required",
"template_kwargs": {
"thinking": true
}
}
@@ -0,0 +1,8 @@
<|open|>message role="system" type="tool-declare"<|sep|># Tools
Here are the available tools, described in JSONSchema.
```json
[{"function":{"description":"weather","name":"get_weather","parameters":{"properties":{"city":{"type":"string"},"days":{"type":"number"}},"required":["city"],"type":"object"}},"type":"function"},{"function":{"description":"calculator","name":"calc","parameters":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}},"type":"function"}]
```<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>Hangzhou weather and calc<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>Need tools.<|close|>think<|sep|><|open|>response<|sep|>I'll check.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool="get_weather" index="1"<|sep|><|open|>argument key="city" type="string"<|sep|>Hangzhou<|close|>argument<|sep|><|open|>argument key="days" type="number"<|sep|>1<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool="calc" index="2"<|sep|><|open|>argument key="x" type="number"<|sep|>2<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|><|end_of_msg|><|open|>message role="tool" tool="get_weather" index="1"<|sep|>{"city":"Hangzhou","condition":"rain"}<|close|>message<|sep|><|end_of_msg|><|open|>message role="tool" tool="calc" index="2"<|sep|>2<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-choice"<|sep|>The system is invoked with `tool_choice=required`.
You MUST call tools in the next message.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
+39
View File
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Native Kimi K3 XTML chat renderer.
mod encoding;
#[cfg(test)]
mod tests;
use vllm_text::Prompt;
use vllm_text::tokenizer::DynTokenizer;
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::Result;
use crate::request::ChatRequest;
/// Dedicated Kimi K3 XTML renderer.
#[derive(Clone)]
pub struct KimiK3ChatRenderer {
tokenizer: DynTokenizer,
}
impl KimiK3ChatRenderer {
/// Create a Kimi K3 renderer.
pub fn new(tokenizer: DynTokenizer) -> Self {
Self { tokenizer }
}
}
impl ChatRenderer for KimiK3ChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
Ok(RenderedPrompt {
prompt: Prompt::TokenIds(encoding::render_request(request, self.tokenizer.as_ref())?),
effective_template_kwargs: request_template_kwargs(request),
})
}
}
+267
View File
@@ -0,0 +1,267 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Golden fixtures generated from HF remote-code `encoding_k3.py`.
use std::path::PathBuf;
use std::sync::Arc;
use expect_test::{expect, expect_file};
use serde_json::json;
use vllm_text::Prompt;
use vllm_text::tokenizer::DynTokenizer;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::KimiK3ChatRenderer;
use crate::AssistantContentBlock;
use crate::ChatRenderer;
use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{ChatContentPart, ChatMessage, GenerationPromptMode, ReasoningEffort};
const OPEN_ID: u32 = 256;
const CLOSE_ID: u32 = 257;
const SEP_ID: u32 = 258;
const END_OF_MSG_ID: u32 = 259;
const MEDIA_ID: u32 = 260;
fn test_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_special_token(OPEN, OPEN_ID)
.with_special_token(CLOSE, CLOSE_ID)
.with_special_token(SEP, SEP_ID)
.with_special_token(END_OF_MSG, END_OF_MSG_ID)
.with_special_token(IMAGE_PLACEHOLDER, MEDIA_ID)
}
fn render_token_ids(request: &crate::request::ChatRequest, tokenizer: DynTokenizer) -> Vec<u32> {
let prompt = KimiK3ChatRenderer::new(tokenizer).render(request).unwrap().prompt;
let Prompt::TokenIds(token_ids) = prompt else {
panic!("kimi k3 renderer should return token IDs")
};
token_ids
}
fn render_request(request: &crate::request::ChatRequest) -> String {
let tokenizer: DynTokenizer = Arc::new(test_tokenizer());
let token_ids = render_token_ids(request, tokenizer.clone());
tokenizer.decode(&token_ids, false).unwrap()
}
fn fixture_path(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("src/renderer/kimi_k3/fixtures")
.join(name)
}
fn kimi_k3_fixture_options() -> FixtureRequestOptions {
FixtureRequestOptions {
// Fixture JSON owns thinking via `template_kwargs`.
enable_thinking: None,
no_generation_prompt_when_last_assistant: false,
}
}
fn assert_golden(name: &str) {
let input_name = format!("{name}_input.json");
let request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options());
let rendered = render_request(&request);
expect_file![format!("fixtures/{name}_output.txt")].assert_eq(&rendered);
}
#[test]
fn golden_history_preserve_and_image() {
assert_golden("history_preserve_and_image");
}
#[test]
fn golden_tools_history_and_required() {
assert_golden("tools_history_and_required");
}
#[test]
fn golden_controls_thinking_off() {
assert_golden("controls_thinking_off");
}
#[test]
fn golden_dynamic_system_tool_declare() {
assert_golden("dynamic_system_tool_declare");
}
#[test]
fn token_writer_protects_literal_control_and_media_markers() {
let tokenizer = Arc::new(test_tokenizer());
let user_text = format!("literal {OPEN} and {}", super::encoding::IMAGE_PLACEHOLDER);
let mut request = crate::request::ChatRequest::for_test();
request.messages = vec![ChatMessage::user(vec![
ChatContentPart::text(user_text),
ChatContentPart::image_url("data:image/png;base64,test"),
])];
request
.chat_options
.template_kwargs
.insert("thinking".to_string(), json!(false));
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
let token_ids = render_token_ids(&request, tokenizer.clone());
assert_eq!(
token_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
1
);
assert_eq!(
token_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
1
);
let flattened = tokenizer.decode(&token_ids, false).unwrap();
let flattened_ids = tokenizer.encode(&flattened, false).unwrap();
assert_eq!(
flattened_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
2
);
assert_eq!(
flattened_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
2
);
}
#[test]
fn thinking_history_renders_empty_think_channel() {
let mut request = crate::request::ChatRequest::for_test();
request.messages = vec![
ChatMessage::user("question"),
ChatMessage::assistant_text("answer"),
ChatMessage::user("follow-up"),
];
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
let rendered = render_request(&request);
assert!(rendered.contains(
"<|open|>message role=\"assistant\"<|sep|>\
<|open|>think<|sep|><|close|>think<|sep|>\
<|open|>response<|sep|>answer<|close|>response<|sep|>"
));
}
#[test]
fn non_thinking_history_omits_reasoning_channel() {
let mut request = crate::request::ChatRequest::for_test();
request.messages = vec![
ChatMessage::user("question"),
ChatMessage::assistant_blocks(vec![
AssistantContentBlock::Reasoning {
text: "hidden reasoning".to_string(),
},
AssistantContentBlock::Text {
text: "answer".to_string(),
},
]),
ChatMessage::user("follow-up"),
];
request
.chat_options
.template_kwargs
.insert("thinking".to_string(), json!(false));
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
let rendered = render_request(&request);
assert!(!rendered.contains("hidden reasoning"));
assert!(!rendered.contains("<|open|>think<|sep|>"));
assert!(rendered.contains(
"<|open|>message role=\"assistant\"<|sep|>\
<|open|>response<|sep|>answer<|close|>response<|sep|>"
));
}
#[test]
fn defaults_thinking_effort_to_max() {
let rendered = render_request(&crate::request::ChatRequest::for_test());
expect![[r#"<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>test<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>"#]]
.assert_eq(&rendered);
}
#[test]
fn translates_standard_thinking_kwargs() {
let mut request = crate::request::ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), json!(true));
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!("high"));
let rendered = render_request(&request);
assert!(rendered.contains("thinking_effort=high"));
assert!(rendered.ends_with("<|open|>think<|sep|>"));
}
#[test]
fn native_k3_kwargs_take_precedence() {
let mut request = crate::request::ChatRequest::for_test();
request.chat_options.template_kwargs.extend([
("thinking".to_string(), json!(true)),
("enable_thinking".to_string(), json!(false)),
("thinking_effort".to_string(), json!("low")),
("reasoning_effort".to_string(), json!("high")),
]);
let rendered = render_request(&request);
assert!(rendered.contains("thinking_effort=low"));
assert!(!rendered.contains("thinking_effort=high"));
}
#[test]
fn standard_none_disables_thinking() {
let mut request = crate::request::ChatRequest::for_test();
request.chat_options.template_kwargs.extend([
("enable_thinking".to_string(), json!(false)),
("reasoning_effort".to_string(), json!("none")),
]);
let rendered = render_request(&request);
assert!(!rendered.contains("type=\"thinking-effort\""));
assert!(rendered.ends_with("<|open|>response<|sep|>"));
}
#[test]
fn typed_none_disables_thinking() {
let mut request = crate::request::ChatRequest::for_test();
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
let rendered = render_request(&request);
assert!(!rendered.contains("type=\"thinking-effort\""));
assert!(rendered.ends_with("<|open|>response<|sep|>"));
}
#[test]
fn rejects_removed_medium_thinking_effort() {
let mut request = crate::request::ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("thinking_effort".to_string(), json!("medium"));
let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()))
.render(&request)
.unwrap_err();
expect![[r#"
ChatTemplate(
"unsupported thinking_effort=\"medium\"; supported values are `low`, `high`, and `max`",
)
"#]]
.assert_debug_eq(&error);
}
+2
View File
@@ -15,6 +15,7 @@ pub mod deepseek_v4;
pub mod harmony;
pub mod hf;
mod inkling;
mod kimi_k3;
mod selection;
#[cfg(test)]
mod test_utils;
@@ -23,6 +24,7 @@ pub use deepseek_v4::DeepSeekV4ChatRenderer;
pub use deepseek_v32::DeepSeekV32ChatRenderer;
pub use harmony::HarmonyChatRenderer;
pub use inkling::InklingChatRenderer;
pub use kimi_k3::KimiK3ChatRenderer;
pub use selection::RendererSelection;
/// Rendered chat prompt submitted to the text backend.
+8 -1
View File
@@ -26,6 +26,8 @@ pub enum RendererSelection {
Harmony,
/// Force the Inkling native token renderer.
Inkling,
/// Force the Kimi K3 XTML renderer.
KimiK3,
}
impl RendererSelection {
@@ -37,6 +39,7 @@ impl RendererSelection {
pub const HF_LITERAL: &str = "hf";
pub const INKLING_LITERAL: &str = "inkling";
pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model";
pub const KIMI_K3_LITERAL: &str = "kimi_k3";
/// Resolve the renderer selection using the given model type string, if
/// it's `Auto`.
@@ -47,6 +50,7 @@ impl RendererSelection {
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
Self::INKLING_MODEL_TYPE => Self::Inkling,
Self::KIMI_K3_LITERAL => Self::KimiK3,
_ => Self::Hf,
},
selection => selection,
@@ -70,6 +74,8 @@ impl FromStr for RendererSelection {
Ok(Self::Harmony)
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
Ok(Self::Inkling)
} else if value.eq_ignore_ascii_case(Self::KIMI_K3_LITERAL) {
Ok(Self::KimiK3)
} else {
Err(format!(
"unknown renderer `{value}` (expected one of: {})",
@@ -88,6 +94,7 @@ impl fmt::Display for RendererSelection {
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL),
}
}
}
@@ -114,7 +121,7 @@ mod tests {
fn renderer_selection_expected_error_message() {
let err = RendererSelection::from_str("unknown").unwrap_err();
expect_test::expect![
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)"
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)"
]
.assert_eq(&err);
}
+41 -11
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::fs;
use std::path::Path;
@@ -16,8 +17,9 @@ use crate::request::{
/// Options for constructing a [`ChatRequest`] from a fixture file.
#[derive(Debug, Clone, Copy)]
pub(crate) struct FixtureRequestOptions {
/// Whether to set the template kwarg `[enable_]thinking=true`.
pub enable_thinking: bool,
/// Optional thinking toggle applied only when the fixture does not already
/// set `thinking` / `enable_thinking` in [`FixtureRequest::template_kwargs`].
pub enable_thinking: Option<bool>,
/// Whether fixtures ending in an assistant message should omit the
/// trailing generation prompt.
pub no_generation_prompt_when_last_assistant: bool,
@@ -46,6 +48,15 @@ pub(crate) struct FixtureRequest {
messages: Vec<FixtureMessage>,
add_generation_prompt: Option<bool>,
reasoning_effort: Option<ReasoningEffort>,
/// Standard response format passed to model-specific renderers.
#[serde(default)]
response_format: Option<Value>,
/// Extra chat-template kwargs (thinking, preserve_thinking, …).
#[serde(default)]
template_kwargs: HashMap<String, Value>,
/// When omitted, defaults to `auto` if tools are present, otherwise `none`.
#[serde(default)]
tool_choice: Option<ChatToolChoice>,
}
impl FixtureFile {
@@ -57,6 +68,9 @@ impl FixtureFile {
messages,
add_generation_prompt: None,
reasoning_effort: None,
response_format: None,
template_kwargs: HashMap::new(),
tool_choice: None,
},
}
}
@@ -69,6 +83,7 @@ pub(crate) enum FixtureMessage {
content: FixtureContent,
},
Developer {
#[serde(default)]
content: FixtureContent,
#[serde(default)]
tools: Vec<FixtureTool>,
@@ -98,6 +113,12 @@ pub(crate) enum FixtureContent {
Parts(Vec<FixtureContentPart>),
}
impl Default for FixtureContent {
fn default() -> Self {
Self::Text(String::new())
}
}
#[derive(Debug, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub(crate) enum FixtureContentPart {
@@ -136,6 +157,13 @@ struct FixtureToolCallFunction {
impl FixtureRequest {
fn into_chat_request(self, options: FixtureRequestOptions) -> ChatRequest {
let tools = to_chat_tools(&self.tools);
let tool_choice = self.tool_choice.unwrap_or(if tools.is_empty() {
ChatToolChoice::None
} else {
ChatToolChoice::Auto
});
let mut request = ChatRequest {
request_id: "renderer-fixture".to_string(),
messages: self
@@ -144,12 +172,8 @@ impl FixtureRequest {
.enumerate()
.map(|(index, message)| fixture_message_to_chat_message(index, message))
.collect(),
tools: to_chat_tools(&self.tools),
tool_choice: if self.tools.is_empty() {
ChatToolChoice::None
} else {
ChatToolChoice::Auto
},
tools,
tool_choice,
..ChatRequest::for_test()
};
@@ -162,9 +186,15 @@ impl FixtureRequest {
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
}
request.chat_options.reasoning_effort = self.reasoning_effort;
if options.enable_thinking {
for key in ["thinking", "enable_thinking"] {
request.chat_options.template_kwargs.insert(key.to_string(), Value::Bool(true));
request.chat_options.response_format = self.response_format;
request.chat_options.template_kwargs.extend(self.template_kwargs);
// Options supply a default thinking toggle only when the fixture did not.
if let Some(thinking) = options.enable_thinking {
let kwargs = &mut request.chat_options.template_kwargs;
if !kwargs.contains_key("thinking") && !kwargs.contains_key("enable_thinking") {
kwargs.insert("thinking".to_string(), Value::Bool(thinking));
kwargs.insert("enable_thinking".to_string(), Value::Bool(thinking));
}
}
+5
View File
@@ -397,6 +397,10 @@ pub struct ChatOptions {
/// Effort level exposed to chat templates for reasoning models.
pub reasoning_effort: Option<ReasoningEffort>,
/// Standard response format available to model-specific renderers.
#[serde(default)]
pub response_format: Option<Value>,
/// Additional keyword arguments exposed to the chat template.
pub template_kwargs: HashMap<String, Value>,
}
@@ -407,6 +411,7 @@ impl Default for ChatOptions {
generation_prompt_mode: GenerationPromptMode::StartNewAssistant,
chat_template: None,
reasoning_effort: None,
response_format: None,
template_kwargs: HashMap::new(),
}
}
+35 -7
View File
@@ -212,6 +212,23 @@ impl RoundtripCase {
}
}
/// Kimi K3 XTML tool/reasoning channels (native renderer + unified parser).
///
/// Needs HF tokenizer files under `HF_HOME` (`tiktoken.model` +
/// `tokenizer_config.json`). Weights are not required for this text-level
/// roundtrip.
fn kimi_k3() -> Self {
Self {
model_id: "moonshotai/Kimi-K3",
assistant_stop_suffix: "<|end_of_msg|>",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
json_fmt: compact_json_fmt(),
sort_json_keys: false,
}
}
/// SeedOSS with `<seed:think>` / `</seed:think>` reasoning tags.
fn seed_oss() -> Self {
Self {
@@ -255,7 +272,7 @@ impl RoundtripCase {
fn gpt_oss() -> Self {
Self {
model_id: "openai/gpt-oss-20b",
assistant_stop_suffix: "", // not applicable for token-id cases
assistant_stop_suffix: "",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
thinking_behavior: ThinkingBehavior::Always { value: true },
@@ -268,7 +285,7 @@ impl RoundtripCase {
fn inkling() -> Self {
Self {
model_id: "thinkingmachines/Inkling",
assistant_stop_suffix: "",
assistant_stop_suffix: "<|content_model_end_sampling|>",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
thinking_behavior: ThinkingBehavior::Always { value: true },
@@ -312,6 +329,8 @@ roundtrip_tests! {
nemotron_v3 => [reasoning_and_content],
gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
// K3 drops plain-assistant reasoning in history; tool-call turns keep it.
kimi_k3 => [tool_call_mix],
gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
inkling => [reasoning_and_content, tool_call_mix],
}
@@ -670,14 +689,23 @@ fn decoded_completion_stream(
.collect()
}
Prompt::TokenIds(token_ids) => {
ensure!(
assistant_stop_suffix.is_empty(),
"token-id roundtrip cases do not support text stop suffixes"
);
let body = if assistant_stop_suffix.is_empty() {
token_ids.as_slice()
} else {
let stop_token_ids = tokenizer
.encode(assistant_stop_suffix, false)
.context("failed to encode token-id completion stop suffix")?;
token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
format!(
"token-id completion did not end with {:?}: {:?}",
assistant_stop_suffix, token_ids
)
})?
};
incremental_decode_chunks(
tokenizer,
&prompt_token_ids,
token_ids,
body,
TOKEN_COMPLETION_CHUNK_TOKENS,
)?
}