[Model] Add Kimi K3 support: Rust frontend [1/2] (#50104)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -20,7 +20,7 @@ use crate::output::{
|
||||
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
|
||||
use crate::renderer::{
|
||||
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
|
||||
InklingChatRenderer,
|
||||
InklingChatRenderer, KimiK3ChatRenderer,
|
||||
};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{DynChatOutputProcessor, RendererSelection};
|
||||
@@ -73,6 +73,7 @@ impl HfChatBackend {
|
||||
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
|
||||
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
|
||||
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
|
||||
RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
|
||||
};
|
||||
|
||||
info!(
|
||||
|
||||
@@ -33,7 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
|
||||
HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection,
|
||||
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
|
||||
RendererSelection,
|
||||
};
|
||||
pub use request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
|
||||
@@ -353,6 +354,12 @@ mod tests {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_parser_overrides_accepts_explicit_kimi_k3() {
|
||||
let selection = ParserSelection::Explicit("kimi_k3".to_string());
|
||||
validate_parser_overrides(&selection, &selection).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_parser_overrides_accepts_auto_and_none() {
|
||||
validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap();
|
||||
@@ -366,7 +373,7 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -377,6 +384,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ pub mod names {
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const KIMI: &str = "kimi";
|
||||
pub const KIMI_K2: &str = "kimi_k2";
|
||||
pub const KIMI_K3: &str = "kimi_k3";
|
||||
pub const MINIMAX_M2: &str = "minimax_m2";
|
||||
pub const MINIMAX_M3: &str = "minimax_m3";
|
||||
pub const NEMOTRON_V3: &str = "nemotron_v3";
|
||||
@@ -68,6 +69,7 @@ impl ReasoningParserFactory {
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
||||
.register_unified_dummy(names::KIMI_K3)
|
||||
.register_parser::<MiniMaxM2ReasoningParser>(names::MINIMAX_M2)
|
||||
.register_parser::<MiniMaxM3ReasoningParser>(names::MINIMAX_M3)
|
||||
.register_parser::<NemotronV3ReasoningParser>(names::NEMOTRON_V3)
|
||||
|
||||
@@ -33,6 +33,7 @@ pub mod names {
|
||||
// also routes to `Internlm2ToolParser` despite the version-agnostic name.
|
||||
pub const INTERNLM: &str = "internlm";
|
||||
pub const KIMI_K2: &str = "kimi_k2";
|
||||
pub const KIMI_K3: &str = "kimi_k3";
|
||||
pub const LLAMA3_JSON: &str = "llama3_json";
|
||||
pub const LLAMA4_JSON: &str = "llama4_json";
|
||||
pub const MINIMAX_M2: &str = "minimax_m2";
|
||||
@@ -78,6 +79,7 @@ impl ToolParserFactory {
|
||||
.register_parser::<HyV3ToolParser>(names::HY_V3)
|
||||
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
|
||||
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
|
||||
.register_unified_dummy(names::KIMI_K3)
|
||||
.register_parser::<Llama3JsonToolParser>(names::LLAMA3_JSON)
|
||||
.register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
|
||||
.register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser};
|
||||
pub use vllm_parser::unified::{
|
||||
Gemma4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, UnifiedParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
@@ -15,6 +17,7 @@ use crate::request::ChatTool;
|
||||
pub mod names {
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const INKLING: &str = "inkling";
|
||||
pub const KIMI_K3: &str = "kimi_k3";
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered unified parser implementation.
|
||||
@@ -39,11 +42,14 @@ impl UnifiedParserFactory {
|
||||
|
||||
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
|
||||
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
|
||||
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
|
||||
|
||||
factory
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
.register_pattern("gemma4", names::GEMMA4)
|
||||
.register_pattern("inkling", names::INKLING);
|
||||
.register_pattern("inkling", names::INKLING)
|
||||
.register_pattern("kimi-k3", names::KIMI_K3)
|
||||
.register_pattern("kimi_k3", names::KIMI_K3);
|
||||
|
||||
factory
|
||||
}
|
||||
@@ -121,4 +127,20 @@ mod tests {
|
||||
);
|
||||
factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_registers_kimi_k3() {
|
||||
let factory = UnifiedParserFactory::new();
|
||||
let tokenizer = TestTokenizer::new()
|
||||
.with_regular_token("<|open|>", 1001)
|
||||
.with_regular_token("<|close|>", 1002)
|
||||
.with_regular_token("<|sep|>", 1003);
|
||||
|
||||
assert!(factory.contains(names::KIMI_K3));
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("moonshotai/Kimi-K3"),
|
||||
Some(names::KIMI_K3)
|
||||
);
|
||||
factory.create(names::KIMI_K3, &[], Arc::new(tokenizer)).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
|
||||
|
||||
fn deepseek_fixture_options() -> FixtureRequestOptions {
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: true,
|
||||
enable_thinking: Some(true),
|
||||
no_generation_prompt_when_last_assistant: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
|
||||
|
||||
fn deepseek_fixture_options() -> FixtureRequestOptions {
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: true,
|
||||
enable_thinking: Some(true),
|
||||
no_generation_prompt_when_last_assistant: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
|
||||
fixture_chat_request(
|
||||
&fixture_path(input_name),
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: false,
|
||||
enable_thinking: Some(false),
|
||||
no_generation_prompt_when_last_assistant: false,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -105,7 +105,7 @@ fn fixture_request(name: &str) -> ChatRequest {
|
||||
|
||||
fn inkling_fixture_options() -> FixtureRequestOptions {
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: false,
|
||||
enable_thinking: Some(false),
|
||||
no_generation_prompt_when_last_assistant: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,608 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Kimi K3 XTML prompt renderer.
|
||||
//!
|
||||
//! Port of Moonshot remote-code `encoding_k3.py::build_chat_segments()`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
|
||||
};
|
||||
use crate::{AssistantContentBlock, AssistantToolCall};
|
||||
|
||||
pub(super) const OPEN: &str = "<|open|>";
|
||||
pub(super) const CLOSE: &str = "<|close|>";
|
||||
pub(super) const SEP: &str = "<|sep|>";
|
||||
pub(super) const END_OF_MSG: &str = "<|end_of_msg|>";
|
||||
pub(super) const IMAGE_PLACEHOLDER: &str = "<|media_pad|>";
|
||||
|
||||
const DEFAULT_THINKING_EFFORT: &str = "max";
|
||||
const VALID_THINKING_EFFORTS: &[&str] = &["low", "high", "max"];
|
||||
|
||||
/// K3 prompt encoder preserving Python's per-segment tokenization boundaries.
|
||||
pub(super) struct K3TokenWriter<'a> {
|
||||
tokenizer: &'a dyn Tokenizer,
|
||||
token_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
impl<'a> K3TokenWriter<'a> {
|
||||
pub(super) fn new(tokenizer: &'a dyn Tokenizer) -> Self {
|
||||
Self {
|
||||
tokenizer,
|
||||
token_ids: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode one trusted segment with normal added-token recognition.
|
||||
pub(super) fn control(&mut self, text: &str) -> Result<()> {
|
||||
if !text.is_empty() {
|
||||
self.token_ids.extend(self.tokenizer.encode(text, false)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encode one literal segment while bypassing every added-token matcher.
|
||||
pub(super) fn ordinary(&mut self, text: &str) -> Result<()> {
|
||||
if !text.is_empty() {
|
||||
self.token_ids.extend(self.tokenizer.encode_ordinary(text)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn finish(self) -> Vec<u32> {
|
||||
self.token_ids
|
||||
}
|
||||
}
|
||||
|
||||
/// Render and tokenize one chat request using K3's segment-aware contract.
|
||||
pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -> Result<Vec<u32>> {
|
||||
let thinking = thinking_enabled(request)?;
|
||||
let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?;
|
||||
let tools = request_tools(request);
|
||||
let mut out = K3TokenWriter::new(tokenizer);
|
||||
|
||||
if !tools.is_empty() {
|
||||
write_tool_declare(&mut out, tools, false)?;
|
||||
}
|
||||
|
||||
if let Some(effort) = thinking_effort {
|
||||
// Preserve the checkpoint's literal guidance text: it still names
|
||||
// `medium`, although the validator above no longer accepts it.
|
||||
write_internal_system(
|
||||
&mut out,
|
||||
"thinking-effort",
|
||||
&format!(
|
||||
"`thinking_effort` guides on how much to think in your \
|
||||
thinking channel (not including the response channel), \
|
||||
supported values include `low`, `medium`, `high`, and `max`.\n\
|
||||
Now the system is invoked with `thinking_effort={effort}`."
|
||||
),
|
||||
)?;
|
||||
}
|
||||
|
||||
// Track prior assistant tool-call ids for tool-result reordering / naming.
|
||||
let mut tool_call_id_index: HashMap<String, (usize, String)> = HashMap::new();
|
||||
let mut pending_tool_run: Vec<(usize, ChatMessage)> = Vec::new();
|
||||
|
||||
let flush_tool_run = |out: &mut K3TokenWriter<'_>,
|
||||
run: &mut Vec<(usize, ChatMessage)>,
|
||||
id_index: &HashMap<String, (usize, String)>|
|
||||
-> Result<()> {
|
||||
if run.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut resolved = Vec::with_capacity(run.len());
|
||||
let mut unresolved = false;
|
||||
for (offset, message) in run.drain(..) {
|
||||
let ChatMessage::ToolResponse {
|
||||
content,
|
||||
tool_call_id,
|
||||
} = message
|
||||
else {
|
||||
unreachable!("pending tool run only holds tool responses");
|
||||
};
|
||||
match id_index.get(&tool_call_id) {
|
||||
Some(&(position, ref name)) => {
|
||||
resolved.push((position, offset, content, Some(name.clone())));
|
||||
}
|
||||
None => {
|
||||
unresolved = true;
|
||||
resolved.push((usize::MAX, offset, content, None));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if unresolved {
|
||||
// Preserve original order when the run cannot be fully matched.
|
||||
resolved.sort_by_key(|item| item.1);
|
||||
} else {
|
||||
resolved.sort_by_key(|item| (item.0, item.1));
|
||||
}
|
||||
|
||||
for (xtml_index, (_, _, content, name)) in resolved.into_iter().enumerate() {
|
||||
let tool_name = name.as_deref().ok_or_else(|| {
|
||||
Error::ChatTemplate(
|
||||
"Kimi K3 tool messages need a resolvable tool name: \
|
||||
carry a matching tool_call_id against a preceding \
|
||||
assistant tool_call"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
write_tool_message(out, tool_name, xtml_index + 1, &content)?;
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
for (message_index, message) in request.messages.iter().enumerate() {
|
||||
match message {
|
||||
ChatMessage::ToolResponse { .. } => {
|
||||
pending_tool_run.push((message_index, message.clone()));
|
||||
continue;
|
||||
}
|
||||
_ => {
|
||||
flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
|
||||
}
|
||||
}
|
||||
|
||||
match message {
|
||||
// Python: role=system with a `tools` field renders a dynamic
|
||||
// tool-declare (`## New Tools Available`). Map that to Developer
|
||||
// messages that carry tools (OpenAI "developer" / system-tools).
|
||||
ChatMessage::Developer {
|
||||
content,
|
||||
tools: Some(local_tools),
|
||||
} if !local_tools.is_empty() => {
|
||||
write_tool_declare(&mut out, local_tools, true)?;
|
||||
if !content_is_empty(content) {
|
||||
write_role_message(&mut out, "system", None, content)?;
|
||||
}
|
||||
}
|
||||
ChatMessage::System { content } | ChatMessage::Developer { content, .. } => {
|
||||
write_role_message(&mut out, "system", None, content)?;
|
||||
}
|
||||
ChatMessage::User { content } => {
|
||||
write_role_message(&mut out, "user", None, content)?;
|
||||
}
|
||||
ChatMessage::Assistant { content } => {
|
||||
tool_call_id_index.clear();
|
||||
let mut call_position = 0usize;
|
||||
for block in content {
|
||||
if let AssistantContentBlock::ToolCall(call) = block {
|
||||
call_position += 1;
|
||||
if !call.id.is_empty() {
|
||||
tool_call_id_index
|
||||
.entry(call.id.clone())
|
||||
.or_insert((call_position, call.name.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
write_assistant_message(&mut out, content, thinking)?;
|
||||
}
|
||||
ChatMessage::ToolResponse { .. } => unreachable!("handled above"),
|
||||
}
|
||||
}
|
||||
flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
|
||||
|
||||
match &request.tool_choice {
|
||||
ChatToolChoice::Required => {
|
||||
write_internal_system(
|
||||
&mut out,
|
||||
"tool-choice",
|
||||
"The system is invoked with `tool_choice=required`.\n\
|
||||
You MUST call tools in the next message.",
|
||||
)?;
|
||||
}
|
||||
// Emit only when tools are present: Rust defaults tool_choice to None
|
||||
// for tool-free requests, which must not inject a tool-choice message.
|
||||
ChatToolChoice::None if !request.tools.is_empty() => {
|
||||
write_internal_system(
|
||||
&mut out,
|
||||
"tool-choice",
|
||||
"The system is invoked with `tool_choice=none`.\n\
|
||||
You MUST NOT call any tools in the next message.",
|
||||
)?;
|
||||
}
|
||||
ChatToolChoice::None | ChatToolChoice::Auto | ChatToolChoice::Function { .. } => {}
|
||||
}
|
||||
|
||||
write_response_format(&mut out, request)?;
|
||||
|
||||
if request.chat_options.add_generation_prompt() {
|
||||
write_open_tag(&mut out, "message", &[("role", "assistant")])?;
|
||||
write_open_tag(&mut out, if thinking { "think" } else { "response" }, &[])?;
|
||||
}
|
||||
|
||||
Ok(out.finish())
|
||||
}
|
||||
|
||||
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
|
||||
// Declare tools whenever the request carries them. K3 tool-declare is
|
||||
// independent of tool_choice; tool_choice only injects control messages.
|
||||
request.tools.as_slice()
|
||||
}
|
||||
|
||||
fn thinking_enabled(request: &ChatRequest) -> Result<bool> {
|
||||
if let Some(thinking) = request.parse_template_bool("thinking")? {
|
||||
return Ok(thinking);
|
||||
}
|
||||
if let Some(enable_thinking) = request.parse_template_bool("enable_thinking")? {
|
||||
return Ok(enable_thinking);
|
||||
}
|
||||
Ok(request
|
||||
.chat_options
|
||||
.reasoning_effort
|
||||
.map(|effort| effort != crate::request::ReasoningEffort::None)
|
||||
.unwrap_or(true))
|
||||
}
|
||||
|
||||
fn thinking_effort(request: &ChatRequest) -> Result<String> {
|
||||
let effort = if let Some(value) = request.chat_options.template_kwargs.get("thinking_effort") {
|
||||
value.as_str().ok_or_else(|| {
|
||||
Error::ChatTemplate(format!(
|
||||
"template kwarg `thinking_effort` must be a string, got {value}"
|
||||
))
|
||||
})?
|
||||
} else if let Some(effort) = request.chat_options.reasoning_effort {
|
||||
effort.as_str()
|
||||
} else if let Some(value) = request.chat_options.template_kwargs.get("reasoning_effort") {
|
||||
value.as_str().ok_or_else(|| {
|
||||
Error::ChatTemplate(format!(
|
||||
"template kwarg `reasoning_effort` must be a string, got {value}"
|
||||
))
|
||||
})?
|
||||
} else {
|
||||
DEFAULT_THINKING_EFFORT
|
||||
};
|
||||
|
||||
if !VALID_THINKING_EFFORTS.contains(&effort) {
|
||||
return Err(Error::ChatTemplate(format!(
|
||||
"unsupported thinking_effort={effort:?}; supported values are `low`, `high`, and `max`"
|
||||
)));
|
||||
}
|
||||
Ok(effort.to_string())
|
||||
}
|
||||
|
||||
fn content_is_empty(content: &ChatContent) -> bool {
|
||||
match content {
|
||||
ChatContent::Text(text) => text.is_empty(),
|
||||
ChatContent::Parts(parts) => parts.iter().all(|part| match part {
|
||||
ChatContentPart::Text { text } => text.is_empty(),
|
||||
ChatContentPart::ImageUrl { .. }
|
||||
| ChatContentPart::VideoUrl { .. }
|
||||
| ChatContentPart::InputAudio { .. }
|
||||
| ChatContentPart::AudioUrl { .. } => false,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_tool_declare(
|
||||
out: &mut K3TokenWriter<'_>,
|
||||
tools: &[ChatTool],
|
||||
dynamic: bool,
|
||||
) -> Result<()> {
|
||||
let mut specs = Vec::with_capacity(tools.len());
|
||||
for tool in tools {
|
||||
let mut function = Map::new();
|
||||
function.insert(
|
||||
"description".to_string(),
|
||||
Value::String(tool.description.clone().unwrap_or_default()),
|
||||
);
|
||||
function.insert("name".to_string(), Value::String(tool.name.clone()));
|
||||
function.insert("parameters".to_string(), sort_json(&tool.parameters));
|
||||
specs.push(json!({
|
||||
"function": Value::Object(function),
|
||||
"type": "function",
|
||||
}));
|
||||
}
|
||||
let payload = compact_json(&sort_json(&Value::Array(specs)))?;
|
||||
|
||||
let body = if dynamic {
|
||||
format!(
|
||||
"## New Tools Available\n\
|
||||
The system dynamically extends the toolset via lazy-loading.\n\
|
||||
You have access to all existing and extended tools.\n\
|
||||
Here are the specs for the extended tools.\n\n\
|
||||
```json\n\
|
||||
{payload}\n\
|
||||
```"
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"# Tools\n\
|
||||
Here are the available tools, described in JSONSchema.\n\n\
|
||||
```json\n\
|
||||
{payload}\n\
|
||||
```"
|
||||
)
|
||||
};
|
||||
|
||||
write_internal_system(out, "tool-declare", &body)
|
||||
}
|
||||
|
||||
fn write_internal_system(
|
||||
out: &mut K3TokenWriter<'_>,
|
||||
message_type: &str,
|
||||
body: &str,
|
||||
) -> Result<()> {
|
||||
write_open_tag(
|
||||
out,
|
||||
"message",
|
||||
&[("role", "system"), ("type", message_type)],
|
||||
)?;
|
||||
out.ordinary(body.trim())?;
|
||||
write_close_tag(out, "message")?;
|
||||
out.control(END_OF_MSG)
|
||||
}
|
||||
|
||||
fn write_role_message(
|
||||
out: &mut K3TokenWriter<'_>,
|
||||
role: &str,
|
||||
name: Option<&str>,
|
||||
content: &ChatContent,
|
||||
) -> Result<()> {
|
||||
let mut attrs = vec![("role", role.to_string())];
|
||||
if let Some(name) = name.filter(|name| !name.is_empty()) {
|
||||
attrs.push(("name", name.to_string()));
|
||||
}
|
||||
let attr_refs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect();
|
||||
write_open_tag(out, "message", &attr_refs)?;
|
||||
write_content(out, content)?;
|
||||
write_close_tag(out, "message")?;
|
||||
out.control(END_OF_MSG)
|
||||
}
|
||||
|
||||
fn write_tool_message(
|
||||
out: &mut K3TokenWriter<'_>,
|
||||
tool_name: &str,
|
||||
index: usize,
|
||||
content: &ChatContent,
|
||||
) -> Result<()> {
|
||||
let index_str = index.to_string();
|
||||
write_open_tag(
|
||||
out,
|
||||
"message",
|
||||
&[("role", "tool"), ("tool", tool_name), ("index", &index_str)],
|
||||
)?;
|
||||
write_content(out, content)?;
|
||||
write_close_tag(out, "message")?;
|
||||
out.control(END_OF_MSG)
|
||||
}
|
||||
|
||||
fn write_assistant_message(
|
||||
out: &mut K3TokenWriter<'_>,
|
||||
content: &[AssistantContentBlock],
|
||||
thinking: bool,
|
||||
) -> Result<()> {
|
||||
write_open_tag(out, "message", &[("role", "assistant")])?;
|
||||
|
||||
let mut reasoning = String::new();
|
||||
let mut response = String::new();
|
||||
let mut tool_calls = Vec::new();
|
||||
for block in content {
|
||||
match block {
|
||||
AssistantContentBlock::Reasoning { text } => reasoning.push_str(text),
|
||||
AssistantContentBlock::Text { text } => response.push_str(text),
|
||||
AssistantContentBlock::ToolCall(call) => tool_calls.push(call),
|
||||
}
|
||||
}
|
||||
|
||||
// The think channel is structural: in thinking mode every assistant
|
||||
// message carries open/close tags even when there is no reasoning content.
|
||||
// In non-thinking mode the channel is dropped entirely.
|
||||
if thinking {
|
||||
write_open_tag(out, "think", &[])?;
|
||||
if !reasoning.trim().is_empty() {
|
||||
out.ordinary(&reasoning)?;
|
||||
}
|
||||
write_close_tag(out, "think")?;
|
||||
}
|
||||
|
||||
write_open_tag(out, "response", &[])?;
|
||||
out.ordinary(&response)?;
|
||||
write_close_tag(out, "response")?;
|
||||
|
||||
if !tool_calls.is_empty() {
|
||||
write_open_tag(out, "tools", &[])?;
|
||||
for (index, tool_call) in tool_calls.into_iter().enumerate() {
|
||||
write_assistant_tool_call(out, tool_call, index + 1)?;
|
||||
}
|
||||
write_close_tag(out, "tools")?;
|
||||
}
|
||||
|
||||
write_close_tag(out, "message")?;
|
||||
out.control(END_OF_MSG)
|
||||
}
|
||||
|
||||
fn write_assistant_tool_call(
|
||||
out: &mut K3TokenWriter<'_>,
|
||||
tool_call: &AssistantToolCall,
|
||||
index: usize,
|
||||
) -> Result<()> {
|
||||
let index_str = index.to_string();
|
||||
write_open_tag(
|
||||
out,
|
||||
"call",
|
||||
&[
|
||||
("tool", tool_call.name.as_str()),
|
||||
("index", index_str.as_str()),
|
||||
],
|
||||
)?;
|
||||
|
||||
let (args, json_block) = normalize_tool_arguments(&tool_call.arguments)?;
|
||||
if let Some(raw) = json_block {
|
||||
write_open_tag(out, "json", &[("type", "object")])?;
|
||||
out.ordinary(&raw)?;
|
||||
write_close_tag(out, "json")?;
|
||||
} else {
|
||||
for (key, value) in args {
|
||||
let typ = xtml_type(&value);
|
||||
write_open_tag(out, "argument", &[("key", key.as_str()), ("type", typ)])?;
|
||||
out.ordinary(&xtml_value(&value))?;
|
||||
write_close_tag(out, "argument")?;
|
||||
}
|
||||
}
|
||||
|
||||
write_close_tag(out, "call")
|
||||
}
|
||||
|
||||
fn write_content(out: &mut K3TokenWriter<'_>, content: &ChatContent) -> Result<()> {
|
||||
match content {
|
||||
ChatContent::Text(text) => write_text_with_images(out, text),
|
||||
ChatContent::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
ChatContentPart::Text { text } => write_text_with_images(out, text)?,
|
||||
ChatContentPart::ImageUrl { .. } => out.control(IMAGE_PLACEHOLDER)?,
|
||||
ChatContentPart::VideoUrl { .. } => {
|
||||
return Err(Error::UnsupportedMultimodalContent("video_url"));
|
||||
}
|
||||
ChatContentPart::InputAudio { .. } => {
|
||||
return Err(Error::UnsupportedMultimodalContent("input_audio"));
|
||||
}
|
||||
ChatContentPart::AudioUrl { .. } => {
|
||||
return Err(Error::UnsupportedMultimodalContent("audio_url"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_text_with_images(out: &mut K3TokenWriter<'_>, text: &str) -> Result<()> {
|
||||
// Placeholder expansion is left as the literal K3 image token; multimodal
|
||||
// preprocessing can replace it once image prompts are known.
|
||||
out.ordinary(text)
|
||||
}
|
||||
|
||||
fn write_response_format(out: &mut K3TokenWriter<'_>, request: &ChatRequest) -> Result<()> {
|
||||
let Some(rf) = request.chat_options.response_format.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let rf_type = rf.get("type").and_then(Value::as_str).or_else(|| rf.as_str()).unwrap_or("");
|
||||
|
||||
match rf_type {
|
||||
"json_object" => {
|
||||
write_internal_system(
|
||||
out,
|
||||
"response-format",
|
||||
"The system is invoked with `response_format=json_object`.\n\
|
||||
Your response must be raw JSON data without markdown code \
|
||||
blocks (```json) or any additional formatting.",
|
||||
)?;
|
||||
}
|
||||
"json_schema" => {
|
||||
let schema = extract_response_schema(rf);
|
||||
let schema_json = compact_json(&sort_json(&schema.unwrap_or(Value::Null)))?;
|
||||
write_internal_system(
|
||||
out,
|
||||
"response-format",
|
||||
&format!(
|
||||
"The system is invoked with `response_format=json_schema`.\n\
|
||||
Your response must be raw JSON data without markdown code \
|
||||
blocks (```json) or any additional formatting.\n\
|
||||
The JSON data must match the following schema:\n\
|
||||
```json\n\
|
||||
{schema_json}\n\
|
||||
```"
|
||||
),
|
||||
)?;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_response_schema(response_format: &Value) -> Option<Value> {
|
||||
let json_schema = response_format.get("json_schema")?;
|
||||
if let Some(schema) = json_schema.get("schema") {
|
||||
return Some(schema.clone());
|
||||
}
|
||||
if let Some(schema) = json_schema.get("json_schema") {
|
||||
return Some(schema.clone());
|
||||
}
|
||||
Some(json_schema.clone())
|
||||
}
|
||||
|
||||
fn normalize_tool_arguments(arguments: &str) -> Result<(Map<String, Value>, Option<String>)> {
|
||||
let trimmed = arguments.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Ok((Map::new(), None));
|
||||
}
|
||||
match serde_json::from_str::<Value>(trimmed) {
|
||||
Ok(Value::Object(map)) => Ok((map, None)),
|
||||
Ok(_) => Err(Error::ChatTemplate(
|
||||
"Kimi K3 tool call arguments must be a JSON object".to_string(),
|
||||
)),
|
||||
Err(_) => Ok((Map::new(), Some(arguments.to_string()))),
|
||||
}
|
||||
}
|
||||
|
||||
fn xtml_type(value: &Value) -> &'static str {
|
||||
match value {
|
||||
Value::Bool(_) => "boolean",
|
||||
Value::Null => "null",
|
||||
Value::Number(_) => "number",
|
||||
Value::String(_) => "string",
|
||||
Value::Object(_) => "object",
|
||||
Value::Array(_) => "array",
|
||||
}
|
||||
}
|
||||
|
||||
fn xtml_value(value: &Value) -> String {
|
||||
match value {
|
||||
Value::String(text) => text.clone(),
|
||||
other => compact_json(other).unwrap_or_else(|_| other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_open_tag(out: &mut K3TokenWriter<'_>, tag: &str, attrs: &[(&str, &str)]) -> Result<()> {
|
||||
out.control(OPEN)?;
|
||||
out.ordinary(tag)?;
|
||||
for (key, value) in attrs {
|
||||
out.ordinary(&format!(" {key}"))?;
|
||||
out.ordinary("=\"")?;
|
||||
out.ordinary(&escape_attr_value(value))?;
|
||||
out.ordinary("\"")?;
|
||||
}
|
||||
out.control(SEP)
|
||||
}
|
||||
|
||||
fn write_close_tag(out: &mut K3TokenWriter<'_>, tag: &str) -> Result<()> {
|
||||
out.control(CLOSE)?;
|
||||
out.ordinary(tag)?;
|
||||
out.control(SEP)
|
||||
}
|
||||
|
||||
fn escape_attr_value(value: &str) -> String {
|
||||
value.replace('&', "&").replace('"', """)
|
||||
}
|
||||
|
||||
fn compact_json(value: &Value) -> Result<String> {
|
||||
serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.to_string()))
|
||||
}
|
||||
|
||||
fn sort_json(value: &Value) -> Value {
|
||||
match value {
|
||||
Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()),
|
||||
Value::Object(map) => {
|
||||
let mut sorted = Map::new();
|
||||
let mut keys = map.keys().collect::<Vec<_>>();
|
||||
keys.sort();
|
||||
for key in keys {
|
||||
sorted.insert(key.clone(), sort_json(&map[key]));
|
||||
}
|
||||
Value::Object(sorted)
|
||||
}
|
||||
_ => value.clone(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "json pls"
|
||||
}
|
||||
],
|
||||
"add_generation_prompt": true,
|
||||
"response_format": {
|
||||
"type": "json_object"
|
||||
},
|
||||
"template_kwargs": {
|
||||
"thinking": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
<|open|>message role="user"<|sep|>json pls<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="response-format"<|sep|>The system is invoked with `response_format=json_object`.
|
||||
Your response must be raw JSON data without markdown code blocks (```json) or any additional formatting.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>response<|sep|>
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string"
|
||||
},
|
||||
"days": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"city"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "hi"
|
||||
},
|
||||
{
|
||||
"role": "developer",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"description": "calculator",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"x": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"x"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "use calc"
|
||||
}
|
||||
],
|
||||
"add_generation_prompt": true,
|
||||
"template_kwargs": {
|
||||
"thinking": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<|open|>message role="system" type="tool-declare"<|sep|># Tools
|
||||
Here are the available tools, described in JSONSchema.
|
||||
|
||||
```json
|
||||
[{"function":{"description":"weather","name":"get_weather","parameters":{"properties":{"city":{"type":"string"},"days":{"type":"number"}},"required":["city"],"type":"object"}},"type":"function"}]
|
||||
```<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
|
||||
Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>hi<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-declare"<|sep|>## New Tools Available
|
||||
The system dynamically extends the toolset via lazy-loading.
|
||||
You have access to all existing and extended tools.
|
||||
Here are the specs for the extended tools.
|
||||
|
||||
```json
|
||||
[{"function":{"description":"calculator","name":"calc","parameters":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}},"type":"function"}]
|
||||
```<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>use calc<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "see this"
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": "https://example.com/a.png"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "old answer",
|
||||
"reasoning_content": "old reasoning"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "continue"
|
||||
}
|
||||
],
|
||||
"add_generation_prompt": true,
|
||||
"template_kwargs": {
|
||||
"thinking": true,
|
||||
"thinking_effort": "high"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
|
||||
Now the system is invoked with `thinking_effort=high`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>see this<|media_pad|><|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>old reasoning<|close|>think<|sep|><|open|>response<|sep|>old answer<|close|>response<|sep|><|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>continue<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string"
|
||||
},
|
||||
"days": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"city"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"description": "calculator",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"x": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"x"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hangzhou weather and calc"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "I'll check.",
|
||||
"reasoning_content": "Need tools.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "get_weather:0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"city\":\"Hangzhou\",\"days\":1}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "calc:1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"arguments": "{\"x\":2}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "calc:1",
|
||||
"content": "2"
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "get_weather:0",
|
||||
"content": "{\"city\":\"Hangzhou\",\"condition\":\"rain\"}"
|
||||
}
|
||||
],
|
||||
"add_generation_prompt": true,
|
||||
"tool_choice": "required",
|
||||
"template_kwargs": {
|
||||
"thinking": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
<|open|>message role="system" type="tool-declare"<|sep|># Tools
|
||||
Here are the available tools, described in JSONSchema.
|
||||
|
||||
```json
|
||||
[{"function":{"description":"weather","name":"get_weather","parameters":{"properties":{"city":{"type":"string"},"days":{"type":"number"}},"required":["city"],"type":"object"}},"type":"function"},{"function":{"description":"calculator","name":"calc","parameters":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}},"type":"function"}]
|
||||
```<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
|
||||
Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>Hangzhou weather and calc<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>Need tools.<|close|>think<|sep|><|open|>response<|sep|>I'll check.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool="get_weather" index="1"<|sep|><|open|>argument key="city" type="string"<|sep|>Hangzhou<|close|>argument<|sep|><|open|>argument key="days" type="number"<|sep|>1<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool="calc" index="2"<|sep|><|open|>argument key="x" type="number"<|sep|>2<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|><|end_of_msg|><|open|>message role="tool" tool="get_weather" index="1"<|sep|>{"city":"Hangzhou","condition":"rain"}<|close|>message<|sep|><|end_of_msg|><|open|>message role="tool" tool="calc" index="2"<|sep|>2<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-choice"<|sep|>The system is invoked with `tool_choice=required`.
|
||||
You MUST call tools in the next message.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
|
||||
@@ -0,0 +1,39 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Native Kimi K3 XTML chat renderer.
|
||||
|
||||
mod encoding;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
/// Dedicated Kimi K3 XTML renderer.
|
||||
#[derive(Clone)]
|
||||
pub struct KimiK3ChatRenderer {
|
||||
tokenizer: DynTokenizer,
|
||||
}
|
||||
|
||||
impl KimiK3ChatRenderer {
|
||||
/// Create a Kimi K3 renderer.
|
||||
pub fn new(tokenizer: DynTokenizer) -> Self {
|
||||
Self { tokenizer }
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for KimiK3ChatRenderer {
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
request.validate()?;
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::TokenIds(encoding::render_request(request, self.tokenizer.as_ref())?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Golden fixtures generated from HF remote-code `encoding_k3.py`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use expect_test::{expect, expect_file};
|
||||
use serde_json::json;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::KimiK3ChatRenderer;
|
||||
use crate::AssistantContentBlock;
|
||||
use crate::ChatRenderer;
|
||||
use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
|
||||
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
|
||||
use crate::request::{ChatContentPart, ChatMessage, GenerationPromptMode, ReasoningEffort};
|
||||
|
||||
const OPEN_ID: u32 = 256;
|
||||
const CLOSE_ID: u32 = 257;
|
||||
const SEP_ID: u32 = 258;
|
||||
const END_OF_MSG_ID: u32 = 259;
|
||||
const MEDIA_ID: u32 = 260;
|
||||
|
||||
fn test_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_special_token(OPEN, OPEN_ID)
|
||||
.with_special_token(CLOSE, CLOSE_ID)
|
||||
.with_special_token(SEP, SEP_ID)
|
||||
.with_special_token(END_OF_MSG, END_OF_MSG_ID)
|
||||
.with_special_token(IMAGE_PLACEHOLDER, MEDIA_ID)
|
||||
}
|
||||
|
||||
fn render_token_ids(request: &crate::request::ChatRequest, tokenizer: DynTokenizer) -> Vec<u32> {
|
||||
let prompt = KimiK3ChatRenderer::new(tokenizer).render(request).unwrap().prompt;
|
||||
let Prompt::TokenIds(token_ids) = prompt else {
|
||||
panic!("kimi k3 renderer should return token IDs")
|
||||
};
|
||||
token_ids
|
||||
}
|
||||
|
||||
fn render_request(request: &crate::request::ChatRequest) -> String {
|
||||
let tokenizer: DynTokenizer = Arc::new(test_tokenizer());
|
||||
let token_ids = render_token_ids(request, tokenizer.clone());
|
||||
tokenizer.decode(&token_ids, false).unwrap()
|
||||
}
|
||||
|
||||
fn fixture_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("src/renderer/kimi_k3/fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
fn kimi_k3_fixture_options() -> FixtureRequestOptions {
|
||||
FixtureRequestOptions {
|
||||
// Fixture JSON owns thinking via `template_kwargs`.
|
||||
enable_thinking: None,
|
||||
no_generation_prompt_when_last_assistant: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_golden(name: &str) {
|
||||
let input_name = format!("{name}_input.json");
|
||||
let request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options());
|
||||
let rendered = render_request(&request);
|
||||
expect_file![format!("fixtures/{name}_output.txt")].assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn golden_history_preserve_and_image() {
|
||||
assert_golden("history_preserve_and_image");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn golden_tools_history_and_required() {
|
||||
assert_golden("tools_history_and_required");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn golden_controls_thinking_off() {
|
||||
assert_golden("controls_thinking_off");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn golden_dynamic_system_tool_declare() {
|
||||
assert_golden("dynamic_system_tool_declare");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_writer_protects_literal_control_and_media_markers() {
|
||||
let tokenizer = Arc::new(test_tokenizer());
|
||||
let user_text = format!("literal {OPEN} and {}", super::encoding::IMAGE_PLACEHOLDER);
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request.messages = vec![ChatMessage::user(vec![
|
||||
ChatContentPart::text(user_text),
|
||||
ChatContentPart::image_url("data:image/png;base64,test"),
|
||||
])];
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), json!(false));
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
|
||||
let token_ids = render_token_ids(&request, tokenizer.clone());
|
||||
|
||||
assert_eq!(
|
||||
token_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
token_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
|
||||
1
|
||||
);
|
||||
|
||||
let flattened = tokenizer.decode(&token_ids, false).unwrap();
|
||||
let flattened_ids = tokenizer.encode(&flattened, false).unwrap();
|
||||
assert_eq!(
|
||||
flattened_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
flattened_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thinking_history_renders_empty_think_channel() {
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request.messages = vec![
|
||||
ChatMessage::user("question"),
|
||||
ChatMessage::assistant_text("answer"),
|
||||
ChatMessage::user("follow-up"),
|
||||
];
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(rendered.contains(
|
||||
"<|open|>message role=\"assistant\"<|sep|>\
|
||||
<|open|>think<|sep|><|close|>think<|sep|>\
|
||||
<|open|>response<|sep|>answer<|close|>response<|sep|>"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_thinking_history_omits_reasoning_channel() {
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request.messages = vec![
|
||||
ChatMessage::user("question"),
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "hidden reasoning".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "answer".to_string(),
|
||||
},
|
||||
]),
|
||||
ChatMessage::user("follow-up"),
|
||||
];
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".to_string(), json!(false));
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(!rendered.contains("hidden reasoning"));
|
||||
assert!(!rendered.contains("<|open|>think<|sep|>"));
|
||||
assert!(rendered.contains(
|
||||
"<|open|>message role=\"assistant\"<|sep|>\
|
||||
<|open|>response<|sep|>answer<|close|>response<|sep|>"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_thinking_effort_to_max() {
|
||||
let rendered = render_request(&crate::request::ChatRequest::for_test());
|
||||
|
||||
expect![[r#"<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
|
||||
Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>test<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>"#]]
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn translates_standard_thinking_kwargs() {
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), json!(true));
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("reasoning_effort".to_string(), json!("high"));
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(rendered.contains("thinking_effort=high"));
|
||||
assert!(rendered.ends_with("<|open|>think<|sep|>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_k3_kwargs_take_precedence() {
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request.chat_options.template_kwargs.extend([
|
||||
("thinking".to_string(), json!(true)),
|
||||
("enable_thinking".to_string(), json!(false)),
|
||||
("thinking_effort".to_string(), json!("low")),
|
||||
("reasoning_effort".to_string(), json!("high")),
|
||||
]);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(rendered.contains("thinking_effort=low"));
|
||||
assert!(!rendered.contains("thinking_effort=high"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_none_disables_thinking() {
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request.chat_options.template_kwargs.extend([
|
||||
("enable_thinking".to_string(), json!(false)),
|
||||
("reasoning_effort".to_string(), json!("none")),
|
||||
]);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(!rendered.contains("type=\"thinking-effort\""));
|
||||
assert!(rendered.ends_with("<|open|>response<|sep|>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typed_none_disables_thinking() {
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
|
||||
|
||||
let rendered = render_request(&request);
|
||||
|
||||
assert!(!rendered.contains("type=\"thinking-effort\""));
|
||||
assert!(rendered.ends_with("<|open|>response<|sep|>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_removed_medium_thinking_effort() {
|
||||
let mut request = crate::request::ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking_effort".to_string(), json!("medium"));
|
||||
|
||||
let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()))
|
||||
.render(&request)
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
ChatTemplate(
|
||||
"unsupported thinking_effort=\"medium\"; supported values are `low`, `high`, and `max`",
|
||||
)
|
||||
"#]]
|
||||
.assert_debug_eq(&error);
|
||||
}
|
||||
@@ -15,6 +15,7 @@ pub mod deepseek_v4;
|
||||
pub mod harmony;
|
||||
pub mod hf;
|
||||
mod inkling;
|
||||
mod kimi_k3;
|
||||
mod selection;
|
||||
#[cfg(test)]
|
||||
mod test_utils;
|
||||
@@ -23,6 +24,7 @@ pub use deepseek_v4::DeepSeekV4ChatRenderer;
|
||||
pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use harmony::HarmonyChatRenderer;
|
||||
pub use inkling::InklingChatRenderer;
|
||||
pub use kimi_k3::KimiK3ChatRenderer;
|
||||
pub use selection::RendererSelection;
|
||||
|
||||
/// Rendered chat prompt submitted to the text backend.
|
||||
|
||||
@@ -26,6 +26,8 @@ pub enum RendererSelection {
|
||||
Harmony,
|
||||
/// Force the Inkling native token renderer.
|
||||
Inkling,
|
||||
/// Force the Kimi K3 XTML renderer.
|
||||
KimiK3,
|
||||
}
|
||||
|
||||
impl RendererSelection {
|
||||
@@ -37,6 +39,7 @@ impl RendererSelection {
|
||||
pub const HF_LITERAL: &str = "hf";
|
||||
pub const INKLING_LITERAL: &str = "inkling";
|
||||
pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model";
|
||||
pub const KIMI_K3_LITERAL: &str = "kimi_k3";
|
||||
|
||||
/// Resolve the renderer selection using the given model type string, if
|
||||
/// it's `Auto`.
|
||||
@@ -47,6 +50,7 @@ impl RendererSelection {
|
||||
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
|
||||
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
|
||||
Self::INKLING_MODEL_TYPE => Self::Inkling,
|
||||
Self::KIMI_K3_LITERAL => Self::KimiK3,
|
||||
_ => Self::Hf,
|
||||
},
|
||||
selection => selection,
|
||||
@@ -70,6 +74,8 @@ impl FromStr for RendererSelection {
|
||||
Ok(Self::Harmony)
|
||||
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
|
||||
Ok(Self::Inkling)
|
||||
} else if value.eq_ignore_ascii_case(Self::KIMI_K3_LITERAL) {
|
||||
Ok(Self::KimiK3)
|
||||
} else {
|
||||
Err(format!(
|
||||
"unknown renderer `{value}` (expected one of: {})",
|
||||
@@ -88,6 +94,7 @@ impl fmt::Display for RendererSelection {
|
||||
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
|
||||
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
|
||||
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
|
||||
Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,7 +121,7 @@ mod tests {
|
||||
fn renderer_selection_expected_error_message() {
|
||||
let err = RendererSelection::from_str("unknown").unwrap_err();
|
||||
expect_test::expect![
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)"
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)"
|
||||
]
|
||||
.assert_eq(&err);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -16,8 +17,9 @@ use crate::request::{
|
||||
/// Options for constructing a [`ChatRequest`] from a fixture file.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct FixtureRequestOptions {
|
||||
/// Whether to set the template kwarg `[enable_]thinking=true`.
|
||||
pub enable_thinking: bool,
|
||||
/// Optional thinking toggle applied only when the fixture does not already
|
||||
/// set `thinking` / `enable_thinking` in [`FixtureRequest::template_kwargs`].
|
||||
pub enable_thinking: Option<bool>,
|
||||
/// Whether fixtures ending in an assistant message should omit the
|
||||
/// trailing generation prompt.
|
||||
pub no_generation_prompt_when_last_assistant: bool,
|
||||
@@ -46,6 +48,15 @@ pub(crate) struct FixtureRequest {
|
||||
messages: Vec<FixtureMessage>,
|
||||
add_generation_prompt: Option<bool>,
|
||||
reasoning_effort: Option<ReasoningEffort>,
|
||||
/// Standard response format passed to model-specific renderers.
|
||||
#[serde(default)]
|
||||
response_format: Option<Value>,
|
||||
/// Extra chat-template kwargs (thinking, preserve_thinking, …).
|
||||
#[serde(default)]
|
||||
template_kwargs: HashMap<String, Value>,
|
||||
/// When omitted, defaults to `auto` if tools are present, otherwise `none`.
|
||||
#[serde(default)]
|
||||
tool_choice: Option<ChatToolChoice>,
|
||||
}
|
||||
|
||||
impl FixtureFile {
|
||||
@@ -57,6 +68,9 @@ impl FixtureFile {
|
||||
messages,
|
||||
add_generation_prompt: None,
|
||||
reasoning_effort: None,
|
||||
response_format: None,
|
||||
template_kwargs: HashMap::new(),
|
||||
tool_choice: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -69,6 +83,7 @@ pub(crate) enum FixtureMessage {
|
||||
content: FixtureContent,
|
||||
},
|
||||
Developer {
|
||||
#[serde(default)]
|
||||
content: FixtureContent,
|
||||
#[serde(default)]
|
||||
tools: Vec<FixtureTool>,
|
||||
@@ -98,6 +113,12 @@ pub(crate) enum FixtureContent {
|
||||
Parts(Vec<FixtureContentPart>),
|
||||
}
|
||||
|
||||
impl Default for FixtureContent {
|
||||
fn default() -> Self {
|
||||
Self::Text(String::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub(crate) enum FixtureContentPart {
|
||||
@@ -136,6 +157,13 @@ struct FixtureToolCallFunction {
|
||||
|
||||
impl FixtureRequest {
|
||||
fn into_chat_request(self, options: FixtureRequestOptions) -> ChatRequest {
|
||||
let tools = to_chat_tools(&self.tools);
|
||||
let tool_choice = self.tool_choice.unwrap_or(if tools.is_empty() {
|
||||
ChatToolChoice::None
|
||||
} else {
|
||||
ChatToolChoice::Auto
|
||||
});
|
||||
|
||||
let mut request = ChatRequest {
|
||||
request_id: "renderer-fixture".to_string(),
|
||||
messages: self
|
||||
@@ -144,12 +172,8 @@ impl FixtureRequest {
|
||||
.enumerate()
|
||||
.map(|(index, message)| fixture_message_to_chat_message(index, message))
|
||||
.collect(),
|
||||
tools: to_chat_tools(&self.tools),
|
||||
tool_choice: if self.tools.is_empty() {
|
||||
ChatToolChoice::None
|
||||
} else {
|
||||
ChatToolChoice::Auto
|
||||
},
|
||||
tools,
|
||||
tool_choice,
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
@@ -162,9 +186,15 @@ impl FixtureRequest {
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
}
|
||||
request.chat_options.reasoning_effort = self.reasoning_effort;
|
||||
if options.enable_thinking {
|
||||
for key in ["thinking", "enable_thinking"] {
|
||||
request.chat_options.template_kwargs.insert(key.to_string(), Value::Bool(true));
|
||||
request.chat_options.response_format = self.response_format;
|
||||
request.chat_options.template_kwargs.extend(self.template_kwargs);
|
||||
|
||||
// Options supply a default thinking toggle only when the fixture did not.
|
||||
if let Some(thinking) = options.enable_thinking {
|
||||
let kwargs = &mut request.chat_options.template_kwargs;
|
||||
if !kwargs.contains_key("thinking") && !kwargs.contains_key("enable_thinking") {
|
||||
kwargs.insert("thinking".to_string(), Value::Bool(thinking));
|
||||
kwargs.insert("enable_thinking".to_string(), Value::Bool(thinking));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -397,6 +397,10 @@ pub struct ChatOptions {
|
||||
/// Effort level exposed to chat templates for reasoning models.
|
||||
pub reasoning_effort: Option<ReasoningEffort>,
|
||||
|
||||
/// Standard response format available to model-specific renderers.
|
||||
#[serde(default)]
|
||||
pub response_format: Option<Value>,
|
||||
|
||||
/// Additional keyword arguments exposed to the chat template.
|
||||
pub template_kwargs: HashMap<String, Value>,
|
||||
}
|
||||
@@ -407,6 +411,7 @@ impl Default for ChatOptions {
|
||||
generation_prompt_mode: GenerationPromptMode::StartNewAssistant,
|
||||
chat_template: None,
|
||||
reasoning_effort: None,
|
||||
response_format: None,
|
||||
template_kwargs: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,6 +212,23 @@ impl RoundtripCase {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kimi K3 XTML tool/reasoning channels (native renderer + unified parser).
|
||||
///
|
||||
/// Needs HF tokenizer files under `HF_HOME` (`tiktoken.model` +
|
||||
/// `tokenizer_config.json`). Weights are not required for this text-level
|
||||
/// roundtrip.
|
||||
fn kimi_k3() -> Self {
|
||||
Self {
|
||||
model_id: "moonshotai/Kimi-K3",
|
||||
assistant_stop_suffix: "<|end_of_msg|>",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// SeedOSS with `<seed:think>` / `</seed:think>` reasoning tags.
|
||||
fn seed_oss() -> Self {
|
||||
Self {
|
||||
@@ -255,7 +272,7 @@ impl RoundtripCase {
|
||||
fn gpt_oss() -> Self {
|
||||
Self {
|
||||
model_id: "openai/gpt-oss-20b",
|
||||
assistant_stop_suffix: "", // not applicable for token-id cases
|
||||
assistant_stop_suffix: "",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Always { value: true },
|
||||
@@ -268,7 +285,7 @@ impl RoundtripCase {
|
||||
fn inkling() -> Self {
|
||||
Self {
|
||||
model_id: "thinkingmachines/Inkling",
|
||||
assistant_stop_suffix: "",
|
||||
assistant_stop_suffix: "<|content_model_end_sampling|>",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Always { value: true },
|
||||
@@ -312,6 +329,8 @@ roundtrip_tests! {
|
||||
nemotron_v3 => [reasoning_and_content],
|
||||
gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
|
||||
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
|
||||
// K3 drops plain-assistant reasoning in history; tool-call turns keep it.
|
||||
kimi_k3 => [tool_call_mix],
|
||||
gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
|
||||
inkling => [reasoning_and_content, tool_call_mix],
|
||||
}
|
||||
@@ -670,14 +689,23 @@ fn decoded_completion_stream(
|
||||
.collect()
|
||||
}
|
||||
Prompt::TokenIds(token_ids) => {
|
||||
ensure!(
|
||||
assistant_stop_suffix.is_empty(),
|
||||
"token-id roundtrip cases do not support text stop suffixes"
|
||||
);
|
||||
let body = if assistant_stop_suffix.is_empty() {
|
||||
token_ids.as_slice()
|
||||
} else {
|
||||
let stop_token_ids = tokenizer
|
||||
.encode(assistant_stop_suffix, false)
|
||||
.context("failed to encode token-id completion stop suffix")?;
|
||||
token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
|
||||
format!(
|
||||
"token-id completion did not end with {:?}: {:?}",
|
||||
assistant_stop_suffix, token_ids
|
||||
)
|
||||
})?
|
||||
};
|
||||
incremental_decode_chunks(
|
||||
tokenizer,
|
||||
&prompt_token_ids,
|
||||
token_ids,
|
||||
body,
|
||||
TOKEN_COMPLETION_CHUNK_TOKENS,
|
||||
)?
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user