[Frontend][Bugfix] Use default tool call IDs for Kimi K3 for conversation-level uniqueness (#50420)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -126,7 +126,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
|
|||||||
resolved.sort_by_key(|item| (item.0, item.1));
|
resolved.sort_by_key(|item| (item.0, item.1));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (xtml_index, (_, _, content, name)) in resolved.into_iter().enumerate() {
|
for (position, _, content, name) in resolved {
|
||||||
let tool_name = name.as_deref().ok_or_else(|| {
|
let tool_name = name.as_deref().ok_or_else(|| {
|
||||||
Error::ChatTemplate(
|
Error::ChatTemplate(
|
||||||
"Kimi K3 tool messages need a resolvable tool name: \
|
"Kimi K3 tool messages need a resolvable tool name: \
|
||||||
@@ -135,7 +135,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
|
|||||||
.to_string(),
|
.to_string(),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
write_tool_message(out, tool_name, xtml_index + 1, &content)?;
|
write_tool_message(out, tool_name, position, &content)?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -14,11 +14,11 @@ use vllm_tokenizer::Tokenizer;
|
|||||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||||
|
|
||||||
use super::KimiK3ChatRenderer;
|
use super::KimiK3ChatRenderer;
|
||||||
use crate::AssistantContentBlock;
|
|
||||||
use crate::ChatRenderer;
|
use crate::ChatRenderer;
|
||||||
use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
|
use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
|
||||||
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
|
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
|
||||||
use crate::request::{ChatContentPart, ChatMessage, GenerationPromptMode, ReasoningEffort};
|
use crate::request::{ChatContentPart, ChatMessage, GenerationPromptMode, ReasoningEffort};
|
||||||
|
use crate::{AssistantContentBlock, AssistantToolCall};
|
||||||
|
|
||||||
const OPEN_ID: u32 = 256;
|
const OPEN_ID: u32 = 256;
|
||||||
const CLOSE_ID: u32 = 257;
|
const CLOSE_ID: u32 = 257;
|
||||||
@@ -178,6 +178,35 @@ fn non_thinking_history_omits_reasoning_channel() {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn partial_tool_results_keep_assistant_call_position() {
|
||||||
|
let mut request = crate::request::ChatRequest::for_test();
|
||||||
|
request.messages = vec![
|
||||||
|
ChatMessage::assistant_blocks(vec![
|
||||||
|
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||||
|
id: "call-a".to_string(),
|
||||||
|
name: "weather".to_string(),
|
||||||
|
arguments: "{}".to_string(),
|
||||||
|
}),
|
||||||
|
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||||
|
id: "call-b".to_string(),
|
||||||
|
name: "search".to_string(),
|
||||||
|
arguments: "{}".to_string(),
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
ChatMessage::tool_response("result-b", "call-b"),
|
||||||
|
];
|
||||||
|
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||||
|
|
||||||
|
let rendered = render_request(&request);
|
||||||
|
|
||||||
|
assert!(rendered.contains("<|open|>call tool=\"search\" index=\"2\"<|sep|>"));
|
||||||
|
assert!(
|
||||||
|
rendered
|
||||||
|
.contains("<|open|>message role=\"tool\" tool=\"search\" index=\"2\"<|sep|>result-b")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn defaults_thinking_effort_to_max() {
|
fn defaults_thinking_effort_to_max() {
|
||||||
let rendered = render_request(&crate::request::ChatRequest::for_test());
|
let rendered = render_request(&crate::request::ChatRequest::for_test());
|
||||||
|
|||||||
@@ -148,9 +148,8 @@ enum KimiK3Mode {
|
|||||||
pub struct KimiK3UnifiedParser {
|
pub struct KimiK3UnifiedParser {
|
||||||
buffer: String,
|
buffer: String,
|
||||||
mode: KimiK3Mode,
|
mode: KimiK3Mode,
|
||||||
/// Parser-provided tool-call IDs (`{tool}:{zero_based_index}`) by tool
|
/// Number of calls emitted in the current response.
|
||||||
/// index; its length is also the count of emitted calls.
|
emitted_call_count: usize,
|
||||||
call_ids: Vec<String>,
|
|
||||||
tokenizer: DynTokenizer,
|
tokenizer: DynTokenizer,
|
||||||
open_token_id: u32,
|
open_token_id: u32,
|
||||||
sep_token_id: u32,
|
sep_token_id: u32,
|
||||||
@@ -165,7 +164,7 @@ impl KimiK3UnifiedParser {
|
|||||||
Ok(Self {
|
Ok(Self {
|
||||||
buffer: String::new(),
|
buffer: String::new(),
|
||||||
mode: KimiK3Mode::default(),
|
mode: KimiK3Mode::default(),
|
||||||
call_ids: Vec::new(),
|
emitted_call_count: 0,
|
||||||
tokenizer,
|
tokenizer,
|
||||||
open_token_id,
|
open_token_id,
|
||||||
sep_token_id,
|
sep_token_id,
|
||||||
@@ -226,7 +225,7 @@ impl KimiK3UnifiedParser {
|
|||||||
}
|
}
|
||||||
KimiK3Event::CallComplete { arguments } => {
|
KimiK3Event::CallComplete { arguments } => {
|
||||||
let mode = std::mem::replace(&mut self.mode, KimiK3Mode::Tools);
|
let mode = std::mem::replace(&mut self.mode, KimiK3Mode::Tools);
|
||||||
let KimiK3Mode::Call { name, index, .. } = mode else {
|
let KimiK3Mode::Call { name, .. } = mode else {
|
||||||
return Err(parsing_failed!(
|
return Err(parsing_failed!(
|
||||||
"Kimi K3 call completion without an active tool call"
|
"Kimi K3 call completion without an active tool call"
|
||||||
));
|
));
|
||||||
@@ -237,8 +236,8 @@ impl KimiK3UnifiedParser {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let tool_index = self.call_ids.len();
|
let tool_index = self.emitted_call_count;
|
||||||
self.call_ids.push(tool_call_id_for(&name, index.as_deref()));
|
self.emitted_call_count += 1;
|
||||||
output.push_call(ToolCallDelta {
|
output.push_call(ToolCallDelta {
|
||||||
tool_index,
|
tool_index,
|
||||||
name: Some(name),
|
name: Some(name),
|
||||||
@@ -251,7 +250,7 @@ impl KimiK3UnifiedParser {
|
|||||||
|
|
||||||
fn reset_state(&mut self) -> String {
|
fn reset_state(&mut self) -> String {
|
||||||
self.mode = KimiK3Mode::Idle;
|
self.mode = KimiK3Mode::Idle;
|
||||||
self.call_ids.clear();
|
self.emitted_call_count = 0;
|
||||||
std::mem::take(&mut self.buffer)
|
std::mem::take(&mut self.buffer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -266,7 +265,7 @@ impl UnifiedParser for KimiK3UnifiedParser {
|
|||||||
|
|
||||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||||
self.buffer.clear();
|
self.buffer.clear();
|
||||||
self.call_ids.clear();
|
self.emitted_call_count = 0;
|
||||||
self.initialize_mode(prompt_token_ids);
|
self.initialize_mode(prompt_token_ids);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -279,10 +278,6 @@ impl UnifiedParser for KimiK3UnifiedParser {
|
|||||||
Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER)
|
Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
|
||||||
self.call_ids.get(tool_index).map(String::as_str)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_into(&mut self, chunk: &str, output: &mut UnifiedParserOutput) -> Result<()> {
|
fn parse_into(&mut self, chunk: &str, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||||
self.buffer.push_str(chunk);
|
self.buffer.push_str(chunk);
|
||||||
|
|
||||||
@@ -313,7 +308,6 @@ impl UnifiedParser for KimiK3UnifiedParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keep call_ids so tool_call_id() stays available after the stream ends.
|
|
||||||
self.mode = KimiK3Mode::Idle;
|
self.mode = KimiK3Mode::Idle;
|
||||||
Ok(output)
|
Ok(output)
|
||||||
}
|
}
|
||||||
@@ -323,20 +317,6 @@ impl UnifiedParser for KimiK3UnifiedParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the API-side tool-call ID from the XTML one-based `index` attribute.
|
|
||||||
///
|
|
||||||
/// The ID uses the zero-based call ordinal; XTML's message index stays
|
|
||||||
/// one-based when rendering tool result messages.
|
|
||||||
fn tool_call_id_for(name: &str, index: Option<&str>) -> String {
|
|
||||||
match index {
|
|
||||||
None => name.to_string(),
|
|
||||||
Some(raw) => match raw.parse::<i64>() {
|
|
||||||
Ok(one_based) => format!("{name}:{}", one_based - 1),
|
|
||||||
Err(_) => format!("{name}:{raw}"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse one Kimi K3 event from buffered streaming input.
|
/// Parse one Kimi K3 event from buffered streaming input.
|
||||||
fn parse_next_kimi_k3_event(
|
fn parse_next_kimi_k3_event(
|
||||||
input: &mut KimiK3Input<'_>,
|
input: &mut KimiK3Input<'_>,
|
||||||
@@ -762,7 +742,6 @@ mod tests {
|
|||||||
"hours": [8, 20],
|
"hours": [8, 20],
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
assert_eq!(parser.tool_call_id(0), Some("get_weather:0"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -929,8 +908,6 @@ mod tests {
|
|||||||
assert_eq!(calls[1].tool_index, 1);
|
assert_eq!(calls[1].tool_index, 1);
|
||||||
assert_eq!(calls[1].name.as_deref(), Some("get_time"));
|
assert_eq!(calls[1].name.as_deref(), Some("get_time"));
|
||||||
assert_eq!(calls[1].arguments, "{}");
|
assert_eq!(calls[1].arguments, "{}");
|
||||||
assert_eq!(parser.tool_call_id(0), Some("get_weather:0"));
|
|
||||||
assert_eq!(parser.tool_call_id(1), Some("get_time:1"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1018,11 +995,10 @@ mod tests {
|
|||||||
assert_eq!(calls.len(), 1);
|
assert_eq!(calls.len(), 1);
|
||||||
assert_eq!(calls[0].tool_index, 0);
|
assert_eq!(calls[0].tool_index, 0);
|
||||||
assert_eq!(calls[0].name.as_deref(), Some("real"));
|
assert_eq!(calls[0].name.as_deref(), Some("real"));
|
||||||
assert_eq!(parser.tool_call_id(0), Some("real:1"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn kimi_k3_tool_call_id_follows_index_attribute() {
|
fn kimi_k3_tool_indices_ignore_xtml_index_attribute() {
|
||||||
let tools_body = format!(
|
let tools_body = format!(
|
||||||
"{}{}{}",
|
"{}{}{}",
|
||||||
call("tool=\"first\" index=\"3\"", ""),
|
call("tool=\"first\" index=\"3\"", ""),
|
||||||
@@ -1032,11 +1008,12 @@ mod tests {
|
|||||||
let text = thinking_output("t", "", &tools_body);
|
let text = thinking_output("t", "", &tools_body);
|
||||||
|
|
||||||
let mut parser = test_parser();
|
let mut parser = test_parser();
|
||||||
parser.parse_complete(&text).unwrap();
|
let output = parser.parse_complete(&text).unwrap();
|
||||||
|
|
||||||
assert_eq!(parser.tool_call_id(0), Some("first:2"));
|
assert_eq!(
|
||||||
assert_eq!(parser.tool_call_id(1), Some("second"));
|
output.calls().iter().map(|call| call.tool_index).collect::<Vec<_>>(),
|
||||||
assert_eq!(parser.tool_call_id(2), Some("third:x"));
|
[0, 1, 2]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -141,7 +141,6 @@ def test_extract_tool_calls_with_response_and_typed_arguments():
|
|||||||
assert extracted.content == "answer"
|
assert extracted.content == "answer"
|
||||||
assert len(extracted.tool_calls) == 1
|
assert len(extracted.tool_calls) == 1
|
||||||
tool_call = extracted.tool_calls[0]
|
tool_call = extracted.tool_calls[0]
|
||||||
assert tool_call.id == "calc:0"
|
|
||||||
assert tool_call.function.name == "calc"
|
assert tool_call.function.name == "calc"
|
||||||
assert json.loads(tool_call.function.arguments) == {
|
assert json.loads(tool_call.function.arguments) == {
|
||||||
"x": 1,
|
"x": 1,
|
||||||
@@ -168,7 +167,6 @@ def test_delegating_parser_preserves_tool_calls_after_reasoning():
|
|||||||
assert content == "answer"
|
assert content == "answer"
|
||||||
assert tool_calls is not None
|
assert tool_calls is not None
|
||||||
assert len(tool_calls) == 1
|
assert len(tool_calls) == 1
|
||||||
assert tool_calls[0].id == "calc:0"
|
|
||||||
assert tool_calls[0].name == "calc"
|
assert tool_calls[0].name == "calc"
|
||||||
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
assert json.loads(tool_calls[0].arguments) == {"x": 1}
|
||||||
|
|
||||||
@@ -364,11 +362,19 @@ def test_streaming_split_markers_do_not_leak():
|
|||||||
assert OPEN not in content
|
assert OPEN not in content
|
||||||
assert SEP not in content
|
assert SEP not in content
|
||||||
assert len(tool_deltas) == 1
|
assert len(tool_deltas) == 1
|
||||||
assert tool_deltas[0].id == "calc:0"
|
|
||||||
assert tool_deltas[0].function.name == "calc"
|
assert tool_deltas[0].function.name == "calc"
|
||||||
assert json.loads(tool_deltas[0].function.arguments) == {"x": 1}
|
assert json.loads(tool_deltas[0].function.arguments) == {"x": 1}
|
||||||
|
|
||||||
|
|
||||||
|
def test_tool_call_ids_are_unique_across_messages():
|
||||||
|
output = _tools(_call("calc", 1))
|
||||||
|
|
||||||
|
first = KimiK3ToolParser(DummyTokenizer()).extract_tool_calls(output, _request())
|
||||||
|
second = KimiK3ToolParser(DummyTokenizer()).extract_tool_calls(output, _request())
|
||||||
|
|
||||||
|
assert first.tool_calls[0].id != second.tool_calls[0].id
|
||||||
|
|
||||||
|
|
||||||
def test_streaming_consumed_response_prefix_no_call_keeps_content():
|
def test_streaming_consumed_response_prefix_no_call_keeps_content():
|
||||||
parser = KimiK3ToolParser(DummyTokenizer())
|
parser = KimiK3ToolParser(DummyTokenizer())
|
||||||
request = _request()
|
request = _request()
|
||||||
|
|||||||
@@ -218,7 +218,6 @@ class KimiK3ToolParser(ToolParser):
|
|||||||
"""
|
"""
|
||||||
call_attrs = self._attrs(attrs)
|
call_attrs = self._attrs(attrs)
|
||||||
tool_name = call_attrs.get("tool", "")
|
tool_name = call_attrs.get("tool", "")
|
||||||
tool_index = call_attrs.get("index", "")
|
|
||||||
arguments: dict = {}
|
arguments: dict = {}
|
||||||
for arg_match in self._arg_re.finditer(body):
|
for arg_match in self._arg_re.finditer(body):
|
||||||
arg_attrs = self._attrs(arg_match["attrs"])
|
arg_attrs = self._attrs(arg_match["attrs"])
|
||||||
@@ -234,16 +233,7 @@ class KimiK3ToolParser(ToolParser):
|
|||||||
arguments[key] = raw_value
|
arguments[key] = raw_value
|
||||||
if not tool_name:
|
if not tool_name:
|
||||||
return None
|
return None
|
||||||
tool_call_id = tool_name
|
|
||||||
if tool_index:
|
|
||||||
try:
|
|
||||||
tool_call_id = f"{tool_name}:{int(tool_index) - 1}"
|
|
||||||
except ValueError:
|
|
||||||
tool_call_id = f"{tool_name}:{tool_index}"
|
|
||||||
# id uses the API-side zero-based call ordinal; XTML's message index
|
|
||||||
# stays one-based when rendering tool result messages.
|
|
||||||
return ToolCall(
|
return ToolCall(
|
||||||
id=tool_call_id,
|
|
||||||
type="function",
|
type="function",
|
||||||
function=FunctionCall(
|
function=FunctionCall(
|
||||||
name=tool_name,
|
name=tool_name,
|
||||||
|
|||||||
Reference in New Issue
Block a user