[Model] Add Kimi K3 support: Rust frontend [1/2] (#50104)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-28 14:31:13 -07:00
committed by GitHub
parent 0b6aa3c47c
commit 2899dca843
35 changed files with 3003 additions and 39 deletions
@@ -89,6 +89,15 @@ pub(super) fn prepare_chat_request(
)?;
let template_kwargs = request.chat_template_kwargs.unwrap_or_default();
let response_format =
request.response_format.as_ref().map(serde_json::to_value).transpose().map_err(
|error| {
ApiError::invalid_request(
format!("failed to serialize response_format: {error}"),
Some("response_format"),
)
},
)?;
let include_usage = (request.stream_options.as_ref())
.and_then(|options| options.include_usage)
@@ -150,6 +159,7 @@ pub(super) fn prepare_chat_request(
generation_prompt_mode,
chat_template: request.chat_template,
reasoning_effort: request.reasoning_effort,
response_format,
template_kwargs,
},
tools: convert_tools(request.tools)?,
@@ -400,6 +410,7 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoic
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use axum::http::HeaderMap;
use expect_test::expect;
@@ -407,16 +418,18 @@ mod tests {
use serde_json::json;
use vllm_chat::{
AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
SamplingParams as VllmSamplingParams,
ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
KimiK3ChatRenderer, SamplingParams as VllmSamplingParams,
};
use vllm_text::output::TextDecodeOptions;
use vllm_text::{Prompt, output::TextDecodeOptions};
use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
use super::prepare_chat_request;
use crate::lora::LoraModelResolution;
use crate::routes::openai::chat_completions::types::{
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
};
use crate::routes::openai::utils::structured_outputs::{JsonSchemaFormat, ResponseFormat};
use crate::routes::openai::utils::types::{
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
@@ -473,6 +486,60 @@ mod tests {
assert!(prepared.chat_request.parallel_tool_calls);
}
#[test]
fn prepare_chat_request_passes_response_format_to_kimi_k3_renderer() {
let mut request = base_request();
request.model = "moonshotai/Kimi-K3".to_string();
let response_format = ResponseFormat::JsonSchema {
json_schema: JsonSchemaFormat {
name: "answer".to_string(),
description: None,
schema: json!({
"type": "object",
"properties": {
"answer": {"type": "string"}
},
"required": ["answer"]
}),
strict: Some(true),
},
};
request.response_format = Some(response_format.clone());
let template_response_format = json!({"type": "json_object"});
request.chat_template_kwargs = Some(HashMap::from([(
"response_format".to_string(),
template_response_format.clone(),
)]));
let prepared = prepare_chat_request(
request,
&served(&["moonshotai/Kimi-K3"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.chat_options.response_format.as_ref(),
Some(&serde_json::to_value(response_format).unwrap())
);
assert_eq!(
prepared.chat_request.chat_options.template_kwargs.get("response_format"),
Some(&template_response_format)
);
let tokenizer = Arc::new(TestTokenizer::new());
let prompt = KimiK3ChatRenderer::new(tokenizer.clone())
.render(&prepared.chat_request)
.expect("Kimi K3 rendering succeeds")
.prompt;
let Prompt::TokenIds(prompt_token_ids) = prompt else {
panic!("Kimi K3 renders token IDs");
};
let prompt = tokenizer.decode(&prompt_token_ids, false).expect("Kimi K3 prompt decodes");
assert!(prompt.contains("response_format=json_schema"));
assert!(prompt.contains(r#""answer":{"type":"string"}"#));
}
#[test]
fn prepare_chat_request_maps_text_parts() {
let mut request = base_request();
@@ -82,6 +82,7 @@ impl TokenizeChatRequest {
generation_prompt_mode,
chat_template: self.chat_template,
reasoning_effort: None,
response_format: None,
template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
},
tools: convert_tools(self.tools)?,