[Model] Add Kimi K3 support: Rust frontend [1/2] (#50104)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -89,6 +89,15 @@ pub(super) fn prepare_chat_request(
|
||||
)?;
|
||||
|
||||
let template_kwargs = request.chat_template_kwargs.unwrap_or_default();
|
||||
let response_format =
|
||||
request.response_format.as_ref().map(serde_json::to_value).transpose().map_err(
|
||||
|error| {
|
||||
ApiError::invalid_request(
|
||||
format!("failed to serialize response_format: {error}"),
|
||||
Some("response_format"),
|
||||
)
|
||||
},
|
||||
)?;
|
||||
|
||||
let include_usage = (request.stream_options.as_ref())
|
||||
.and_then(|options| options.include_usage)
|
||||
@@ -150,6 +159,7 @@ pub(super) fn prepare_chat_request(
|
||||
generation_prompt_mode,
|
||||
chat_template: request.chat_template,
|
||||
reasoning_effort: request.reasoning_effort,
|
||||
response_format,
|
||||
template_kwargs,
|
||||
},
|
||||
tools: convert_tools(request.tools)?,
|
||||
@@ -400,6 +410,7 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoic
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use expect_test::expect;
|
||||
@@ -407,16 +418,18 @@ mod tests {
|
||||
use serde_json::json;
|
||||
use vllm_chat::{
|
||||
AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
|
||||
ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
|
||||
SamplingParams as VllmSamplingParams,
|
||||
ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
|
||||
KimiK3ChatRenderer, SamplingParams as VllmSamplingParams,
|
||||
};
|
||||
use vllm_text::output::TextDecodeOptions;
|
||||
use vllm_text::{Prompt, output::TextDecodeOptions};
|
||||
use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
|
||||
|
||||
use super::prepare_chat_request;
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
|
||||
};
|
||||
use crate::routes::openai::utils::structured_outputs::{JsonSchemaFormat, ResponseFormat};
|
||||
use crate::routes::openai::utils::types::{
|
||||
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
|
||||
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
||||
@@ -473,6 +486,60 @@ mod tests {
|
||||
assert!(prepared.chat_request.parallel_tool_calls);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_passes_response_format_to_kimi_k3_renderer() {
|
||||
let mut request = base_request();
|
||||
request.model = "moonshotai/Kimi-K3".to_string();
|
||||
let response_format = ResponseFormat::JsonSchema {
|
||||
json_schema: JsonSchemaFormat {
|
||||
name: "answer".to_string(),
|
||||
description: None,
|
||||
schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"answer": {"type": "string"}
|
||||
},
|
||||
"required": ["answer"]
|
||||
}),
|
||||
strict: Some(true),
|
||||
},
|
||||
};
|
||||
request.response_format = Some(response_format.clone());
|
||||
let template_response_format = json!({"type": "json_object"});
|
||||
request.chat_template_kwargs = Some(HashMap::from([(
|
||||
"response_format".to_string(),
|
||||
template_response_format.clone(),
|
||||
)]));
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["moonshotai/Kimi-K3"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.response_format.as_ref(),
|
||||
Some(&serde_json::to_value(response_format).unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
prepared.chat_request.chat_options.template_kwargs.get("response_format"),
|
||||
Some(&template_response_format)
|
||||
);
|
||||
|
||||
let tokenizer = Arc::new(TestTokenizer::new());
|
||||
let prompt = KimiK3ChatRenderer::new(tokenizer.clone())
|
||||
.render(&prepared.chat_request)
|
||||
.expect("Kimi K3 rendering succeeds")
|
||||
.prompt;
|
||||
let Prompt::TokenIds(prompt_token_ids) = prompt else {
|
||||
panic!("Kimi K3 renders token IDs");
|
||||
};
|
||||
let prompt = tokenizer.decode(&prompt_token_ids, false).expect("Kimi K3 prompt decodes");
|
||||
assert!(prompt.contains("response_format=json_schema"));
|
||||
assert!(prompt.contains(r#""answer":{"type":"string"}"#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_text_parts() {
|
||||
let mut request = base_request();
|
||||
|
||||
@@ -82,6 +82,7 @@ impl TokenizeChatRequest {
|
||||
generation_prompt_mode,
|
||||
chat_template: self.chat_template,
|
||||
reasoning_effort: None,
|
||||
response_format: None,
|
||||
template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
|
||||
},
|
||||
tools: convert_tools(self.tools)?,
|
||||
|
||||
Reference in New Issue
Block a user