[Model] Add Kimi K3 support: Rust frontend [1/2] (#50104)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -532,7 +532,7 @@ fn detect_bpe_pattern(config: &TiktokenModelConfig) -> &'static str {
|
||||
let model_type = config.effective_model_type();
|
||||
|
||||
match model_type {
|
||||
Some("kimi" | "kimi_k2" | "kimi_k25" | "deepseek_v3") => KIMI_PATTERN,
|
||||
Some("kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "deepseek_v3") => KIMI_PATTERN,
|
||||
_ => CL100K_BASE_PATTERN,
|
||||
}
|
||||
}
|
||||
@@ -817,6 +817,7 @@ mod tests {
|
||||
#[test]
|
||||
fn tiktoken_detects_kimi_pattern_from_model_type() {
|
||||
let kimi = config_json!({ "model_type": "kimi_k25" });
|
||||
let kimi_k3 = config_json!({ "model_type": "kimi_k3" });
|
||||
let baseten_kimi = config_json!({ "model_type": "deepseek_v3" });
|
||||
let nested_kimi = config_json!({
|
||||
"model_type": "composite_wrapper",
|
||||
@@ -830,6 +831,7 @@ mod tests {
|
||||
let missing = config_json!({ "text_config": {} });
|
||||
|
||||
assert_eq!(detect_bpe_pattern(&kimi), KIMI_PATTERN);
|
||||
assert_eq!(detect_bpe_pattern(&kimi_k3), KIMI_PATTERN);
|
||||
assert_eq!(detect_bpe_pattern(&baseten_kimi), KIMI_PATTERN);
|
||||
assert_eq!(detect_bpe_pattern(&nested_kimi), CL100K_BASE_PATTERN);
|
||||
assert_eq!(detect_bpe_pattern(&generic), CL100K_BASE_PATTERN);
|
||||
|
||||
Reference in New Issue
Block a user