[Model][Frontend] Support DeepSeek-V4.1-Flash in Rust and Python frontends (#56208)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -19,8 +19,8 @@ use crate::output::{
|
||||
};
|
||||
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
|
||||
use crate::renderer::{
|
||||
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
|
||||
InklingChatRenderer, KimiK3ChatRenderer,
|
||||
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer,
|
||||
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer,
|
||||
};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{DynChatOutputProcessor, RendererSelection};
|
||||
@@ -72,6 +72,7 @@ impl HfChatBackend {
|
||||
)?),
|
||||
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
|
||||
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
|
||||
RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()),
|
||||
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
|
||||
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
|
||||
RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
|
||||
|
||||
@@ -17,6 +17,8 @@ pub enum Error {
|
||||
MissingChatTemplate,
|
||||
#[error("chat template error: {0}")]
|
||||
ChatTemplate(String),
|
||||
#[error("{0}")]
|
||||
InvalidReasoningEffort(String),
|
||||
#[error("multimodal input is not supported by this chat renderer")]
|
||||
UnsupportedMultimodalRenderer,
|
||||
#[error("unsupported multimodal content: {0}")]
|
||||
@@ -96,6 +98,7 @@ impl Error {
|
||||
pub fn is_request_validation_error(&self) -> bool {
|
||||
match self {
|
||||
Self::PromptTooLong { .. }
|
||||
| Self::InvalidReasoningEffort(_)
|
||||
| Self::DuplicateToolName { .. }
|
||||
| Self::ToolChoiceRequiresTools
|
||||
| Self::ToolChoiceFunctionNotFound { .. } => true,
|
||||
|
||||
@@ -33,8 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use parser::{ParserSelection, validate_parser_overrides};
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
|
||||
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
|
||||
DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
|
||||
RendererSelection,
|
||||
};
|
||||
pub use request::{
|
||||
@@ -363,7 +363,7 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -374,6 +374,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,6 +153,10 @@ struct ResolvedMultimodalSpec {
|
||||
modality: Modality,
|
||||
field_layouts: EncoderFieldLayouts,
|
||||
keep_on_cpu_keys: HashSet<String>,
|
||||
/// Spec-declared wire key for the primary encoder input tensor, when it
|
||||
/// differs from the per-modality default (e.g. `"patches"` for
|
||||
/// DeepSeek-V4.1 images).
|
||||
encoder_input_key: Option<String>,
|
||||
}
|
||||
|
||||
impl ResolvedMultimodalSpec {
|
||||
@@ -162,10 +166,14 @@ impl ResolvedMultimodalSpec {
|
||||
modality,
|
||||
field_layouts: raw.encoder_field_layouts_for(modality),
|
||||
keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
|
||||
encoder_input_key: raw.encoder_input_key_for(modality),
|
||||
}
|
||||
}
|
||||
|
||||
fn primary_key(&self) -> &'static str {
|
||||
fn primary_key(&self) -> &str {
|
||||
if let Some(key) = &self.encoder_input_key {
|
||||
return key;
|
||||
}
|
||||
match self.modality {
|
||||
Modality::Image => image::IMAGE_PRIMARY_KEY,
|
||||
Modality::Video => video::VIDEO_PRIMARY_KEY,
|
||||
@@ -841,6 +849,9 @@ mod tests {
|
||||
pub(super) const QWEN3_IMAGE_PAD_ID: u32 = 151655;
|
||||
pub(super) const QWEN3_VIDEO_PAD_ID: u32 = 151656;
|
||||
|
||||
pub(super) const DEEPSEEK_V41_IMAGE_ID: u32 = 129264;
|
||||
pub(super) const DEEPSEEK_V41_IMAGE_PAD_ID: u32 = 129265;
|
||||
|
||||
fn llama4_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
|
||||
@@ -954,6 +965,33 @@ mod tests {
|
||||
assert_eq!(info.placeholder_token(Modality::Video), None);
|
||||
}
|
||||
|
||||
fn deepseek_v41_info() -> MultimodalModelInfo {
|
||||
let config = serde_json::json!({
|
||||
"model_type": "deepseek_v41",
|
||||
"image_token_id": DEEPSEEK_V41_IMAGE_ID,
|
||||
});
|
||||
let tokenizer = TestTokenizer::new()
|
||||
.with_regular_token("<|deepseek_image|>", DEEPSEEK_V41_IMAGE_ID)
|
||||
.with_regular_token("<|place_holder_mm_span_0436|>", DEEPSEEK_V41_IMAGE_PAD_ID);
|
||||
test_info("deepseek_v41", config, tokenizer)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_v41_resolves_image_support_only() {
|
||||
let info = deepseek_v41_info();
|
||||
|
||||
assert_eq!(
|
||||
info.placeholder_token(Modality::Image),
|
||||
Some("<|deepseek_image|>")
|
||||
);
|
||||
assert_eq!(info.placeholder_token(Modality::Video), None);
|
||||
let image = info.image.as_ref().expect("image support");
|
||||
assert_eq!(image.placeholder.marker_token_id, DEEPSEEK_V41_IMAGE_ID);
|
||||
assert_eq!(image.placeholder.embed_token_id, DEEPSEEK_V41_IMAGE_ID);
|
||||
// The engine's forward kwargs pop `patches` (not `pixel_values`).
|
||||
assert_eq!(image.spec.primary_key(), "patches");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_audio_uses_connector_data_urls() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -45,6 +45,14 @@ impl<'a> ExpansionLane<'a> {
|
||||
/// matching the original media-part order within that modality; markers of
|
||||
/// different modalities may interleave freely.
|
||||
///
|
||||
/// Replacements may carry an [`AlignmentPad`](llm_multimodal::AlignmentPad)
|
||||
/// (e.g. DeepSeek-V4.1's compressor-alignment pad): `period - 1 - offset %
|
||||
/// period` pad tokens are
|
||||
/// prepended at splice time, where `offset` is the position the replacement
|
||||
/// lands at in the expanded prompt. The pads are part of the reported range
|
||||
/// and count as embed positions only if the pad token id equals the
|
||||
/// modality's embed token id.
|
||||
///
|
||||
/// The returned ranges point into the already-expanded prompt, grouped per
|
||||
/// modality in item order.
|
||||
pub(super) fn expand_prompt_token_ids(
|
||||
@@ -56,12 +64,14 @@ pub(super) fn expand_prompt_token_ids(
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
let replacement_growth = lanes
|
||||
.iter()
|
||||
.flat_map(|lane| lane.replacements.iter())
|
||||
.fold(0usize, |total, replacement| {
|
||||
total.saturating_add(replacement.tokens.len().saturating_sub(1))
|
||||
});
|
||||
let replacement_growth = lanes.iter().flat_map(|lane| lane.replacements.iter()).fold(
|
||||
0usize,
|
||||
|total, replacement| {
|
||||
// Alignment pads are position-dependent; reserve the worst case.
|
||||
let max_pads = replacement.alignment_pad.map_or(0, |pad| pad.period - 1);
|
||||
total.saturating_add(max_pads + replacement.tokens.len().saturating_sub(1))
|
||||
},
|
||||
);
|
||||
let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth);
|
||||
|
||||
let mut expanded = Vec::with_capacity(expanded_len);
|
||||
@@ -95,10 +105,20 @@ pub(super) fn expand_prompt_token_ids(
|
||||
expanded.len()
|
||||
);
|
||||
}
|
||||
let range_len = replacement_len.saturating_add(structural_prefix);
|
||||
// Position-dependent alignment padding, computed from the offset the
|
||||
// replacement block lands at in the expanded prompt (mirroring
|
||||
// vLLM's `_apply_token_matches_with_placeholders` splice-time pad).
|
||||
let pad_count = replacement.alignment_pad.map_or(0, |pad| pad.count_at(expanded.len()));
|
||||
let range_len = structural_prefix + pad_count + replacement_len;
|
||||
let is_embed = {
|
||||
let mut mask = Vec::with_capacity(range_len);
|
||||
mask.extend(std::iter::repeat_n(false, structural_prefix));
|
||||
if let Some(pad) = &replacement.alignment_pad {
|
||||
mask.extend(std::iter::repeat_n(
|
||||
pad.token_id as u32 == lane.embed_token_id,
|
||||
pad_count,
|
||||
));
|
||||
}
|
||||
mask.extend(
|
||||
replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
|
||||
);
|
||||
@@ -106,6 +126,9 @@ pub(super) fn expand_prompt_token_ids(
|
||||
};
|
||||
|
||||
let expanded_offset = expanded.len() - structural_prefix;
|
||||
if let Some(pad) = &replacement.alignment_pad {
|
||||
expanded.extend(std::iter::repeat_n(pad.token_id as u32, pad_count));
|
||||
}
|
||||
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
|
||||
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
|
||||
offset: expanded_offset,
|
||||
@@ -454,7 +477,6 @@ mod tests {
|
||||
}
|
||||
|
||||
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
|
||||
|
||||
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
|
||||
prepared_media(
|
||||
Modality::Audio,
|
||||
@@ -542,4 +564,110 @@ mod tests {
|
||||
));
|
||||
assert_eq!(prompt_token_ids, original_prompt_token_ids);
|
||||
}
|
||||
|
||||
// DeepSeek-V4.1 image spans: every span position carries the image token
|
||||
// id; a compressor-alignment pad (id 129265, period 2) is prepended at
|
||||
// splice time so spans always start at the same compressor phase.
|
||||
const DS_IMAGE_ID: u32 = 129_264;
|
||||
const DS_IMAGE_PAD_ID: u32 = 129_265;
|
||||
|
||||
fn deepseek_prepared(span_lens: Vec<usize>) -> PreparedMedia {
|
||||
prepared_media(
|
||||
Modality::Image,
|
||||
"<|deepseek_image|>",
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
span_lens
|
||||
.into_iter()
|
||||
.map(|len| {
|
||||
PromptReplacement::repeated(
|
||||
Modality::Image,
|
||||
"<|deepseek_image|>",
|
||||
DS_IMAGE_ID as TokenId,
|
||||
len,
|
||||
)
|
||||
.with_alignment_pad(DS_IMAGE_PAD_ID as TokenId, 2)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_skips_alignment_pad_at_odd_offset() {
|
||||
// Odd block start: already at the target compressor phase, no pad.
|
||||
let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2];
|
||||
let prepared = vec![deepseek_prepared(vec![3])];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
let ranges = &ranges[&Modality::Image];
|
||||
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![1, DS_IMAGE_ID, DS_IMAGE_ID, DS_IMAGE_ID, 2]
|
||||
);
|
||||
assert_eq!(ranges[0].offset, 1);
|
||||
assert_eq!(ranges[0].length, 3);
|
||||
assert_bool_mask(&ranges[0], &[true, true, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_prepends_alignment_pad_at_even_offset() {
|
||||
// Even block start: one pad token aligns the span to phase 1.
|
||||
let mut prompt_token_ids = vec![1, 3, DS_IMAGE_ID, 2];
|
||||
let prepared = vec![deepseek_prepared(vec![3])];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
let ranges = &ranges[&Modality::Image];
|
||||
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![
|
||||
1,
|
||||
3,
|
||||
DS_IMAGE_PAD_ID,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
2
|
||||
]
|
||||
);
|
||||
// The range starts at the pad; pads are not embed positions.
|
||||
assert_eq!(ranges[0].offset, 2);
|
||||
assert_eq!(ranges[0].length, 4);
|
||||
assert_bool_mask(&ranges[0], &[false, true, true, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_alignment_pad_realigns_each_span() {
|
||||
// Every image span must start at phase 1 (odd offset) regardless of
|
||||
// where the previous span landed.
|
||||
let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2, DS_IMAGE_ID, 3];
|
||||
let prepared = vec![deepseek_prepared(vec![2, 2])];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
let ranges = &ranges[&Modality::Image];
|
||||
|
||||
// First span at odd offset 1: no pad (block [1,3)).
|
||||
// Second marker originally at index 3; after the first expansion the
|
||||
// block starts at offset 4 (even): one pad, span at odd offset 5.
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![
|
||||
1,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
2,
|
||||
DS_IMAGE_PAD_ID,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
3,
|
||||
]
|
||||
);
|
||||
assert_eq!((ranges[0].offset, ranges[0].length), (1, 2));
|
||||
assert_bool_mask(&ranges[0], &[true, true]);
|
||||
assert_eq!((ranges[1].offset, ranges[1].length), (4, 3));
|
||||
assert_bool_mask(&ranges[1], &[false, true, true]);
|
||||
// Both spans (past the pad) start at odd offsets.
|
||||
assert_eq!((ranges[1].offset + 1) % 2, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,11 @@ use std::sync::{Arc, LazyLock};
|
||||
|
||||
pub use vllm_parser::reasoning::{
|
||||
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
|
||||
DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser,
|
||||
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
|
||||
NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
|
||||
ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
|
||||
DeepSeekV4ReasoningParser, DeepSeekV41ReasoningParser, Glm45ReasoningParser,
|
||||
Glm47ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser, MiniMaxM2ReasoningParser,
|
||||
MiniMaxM3ReasoningParser, NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta,
|
||||
ReasoningError, ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser,
|
||||
Step3p5ReasoningParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
@@ -22,6 +23,7 @@ pub mod names {
|
||||
pub const DEEPSEEK_R1: &str = "deepseek_r1";
|
||||
pub const DEEPSEEK_V3: &str = "deepseek_v3";
|
||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||
pub const DEEPSEEK_V41: &str = "deepseek_v41";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const GLM47: &str = "glm47";
|
||||
pub const KIMI: &str = "kimi";
|
||||
@@ -62,6 +64,7 @@ impl ReasoningParserFactory {
|
||||
.register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
|
||||
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
|
||||
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<DeepSeekV41ReasoningParser>(names::DEEPSEEK_V41)
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_parser::<Glm47ReasoningParser>(names::GLM47)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
@@ -76,6 +79,7 @@ impl ReasoningParserFactory {
|
||||
|
||||
factory
|
||||
.register_pattern("deepseek-r1", names::DEEPSEEK_R1)
|
||||
.register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
|
||||
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek-v3", names::DEEPSEEK_V3)
|
||||
|
||||
@@ -33,6 +33,10 @@ fn factory_resolves_deepseek_v4() {
|
||||
factory.resolve_name_for_model("deepseek_v4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
|
||||
Some(names::DEEPSEEK_V41)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock};
|
||||
|
||||
pub use vllm_parser::tool::{
|
||||
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
|
||||
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
|
||||
Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
|
||||
MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
|
||||
Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
|
||||
DeepSeekV41ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser,
|
||||
HermesToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
|
||||
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
|
||||
Qwen3CoderToolParser, Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
|
||||
};
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
@@ -22,6 +22,7 @@ pub mod names {
|
||||
pub const DEEPSEEK_V31: &str = "deepseek_v31";
|
||||
pub const DEEPSEEK_V32: &str = "deepseek_v32";
|
||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||
pub const DEEPSEEK_V41: &str = "deepseek_v41";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const GLM47: &str = "glm47";
|
||||
pub const GRANITE4: &str = "granite4";
|
||||
@@ -66,6 +67,7 @@ impl ToolParserFactory {
|
||||
.register_parser::<DeepSeekV31ToolParser>(names::DEEPSEEK_V31)
|
||||
.register_parser::<DeepSeekV32ToolParser>(names::DEEPSEEK_V32)
|
||||
.register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<DeepSeekV41ToolParser>(names::DEEPSEEK_V41)
|
||||
.register_parser::<Glm45MoeToolParser>(names::GLM45)
|
||||
.register_parser::<Glm47MoeToolParser>(names::GLM47)
|
||||
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
||||
@@ -101,6 +103,7 @@ impl ToolParserFactory {
|
||||
.register_pattern("llama-3.2", names::LLAMA3_JSON)
|
||||
.register_pattern("llama-3.1", names::LLAMA3_JSON)
|
||||
.register_pattern("deepseek-r1", names::DEEPSEEK_V3)
|
||||
.register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
|
||||
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek-v3.2", names::DEEPSEEK_V32)
|
||||
|
||||
@@ -147,6 +147,10 @@ fn factory_new_resolves_default_patterns() {
|
||||
factory.resolve_name_for_model("deepseek_v4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
|
||||
Some(names::DEEPSEEK_V41)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-R1-0528"),
|
||||
Some(names::DEEPSEEK_V3)
|
||||
|
||||
+230
-59
@@ -1,11 +1,12 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! DeepSeek V4 prompt renderer.
|
||||
//! Shared DeepSeek V4 and V4.1 prompt rendering.
|
||||
//!
|
||||
//! Official Python reference:
|
||||
//! <https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py>
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
@@ -13,8 +14,12 @@ use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use serde_json_fmt::JsonFormat;
|
||||
|
||||
use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, ReasoningEffort};
|
||||
use crate::request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort,
|
||||
};
|
||||
use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
|
||||
|
||||
const BOS_TOKEN: &str = "<|begin▁of▁sentence|>";
|
||||
@@ -23,6 +28,7 @@ const THINKING_START_TOKEN: &str = "<think>";
|
||||
const THINKING_END_TOKEN: &str = "</think>";
|
||||
const DSML_TOKEN: &str = "|DSML|";
|
||||
const USER_SP_TOKEN: &str = "<|User|>";
|
||||
const SYSTEM_SP_TOKEN: &str = "<|System|>";
|
||||
const ASSISTANT_SP_TOKEN: &str = "<|Assistant|>";
|
||||
const REASONING_EFFORT_HIGH: &str = concat!(
|
||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n",
|
||||
@@ -41,6 +47,35 @@ enum ThinkingMode {
|
||||
Thinking,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum DsDialect {
|
||||
V4,
|
||||
V41,
|
||||
}
|
||||
|
||||
impl DsDialect {
|
||||
fn tool_calls_tag(self) -> &'static str {
|
||||
match self {
|
||||
Self::V4 => "tool_calls",
|
||||
Self::V41 => " calls",
|
||||
}
|
||||
}
|
||||
|
||||
fn invoke_tag(self) -> &'static str {
|
||||
match self {
|
||||
Self::V4 => "invoke",
|
||||
Self::V41 => " invoke",
|
||||
}
|
||||
}
|
||||
|
||||
fn parameter_tag(self) -> &'static str {
|
||||
match self {
|
||||
Self::V4 => "parameter",
|
||||
Self::V41 => " parameter",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RenderedToolSchema<'a> {
|
||||
@@ -51,25 +86,47 @@ struct RenderedToolSchema<'a> {
|
||||
}
|
||||
|
||||
/// Render one chat request into the final prompt string.
|
||||
pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
let (thinking_mode, reasoning_effort_prompt) = resolve_thinking_options(request)?;
|
||||
pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result<String> {
|
||||
let (thinking_mode, reasoning_effort_prompt) = match dialect {
|
||||
DsDialect::V4 => {
|
||||
resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))?
|
||||
}
|
||||
DsDialect::V41 => resolve_v41_thinking_options(request)?,
|
||||
};
|
||||
let request_tools = request_tools(request);
|
||||
let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools);
|
||||
let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true)
|
||||
&& !rendered_tools_present(request, request_tools);
|
||||
let drop_historical_developers = thinking_mode == ThinkingMode::Thinking && drop_thinking;
|
||||
let last_user_like_message_index = request.messages.iter().rposition(is_user_like_entry);
|
||||
let last_user_render_index =
|
||||
find_last_user_render_index(request.messages.as_slice(), synthetic_tool_system);
|
||||
let last_user_like_message_index = request
|
||||
.messages
|
||||
.iter()
|
||||
.enumerate()
|
||||
.rfind(|(index, message)| is_user_like_entry(message, *index, dialect))
|
||||
.map(|(index, _)| index);
|
||||
let last_user_render_index = find_last_user_render_index(
|
||||
request.messages.as_slice(),
|
||||
synthetic_tool_system,
|
||||
drop_historical_developers,
|
||||
last_user_like_message_index,
|
||||
dialect,
|
||||
);
|
||||
let mut out = String::from(BOS_TOKEN);
|
||||
if dialect == DsDialect::V41
|
||||
&& (thinking_mode == ThinkingMode::Thinking
|
||||
|| synthetic_tool_system
|
||||
|| matches!(request.messages.first(), Some(ChatMessage::System { .. })))
|
||||
{
|
||||
out.push_str(SYSTEM_SP_TOKEN);
|
||||
}
|
||||
if thinking_mode == ThinkingMode::Thinking {
|
||||
out.push_str(reasoning_effort_prompt);
|
||||
out.push_str(&reasoning_effort_prompt);
|
||||
}
|
||||
|
||||
let mut request_tools_attached = false;
|
||||
let mut render_index = 0isize;
|
||||
if synthetic_tool_system {
|
||||
render_system_message(&mut out, None, request_tools)?;
|
||||
render_system_message(&mut out, None, request_tools, dialect)?;
|
||||
request_tools_attached = true;
|
||||
render_index += 1;
|
||||
}
|
||||
@@ -80,27 +137,35 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
continue;
|
||||
}
|
||||
|
||||
let current_render_index = render_index;
|
||||
render_index += 1;
|
||||
|
||||
if drop_historical_developers
|
||||
&& is_historical_developer(message, message_index, last_user_like_message_index)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let current_render_index = render_index;
|
||||
render_index += 1;
|
||||
|
||||
match message {
|
||||
ChatMessage::System { content } => {
|
||||
if dialect == DsDialect::V41 && current_render_index > 0 {
|
||||
out.push_str(SYSTEM_SP_TOKEN);
|
||||
}
|
||||
let tools = if !request_tools_attached {
|
||||
request_tools_attached = true;
|
||||
request_tools
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
render_system_message(&mut out, Some(content), tools)?;
|
||||
render_system_message(&mut out, Some(content), tools, dialect)?;
|
||||
}
|
||||
ChatMessage::Developer { content, tools } => {
|
||||
render_developer_message(&mut out, content, tools.as_deref().unwrap_or(&[]))?;
|
||||
render_developer_message(
|
||||
&mut out,
|
||||
content,
|
||||
tools.as_deref().unwrap_or(&[]),
|
||||
dialect,
|
||||
)?;
|
||||
}
|
||||
ChatMessage::User { .. } | ChatMessage::ToolResponse { .. } => {
|
||||
render_user_content_block(
|
||||
@@ -108,6 +173,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
request.messages.as_slice(),
|
||||
message_index,
|
||||
&last_tool_call_order,
|
||||
dialect,
|
||||
)?;
|
||||
}
|
||||
ChatMessage::Assistant { content } => {
|
||||
@@ -119,7 +185,13 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
&& (!drop_thinking || current_render_index > last_user_render_index);
|
||||
let append_eos = !(message_index + 1 == request.messages.len()
|
||||
&& request.chat_options.continue_final_message());
|
||||
render_assistant_message(&mut out, emit_thinking_block, append_eos, content)?;
|
||||
render_assistant_message(
|
||||
&mut out,
|
||||
emit_thinking_block,
|
||||
append_eos,
|
||||
content,
|
||||
dialect,
|
||||
)?;
|
||||
|
||||
if content.has_tool_calls() {
|
||||
last_tool_call_order.clear();
|
||||
@@ -133,7 +205,8 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
}
|
||||
}
|
||||
|
||||
if (is_user_like_entry(message) || matches!(message, ChatMessage::System { .. }))
|
||||
if (is_user_like_entry(message, current_render_index as usize, dialect)
|
||||
|| (dialect == DsDialect::V4 && matches!(message, ChatMessage::System { .. })))
|
||||
&& next_rendered_entry_is_assistant_or_end(
|
||||
request.messages.as_slice(),
|
||||
message_index,
|
||||
@@ -181,6 +254,57 @@ fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'st
|
||||
Ok((thinking_mode, reasoning_effort_prompt))
|
||||
}
|
||||
|
||||
/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs.
|
||||
fn resolve_v41_thinking_options(
|
||||
request: &ChatRequest,
|
||||
) -> Result<(ThinkingMode, Cow<'static, str>)> {
|
||||
let mut thinking = request.enable_thinking()?.unwrap_or(true);
|
||||
let budget = match request.chat_options.reasoning_effort {
|
||||
Some(ReasoningEffort::None) => {
|
||||
thinking = false;
|
||||
50
|
||||
}
|
||||
Some(ReasoningEffort::Low) => 25,
|
||||
Some(ReasoningEffort::High) => 50,
|
||||
Some(ReasoningEffort::XHigh) => 75,
|
||||
Some(ReasoningEffort::Max) => 100,
|
||||
Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => {
|
||||
return Err(invalid_v41_reasoning_effort());
|
||||
}
|
||||
None => match request.chat_options.template_kwargs.get("reasoning_effort") {
|
||||
None | Some(Value::Null) => 50,
|
||||
Some(Value::String(effort)) => match effort.as_str() {
|
||||
"low" => 25,
|
||||
"high" => 50,
|
||||
"xhigh" => 75,
|
||||
"max" => 100,
|
||||
_ => return Err(invalid_v41_reasoning_effort()),
|
||||
},
|
||||
Some(Value::Number(budget)) => budget
|
||||
.as_u64()
|
||||
.filter(|budget| (1..=100).contains(budget))
|
||||
.ok_or_else(invalid_v41_reasoning_effort)?,
|
||||
Some(_) => return Err(invalid_v41_reasoning_effort()),
|
||||
},
|
||||
};
|
||||
if thinking {
|
||||
Ok((
|
||||
ThinkingMode::Thinking,
|
||||
Cow::Owned(format!(
|
||||
"Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
|
||||
)),
|
||||
))
|
||||
} else {
|
||||
Ok((ThinkingMode::Chat, Cow::Borrowed("")))
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_v41_reasoning_effort() -> Error {
|
||||
Error::InvalidReasoningEffort(
|
||||
"DeepSeek V4.1 reasoning_effort must be low, high, xhigh, max, or an integer within [1, 100] in chat_template_kwargs".to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Return request-level tools only when native tool parsing is enabled.
|
||||
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
|
||||
if request.tool_parsing_enabled() {
|
||||
@@ -214,16 +338,25 @@ fn rendered_tools_present(request: &ChatRequest, request_tools: &[ChatTool]) ->
|
||||
}
|
||||
|
||||
/// Find the last user-like turn after inline tool-response merging.
|
||||
fn find_last_user_render_index(messages: &[ChatMessage], synthetic_tool_system: bool) -> isize {
|
||||
fn find_last_user_render_index(
|
||||
messages: &[ChatMessage],
|
||||
synthetic_tool_system: bool,
|
||||
drop_historical_developers: bool,
|
||||
last_user_like_message_index: Option<usize>,
|
||||
dialect: DsDialect,
|
||||
) -> isize {
|
||||
let mut render_index = isize::from(synthetic_tool_system);
|
||||
let mut last_user_index = -1;
|
||||
|
||||
for (message_index, message) in messages.iter().enumerate() {
|
||||
if is_following_user_content(messages, message_index) {
|
||||
if is_following_user_content(messages, message_index)
|
||||
|| (drop_historical_developers
|
||||
&& is_historical_developer(message, message_index, last_user_like_message_index))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_user_like_entry(message) {
|
||||
if is_user_like_entry(message, render_index as usize, dialect) {
|
||||
last_user_index = render_index;
|
||||
}
|
||||
render_index += 1;
|
||||
@@ -249,11 +382,13 @@ fn is_user_content_entry(message: &ChatMessage) -> bool {
|
||||
}
|
||||
|
||||
/// Return whether one rendered entry should be treated as user-like.
|
||||
fn is_user_like_entry(message: &ChatMessage) -> bool {
|
||||
fn is_user_like_entry(message: &ChatMessage, message_index: usize, dialect: DsDialect) -> bool {
|
||||
matches!(
|
||||
message,
|
||||
ChatMessage::Developer { .. } | ChatMessage::User { .. } | ChatMessage::ToolResponse { .. }
|
||||
)
|
||||
) || (dialect == DsDialect::V41
|
||||
&& message_index > 0
|
||||
&& matches!(message, ChatMessage::System { .. }))
|
||||
}
|
||||
|
||||
/// Return whether a developer entry precedes another user-like turn.
|
||||
@@ -293,22 +428,26 @@ fn next_rendered_entry_is_assistant_or_end(
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
/// Render the tool preamble shown to the model, V4 flavor.
|
||||
fn render_tools(out: &mut String, tools: &[ChatTool]) -> Result<()> {
|
||||
out.push_str(
|
||||
/// Render the tool preamble shown to the model for one DeepSeek dialect.
|
||||
fn render_tools(out: &mut String, tools: &[ChatTool], dialect: DsDialect) -> Result<()> {
|
||||
let tool_calls_tag = dialect.tool_calls_tag();
|
||||
let invoke_tag = dialect.invoke_tag();
|
||||
let parameter_tag = dialect.parameter_tag();
|
||||
write!(
|
||||
out,
|
||||
r#"## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|{tool_calls_tag}>" block like the following:
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="$TOOL_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
<|DSML|{tool_calls_tag}>
|
||||
<|DSML|{invoke_tag} name="$TOOL_NAME">
|
||||
<|DSML|{parameter_tag} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|{parameter_tag}>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$TOOL_NAME2">
|
||||
</|DSML|{invoke_tag}>
|
||||
<|DSML|{invoke_tag} name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls>
|
||||
</|DSML|{invoke_tag}>
|
||||
</|DSML|{tool_calls_tag}>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
@@ -319,7 +458,8 @@ Otherwise, output directly after </think> with tool calls or final response.
|
||||
### Available Tool Schemas
|
||||
|
||||
"#,
|
||||
);
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
|
||||
for (index, tool) in tools.iter().enumerate() {
|
||||
if index > 0 {
|
||||
@@ -350,13 +490,14 @@ fn render_system_message(
|
||||
out: &mut String,
|
||||
content: Option<&ChatContent>,
|
||||
tools: &[ChatTool],
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
if let Some(content) = content {
|
||||
write_chat_content(out, content)?;
|
||||
write_chat_content(out, content, dialect)?;
|
||||
}
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
render_tools(out, tools, dialect)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -366,18 +507,19 @@ fn render_developer_message(
|
||||
out: &mut String,
|
||||
content: &ChatContent,
|
||||
tools: &[ChatTool],
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
if content.is_empty() {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V4 developer message: empty content".to_string(),
|
||||
"invalid DeepSeek developer message: empty content".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
out.push_str(USER_SP_TOKEN);
|
||||
write_chat_content(out, content)?;
|
||||
write_chat_content(out, content, dialect)?;
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
render_tools(out, tools, dialect)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -388,6 +530,7 @@ fn render_user_content_block(
|
||||
messages: &[ChatMessage],
|
||||
message_index: usize,
|
||||
tool_call_order: &HashMap<String, usize>,
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let (block_start, block_end) = user_content_block_bounds(messages, message_index);
|
||||
let mut sorted_tool_indices =
|
||||
@@ -399,7 +542,7 @@ fn render_user_content_block(
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
match &messages[message_index] {
|
||||
ChatMessage::User { content } => write_chat_content(out, content)?,
|
||||
ChatMessage::User { content } => write_chat_content(out, content, dialect)?,
|
||||
ChatMessage::ToolResponse { .. } => {
|
||||
let sorted_index = sorted_tool_indices
|
||||
.next()
|
||||
@@ -407,7 +550,7 @@ fn render_user_content_block(
|
||||
let ChatMessage::ToolResponse { content, .. } = &messages[sorted_index] else {
|
||||
unreachable!("sorted tool response index should reference a tool message");
|
||||
};
|
||||
write_tool_result(out, content)?;
|
||||
write_tool_result(out, content, dialect)?;
|
||||
}
|
||||
_ => unreachable!("user content block should only contain user content messages"),
|
||||
}
|
||||
@@ -454,9 +597,9 @@ fn sorted_tool_response_indices(
|
||||
}
|
||||
|
||||
/// Render one tool response payload inside a V4 `<tool_result>` block.
|
||||
fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
fn write_tool_result(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
|
||||
out.push_str("<tool_result>");
|
||||
write_chat_content(out, content)?;
|
||||
write_chat_content(out, content, dialect)?;
|
||||
out.push_str("</tool_result>");
|
||||
Ok(())
|
||||
}
|
||||
@@ -483,6 +626,7 @@ fn render_assistant_message(
|
||||
emit_thinking_block: bool,
|
||||
append_eos: bool,
|
||||
content: &[AssistantContentBlock],
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let has_tool_calls = content.has_tool_calls();
|
||||
|
||||
@@ -496,14 +640,15 @@ fn render_assistant_message(
|
||||
write_assistant_text(out, content);
|
||||
|
||||
if has_tool_calls {
|
||||
out.push_str("\n\n<|DSML|tool_calls>\n");
|
||||
let tool_calls_tag = dialect.tool_calls_tag();
|
||||
writeln!(out, "\n\n<{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
|
||||
for (index, tool_call) in content.tool_calls().enumerate() {
|
||||
if index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
render_tool_call(out, tool_call)?;
|
||||
render_tool_call(out, tool_call, dialect)?;
|
||||
}
|
||||
out.push_str("\n</|DSML|tool_calls>");
|
||||
write!(out, "\n</{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
|
||||
}
|
||||
|
||||
if append_eos {
|
||||
@@ -513,11 +658,20 @@ fn render_assistant_message(
|
||||
}
|
||||
|
||||
/// Render one assistant tool call in DSML XML-like format.
|
||||
fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
writeln!(out, "<{DSML_TOKEN}invoke name=\"{}\">", tool_call.name)
|
||||
.expect("writing to String cannot fail");
|
||||
encode_arguments_to_dsml(out, tool_call)?;
|
||||
write!(out, "\n</{DSML_TOKEN}invoke>").expect("writing to String cannot fail");
|
||||
fn render_tool_call(
|
||||
out: &mut String,
|
||||
tool_call: &AssistantToolCall,
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let invoke_tag = dialect.invoke_tag();
|
||||
writeln!(
|
||||
out,
|
||||
"<{DSML_TOKEN}{invoke_tag} name=\"{}\">",
|
||||
tool_call.name
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
encode_arguments_to_dsml(out, tool_call, dialect)?;
|
||||
write!(out, "\n</{DSML_TOKEN}{invoke_tag}>").expect("writing to String cannot fail");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -525,15 +679,20 @@ fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<(
|
||||
///
|
||||
/// String values are emitted raw with `string="true"`, while all other JSON
|
||||
/// values are rendered with JSON syntax and `string="false"`.
|
||||
fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
fn encode_arguments_to_dsml(
|
||||
out: &mut String,
|
||||
tool_call: &AssistantToolCall,
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let parameter_tag = dialect.parameter_tag();
|
||||
let arguments: Value = serde_json::from_str(&tool_call.arguments).map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"assistant tool call has invalid JSON arguments for DeepSeek V4: {error}"
|
||||
"assistant tool call has invalid JSON arguments for DeepSeek: {error}"
|
||||
))
|
||||
})?;
|
||||
let Some(arguments) = arguments.as_object() else {
|
||||
return Err(Error::ChatTemplate(
|
||||
"assistant tool call arguments for DeepSeek V4 must be a JSON object".to_string(),
|
||||
"assistant tool call arguments for DeepSeek must be a JSON object".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
@@ -546,7 +705,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
|
||||
let is_string = matches!(value, Value::String(_));
|
||||
write!(
|
||||
out,
|
||||
"<{DSML_TOKEN}parameter name=\"{key}\" string=\"{}\">",
|
||||
"<{DSML_TOKEN}{parameter_tag} name=\"{key}\" string=\"{}\">",
|
||||
if is_string { "true" } else { "false" }
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
@@ -556,7 +715,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
|
||||
value => out.push_str(&json_dumps(value)?),
|
||||
}
|
||||
|
||||
write!(out, "</{DSML_TOKEN}parameter>").expect("writing to String cannot fail");
|
||||
write!(out, "</{DSML_TOKEN}{parameter_tag}>").expect("writing to String cannot fail");
|
||||
wrote_parameter = true;
|
||||
}
|
||||
|
||||
@@ -565,12 +724,24 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
|
||||
|
||||
/// Write chat content directly into the destination buffer without flattening
|
||||
/// it into an intermediate `String`.
|
||||
fn write_chat_content(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
///
|
||||
/// The V4.1 dialect inlines the image placeholder at each image part's
|
||||
/// position unconditionally (matching the Python encoding's
|
||||
/// `IMAGE_PLACEHOLDER`); other dialects reject multimodal parts.
|
||||
fn write_chat_content(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
|
||||
match content {
|
||||
ChatContent::Text(text) => out.push_str(text),
|
||||
ChatContent::Parts(parts) => {
|
||||
for part in parts {
|
||||
out.push_str(part.as_text()?);
|
||||
for (index, part) in parts.iter().enumerate() {
|
||||
if index > 0 && dialect == DsDialect::V41 {
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
match part {
|
||||
ChatContentPart::ImageUrl { .. } if dialect == DsDialect::V41 => {
|
||||
out.push_str(DEEPSEEK_V41_IMAGE_PLACEHOLDER);
|
||||
}
|
||||
_ => out.push_str(part.as_text()?),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -606,7 +777,7 @@ fn json_dumps<T: Serialize>(value: &T) -> Result<String> {
|
||||
.format_to_string(value)
|
||||
.map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"failed to serialize DeepSeek V4 JSON payload: {error}"
|
||||
"failed to serialize DeepSeek JSON payload: {error}"
|
||||
))
|
||||
})
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::deepseek::{self, DsDialect};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
@@ -24,7 +23,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
|
||||
request.validate()?;
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="$TOOL_NAME">
|
||||
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
|
||||
...
|
||||
</|DSML| invoke>
|
||||
<|DSML| invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
|
||||
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="get_weather">
|
||||
<|DSML| parameter name="location" string="true">Beijing</|DSML| parameter>
|
||||
<|DSML| parameter name="unit" string="true">celsius</|DSML| parameter>
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,3 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,30 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
<|User|>question<|User|>policy
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="$TOOL_NAME">
|
||||
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
|
||||
...
|
||||
</|DSML| invoke>
|
||||
<|DSML| invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "lookup", "parameters": {"type": "object"}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|Assistant|><think></think>answer<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,33 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::deepseek::{self, DsDialect};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
/// Dedicated DeepSeek V4.1 renderer.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct DeepSeekV41ChatRenderer;
|
||||
|
||||
impl DeepSeekV41ChatRenderer {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for DeepSeekV41ChatRenderer {
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
request.validate()?;
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V41)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,223 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use expect_test::{expect, expect_file};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::DeepSeekV41ChatRenderer;
|
||||
use crate::ChatRenderer;
|
||||
use crate::event::AssistantContentBlock;
|
||||
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ReasoningEffort};
|
||||
|
||||
fn render(request: &ChatRequest) -> String {
|
||||
DeepSeekV41ChatRenderer::new()
|
||||
.render(request)
|
||||
.unwrap()
|
||||
.prompt
|
||||
.into_text()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn request() -> ChatRequest {
|
||||
ChatRequest {
|
||||
messages: vec![ChatMessage::user("question")],
|
||||
..ChatRequest::for_test()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_shared_deepseek_fixtures_with_v41_reference_encoding() {
|
||||
let fixture_dir =
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/renderer/deepseek_v4/fixtures");
|
||||
for (input, expected) in [
|
||||
(
|
||||
"test_input_1.json",
|
||||
expect_file!["fixtures/test_output_1.txt"],
|
||||
),
|
||||
(
|
||||
"test_input_2.json",
|
||||
expect_file!["fixtures/test_output_2.txt"],
|
||||
),
|
||||
(
|
||||
"test_input_developer_tools.json",
|
||||
expect_file!["fixtures/test_output_developer_tools.txt"],
|
||||
),
|
||||
] {
|
||||
let request = fixture_chat_request(
|
||||
&fixture_dir.join(input),
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: Some(true),
|
||||
no_generation_prompt_when_last_assistant: true,
|
||||
},
|
||||
);
|
||||
expected.assert_eq(&render(&request));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inlines_image_placeholder_at_image_part_positions() {
|
||||
// The placeholder is unconditional (matching the Python encoding's
|
||||
// `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later,
|
||||
// at media preparation.
|
||||
let renderer = DeepSeekV41ChatRenderer::new();
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::User {
|
||||
content: ChatContent::Parts(vec![
|
||||
ChatContentPart::image_url("https://example.com/carrots.jpeg"),
|
||||
ChatContentPart::text("what is in the image?"),
|
||||
]),
|
||||
}],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let rendered = renderer.render(&request).unwrap().prompt.into_text().unwrap();
|
||||
|
||||
// Image parts inline the placeholder at their position; parts stay
|
||||
// separated by "\n\n" (the V4.1 reference encoding's part separator).
|
||||
assert!(
|
||||
rendered.contains("<|User|><|deepseek_image|>\n\nwhat is in the image?"),
|
||||
"unexpected prompt: {rendered:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_reasoning_effort_to_reference_numeric_budget() {
|
||||
for (effort, budget) in [
|
||||
(None, 50),
|
||||
(Some(ReasoningEffort::Low), 25),
|
||||
(Some(ReasoningEffort::High), 50),
|
||||
(Some(ReasoningEffort::XHigh), 75),
|
||||
(Some(ReasoningEffort::Max), 100),
|
||||
] {
|
||||
let mut request = request();
|
||||
request.chat_options.reasoning_effort = effort;
|
||||
assert_eq!(
|
||||
render(&request),
|
||||
format!(
|
||||
"<|begin▁of▁sentence|><|System|>Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>question<|Assistant|><think>"
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_numeric_template_effort_with_top_level_precedence() {
|
||||
for (effort, budget) in [
|
||||
(json!(1), 1),
|
||||
(json!(42), 42),
|
||||
(json!(100), 100),
|
||||
(json!("low"), 25),
|
||||
] {
|
||||
let mut request = request();
|
||||
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
|
||||
assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100")));
|
||||
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
|
||||
assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
|
||||
for effort in [
|
||||
json!("medium"),
|
||||
json!("minimal"),
|
||||
json!(-1),
|
||||
json!(0),
|
||||
json!(101),
|
||||
json!(true),
|
||||
json!(1.5),
|
||||
] {
|
||||
let mut request = request();
|
||||
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
|
||||
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
|
||||
assert!(error.is_request_validation_error());
|
||||
}
|
||||
for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
|
||||
let mut request = request();
|
||||
request.chat_options.reasoning_effort = Some(effort);
|
||||
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
|
||||
assert!(error.is_request_validation_error());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_mode_and_none_effort_use_closed_thinking_prefix() {
|
||||
let mut request = request();
|
||||
request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
|
||||
let expected = expect!["<|begin▁of▁sentence|><|User|>question<|Assistant|></think>"];
|
||||
expected.assert_eq(&render(&request));
|
||||
|
||||
request.chat_options.template_kwargs.insert("thinking".into(), json!(true));
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
|
||||
expected.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leading_system_in_chat_mode_uses_system_token() {
|
||||
let mut request = request();
|
||||
request.messages.insert(0, ChatMessage::system("policy"));
|
||||
request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
|
||||
expect!["<|begin▁of▁sentence|><|System|>policy<|User|>question<|Assistant|></think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_after_dropped_developer_becomes_leading_system() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::developer("old policy", None),
|
||||
ChatMessage::system("sys"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys"]
|
||||
.assert_eq(&render(&request));
|
||||
|
||||
request.messages.push(ChatMessage::user("question"));
|
||||
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>question<|Assistant|><think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_system_advances_last_user_boundary_and_drops_old_reasoning() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::system("sys"),
|
||||
ChatMessage::user("q1"),
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning { text: "r1".into() },
|
||||
AssistantContentBlock::Text { text: "a1".into() },
|
||||
]),
|
||||
ChatMessage::developer("old policy", None),
|
||||
ChatMessage::system("mid sys"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("reasoning_effort".into(), json!(88));
|
||||
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 88 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>q1<|Assistant|></think>a1<|end▁of▁sentence|><|System|>mid sys<|Assistant|><think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn joins_text_parts_with_reference_separator_and_preserves_literal_tags() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![ChatMessage::user(ChatContent::Parts(vec![
|
||||
ChatContentPart::text("first"),
|
||||
ChatContentPart::text("<|DSML|invoke name=\"literal\">"),
|
||||
]))],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".into(), Value::Bool(false));
|
||||
expect!["<|begin▁of▁sentence|><|User|>first\n\n<|DSML|invoke name=\"literal\"><|Assistant|></think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
@@ -10,8 +10,10 @@ use vllm_text::Prompt;
|
||||
use crate::error::Result;
|
||||
use crate::request::{ChatRequest, ReasoningEffort};
|
||||
|
||||
mod deepseek;
|
||||
pub mod deepseek_v32;
|
||||
pub mod deepseek_v4;
|
||||
pub mod deepseek_v41;
|
||||
pub mod harmony;
|
||||
pub mod hf;
|
||||
mod inkling;
|
||||
@@ -22,6 +24,7 @@ mod test_utils;
|
||||
|
||||
pub use deepseek_v4::DeepSeekV4ChatRenderer;
|
||||
pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use deepseek_v41::DeepSeekV41ChatRenderer;
|
||||
pub use harmony::HarmonyChatRenderer;
|
||||
pub use inkling::InklingChatRenderer;
|
||||
pub use kimi_k3::KimiK3ChatRenderer;
|
||||
|
||||
@@ -22,6 +22,8 @@ pub enum RendererSelection {
|
||||
DeepSeekV32,
|
||||
/// Force the DeepSeek V4 renderer.
|
||||
DeepSeekV4,
|
||||
/// Force the DeepSeek V4.1 renderer.
|
||||
DeepSeekV41,
|
||||
/// Force the GPT-OSS Harmony renderer.
|
||||
Harmony,
|
||||
/// Force the Inkling native token renderer.
|
||||
@@ -34,6 +36,7 @@ impl RendererSelection {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32";
|
||||
pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4";
|
||||
pub const DEEPSEEK_V41_LITERAL: &str = "deepseek_v41";
|
||||
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
|
||||
pub const HARMONY_LITERAL: &str = "harmony";
|
||||
pub const HF_LITERAL: &str = "hf";
|
||||
@@ -48,6 +51,7 @@ impl RendererSelection {
|
||||
Self::Auto => match model_type {
|
||||
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
|
||||
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
|
||||
Self::DEEPSEEK_V41_LITERAL => Self::DeepSeekV41,
|
||||
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
|
||||
Self::INKLING_MODEL_TYPE => Self::Inkling,
|
||||
Self::KIMI_K3_LITERAL => Self::KimiK3,
|
||||
@@ -70,6 +74,8 @@ impl FromStr for RendererSelection {
|
||||
Ok(Self::DeepSeekV32)
|
||||
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) {
|
||||
Ok(Self::DeepSeekV4)
|
||||
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V41_LITERAL) {
|
||||
Ok(Self::DeepSeekV41)
|
||||
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
|
||||
Ok(Self::Harmony)
|
||||
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
|
||||
@@ -92,6 +98,7 @@ impl fmt::Display for RendererSelection {
|
||||
Self::Hf => f.write_str(Self::HF_LITERAL),
|
||||
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
|
||||
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
|
||||
Self::DeepSeekV41 => f.write_str(Self::DEEPSEEK_V41_LITERAL),
|
||||
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
|
||||
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
|
||||
Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL),
|
||||
@@ -121,7 +128,7 @@ mod tests {
|
||||
fn renderer_selection_expected_error_message() {
|
||||
let err = RendererSelection::from_str("unknown").unwrap_err();
|
||||
expect_test::expect![
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)"
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)"
|
||||
]
|
||||
.assert_eq(&err);
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ const TOKEN_COMPLETION_CHUNK_TOKENS: usize = 1;
|
||||
/// One model/parser configuration used to run the fixed roundtrip fixtures.
|
||||
#[derive(Clone)]
|
||||
struct RoundtripCase {
|
||||
/// Hugging Face model id resolved through the production backend loader.
|
||||
/// Hugging Face model id or local checkpoint resolved through the production backend loader.
|
||||
model_id: &'static str,
|
||||
/// Final assistant-history suffix rendered by the chat template but not
|
||||
/// generated by the model body consumed by the output processor.
|
||||
@@ -180,6 +180,19 @@ impl RoundtripCase {
|
||||
}
|
||||
}
|
||||
|
||||
/// DeepSeek V4.1 Flash DSML tool-call format.
|
||||
fn deepseek_v41() -> Self {
|
||||
Self {
|
||||
model_id: "deepseek-ai/DeepSeek-V4.1-Flash",
|
||||
assistant_stop_suffix: "<|end▁of▁sentence|>",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// DeepSeek V3.2 DSML tool-call format.
|
||||
fn deepseek_v32() -> Self {
|
||||
Self {
|
||||
@@ -385,6 +398,7 @@ roundtrip_tests! {
|
||||
minimax_m25 => [reasoning_and_content, tool_call_mix],
|
||||
minimax_m3 => [reasoning_and_content, tool_call_mix],
|
||||
deepseek_v4 => [reasoning_and_content, tool_call_mix],
|
||||
deepseek_v41 => [reasoning_and_content, tool_call_mix],
|
||||
deepseek_v32 => [tool_call_mix],
|
||||
glm45 => [reasoning_and_content, tool_call_mix],
|
||||
glm47 => [reasoning_and_content, tool_call_mix],
|
||||
|
||||
Reference in New Issue
Block a user