[Model][Frontend] Support DeepSeek-V4.1-Flash in Rust and Python frontends (#56208)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -19,8 +19,8 @@ use crate::output::{
|
||||
};
|
||||
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
|
||||
use crate::renderer::{
|
||||
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
|
||||
InklingChatRenderer, KimiK3ChatRenderer,
|
||||
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer,
|
||||
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer,
|
||||
};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{DynChatOutputProcessor, RendererSelection};
|
||||
@@ -72,6 +72,7 @@ impl HfChatBackend {
|
||||
)?),
|
||||
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
|
||||
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
|
||||
RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()),
|
||||
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
|
||||
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
|
||||
RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
|
||||
|
||||
@@ -17,6 +17,8 @@ pub enum Error {
|
||||
MissingChatTemplate,
|
||||
#[error("chat template error: {0}")]
|
||||
ChatTemplate(String),
|
||||
#[error("{0}")]
|
||||
InvalidReasoningEffort(String),
|
||||
#[error("multimodal input is not supported by this chat renderer")]
|
||||
UnsupportedMultimodalRenderer,
|
||||
#[error("unsupported multimodal content: {0}")]
|
||||
@@ -96,6 +98,7 @@ impl Error {
|
||||
pub fn is_request_validation_error(&self) -> bool {
|
||||
match self {
|
||||
Self::PromptTooLong { .. }
|
||||
| Self::InvalidReasoningEffort(_)
|
||||
| Self::DuplicateToolName { .. }
|
||||
| Self::ToolChoiceRequiresTools
|
||||
| Self::ToolChoiceFunctionNotFound { .. } => true,
|
||||
|
||||
@@ -33,8 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use parser::{ParserSelection, validate_parser_overrides};
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
|
||||
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
|
||||
DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
|
||||
RendererSelection,
|
||||
};
|
||||
pub use request::{
|
||||
@@ -363,7 +363,7 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -374,6 +374,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,6 +153,10 @@ struct ResolvedMultimodalSpec {
|
||||
modality: Modality,
|
||||
field_layouts: EncoderFieldLayouts,
|
||||
keep_on_cpu_keys: HashSet<String>,
|
||||
/// Spec-declared wire key for the primary encoder input tensor, when it
|
||||
/// differs from the per-modality default (e.g. `"patches"` for
|
||||
/// DeepSeek-V4.1 images).
|
||||
encoder_input_key: Option<String>,
|
||||
}
|
||||
|
||||
impl ResolvedMultimodalSpec {
|
||||
@@ -162,10 +166,14 @@ impl ResolvedMultimodalSpec {
|
||||
modality,
|
||||
field_layouts: raw.encoder_field_layouts_for(modality),
|
||||
keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
|
||||
encoder_input_key: raw.encoder_input_key_for(modality),
|
||||
}
|
||||
}
|
||||
|
||||
fn primary_key(&self) -> &'static str {
|
||||
fn primary_key(&self) -> &str {
|
||||
if let Some(key) = &self.encoder_input_key {
|
||||
return key;
|
||||
}
|
||||
match self.modality {
|
||||
Modality::Image => image::IMAGE_PRIMARY_KEY,
|
||||
Modality::Video => video::VIDEO_PRIMARY_KEY,
|
||||
@@ -841,6 +849,9 @@ mod tests {
|
||||
pub(super) const QWEN3_IMAGE_PAD_ID: u32 = 151655;
|
||||
pub(super) const QWEN3_VIDEO_PAD_ID: u32 = 151656;
|
||||
|
||||
pub(super) const DEEPSEEK_V41_IMAGE_ID: u32 = 129264;
|
||||
pub(super) const DEEPSEEK_V41_IMAGE_PAD_ID: u32 = 129265;
|
||||
|
||||
fn llama4_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
|
||||
@@ -954,6 +965,33 @@ mod tests {
|
||||
assert_eq!(info.placeholder_token(Modality::Video), None);
|
||||
}
|
||||
|
||||
fn deepseek_v41_info() -> MultimodalModelInfo {
|
||||
let config = serde_json::json!({
|
||||
"model_type": "deepseek_v41",
|
||||
"image_token_id": DEEPSEEK_V41_IMAGE_ID,
|
||||
});
|
||||
let tokenizer = TestTokenizer::new()
|
||||
.with_regular_token("<|deepseek_image|>", DEEPSEEK_V41_IMAGE_ID)
|
||||
.with_regular_token("<|place_holder_mm_span_0436|>", DEEPSEEK_V41_IMAGE_PAD_ID);
|
||||
test_info("deepseek_v41", config, tokenizer)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepseek_v41_resolves_image_support_only() {
|
||||
let info = deepseek_v41_info();
|
||||
|
||||
assert_eq!(
|
||||
info.placeholder_token(Modality::Image),
|
||||
Some("<|deepseek_image|>")
|
||||
);
|
||||
assert_eq!(info.placeholder_token(Modality::Video), None);
|
||||
let image = info.image.as_ref().expect("image support");
|
||||
assert_eq!(image.placeholder.marker_token_id, DEEPSEEK_V41_IMAGE_ID);
|
||||
assert_eq!(image.placeholder.embed_token_id, DEEPSEEK_V41_IMAGE_ID);
|
||||
// The engine's forward kwargs pop `patches` (not `pixel_values`).
|
||||
assert_eq!(image.spec.primary_key(), "patches");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_audio_uses_connector_data_urls() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -45,6 +45,14 @@ impl<'a> ExpansionLane<'a> {
|
||||
/// matching the original media-part order within that modality; markers of
|
||||
/// different modalities may interleave freely.
|
||||
///
|
||||
/// Replacements may carry an [`AlignmentPad`](llm_multimodal::AlignmentPad)
|
||||
/// (e.g. DeepSeek-V4.1's compressor-alignment pad): `period - 1 - offset %
|
||||
/// period` pad tokens are
|
||||
/// prepended at splice time, where `offset` is the position the replacement
|
||||
/// lands at in the expanded prompt. The pads are part of the reported range
|
||||
/// and count as embed positions only if the pad token id equals the
|
||||
/// modality's embed token id.
|
||||
///
|
||||
/// The returned ranges point into the already-expanded prompt, grouped per
|
||||
/// modality in item order.
|
||||
pub(super) fn expand_prompt_token_ids(
|
||||
@@ -56,12 +64,14 @@ pub(super) fn expand_prompt_token_ids(
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
let replacement_growth = lanes
|
||||
.iter()
|
||||
.flat_map(|lane| lane.replacements.iter())
|
||||
.fold(0usize, |total, replacement| {
|
||||
total.saturating_add(replacement.tokens.len().saturating_sub(1))
|
||||
});
|
||||
let replacement_growth = lanes.iter().flat_map(|lane| lane.replacements.iter()).fold(
|
||||
0usize,
|
||||
|total, replacement| {
|
||||
// Alignment pads are position-dependent; reserve the worst case.
|
||||
let max_pads = replacement.alignment_pad.map_or(0, |pad| pad.period - 1);
|
||||
total.saturating_add(max_pads + replacement.tokens.len().saturating_sub(1))
|
||||
},
|
||||
);
|
||||
let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth);
|
||||
|
||||
let mut expanded = Vec::with_capacity(expanded_len);
|
||||
@@ -95,10 +105,20 @@ pub(super) fn expand_prompt_token_ids(
|
||||
expanded.len()
|
||||
);
|
||||
}
|
||||
let range_len = replacement_len.saturating_add(structural_prefix);
|
||||
// Position-dependent alignment padding, computed from the offset the
|
||||
// replacement block lands at in the expanded prompt (mirroring
|
||||
// vLLM's `_apply_token_matches_with_placeholders` splice-time pad).
|
||||
let pad_count = replacement.alignment_pad.map_or(0, |pad| pad.count_at(expanded.len()));
|
||||
let range_len = structural_prefix + pad_count + replacement_len;
|
||||
let is_embed = {
|
||||
let mut mask = Vec::with_capacity(range_len);
|
||||
mask.extend(std::iter::repeat_n(false, structural_prefix));
|
||||
if let Some(pad) = &replacement.alignment_pad {
|
||||
mask.extend(std::iter::repeat_n(
|
||||
pad.token_id as u32 == lane.embed_token_id,
|
||||
pad_count,
|
||||
));
|
||||
}
|
||||
mask.extend(
|
||||
replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
|
||||
);
|
||||
@@ -106,6 +126,9 @@ pub(super) fn expand_prompt_token_ids(
|
||||
};
|
||||
|
||||
let expanded_offset = expanded.len() - structural_prefix;
|
||||
if let Some(pad) = &replacement.alignment_pad {
|
||||
expanded.extend(std::iter::repeat_n(pad.token_id as u32, pad_count));
|
||||
}
|
||||
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
|
||||
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
|
||||
offset: expanded_offset,
|
||||
@@ -454,7 +477,6 @@ mod tests {
|
||||
}
|
||||
|
||||
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
|
||||
|
||||
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
|
||||
prepared_media(
|
||||
Modality::Audio,
|
||||
@@ -542,4 +564,110 @@ mod tests {
|
||||
));
|
||||
assert_eq!(prompt_token_ids, original_prompt_token_ids);
|
||||
}
|
||||
|
||||
// DeepSeek-V4.1 image spans: every span position carries the image token
|
||||
// id; a compressor-alignment pad (id 129265, period 2) is prepended at
|
||||
// splice time so spans always start at the same compressor phase.
|
||||
const DS_IMAGE_ID: u32 = 129_264;
|
||||
const DS_IMAGE_PAD_ID: u32 = 129_265;
|
||||
|
||||
fn deepseek_prepared(span_lens: Vec<usize>) -> PreparedMedia {
|
||||
prepared_media(
|
||||
Modality::Image,
|
||||
"<|deepseek_image|>",
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
span_lens
|
||||
.into_iter()
|
||||
.map(|len| {
|
||||
PromptReplacement::repeated(
|
||||
Modality::Image,
|
||||
"<|deepseek_image|>",
|
||||
DS_IMAGE_ID as TokenId,
|
||||
len,
|
||||
)
|
||||
.with_alignment_pad(DS_IMAGE_PAD_ID as TokenId, 2)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_skips_alignment_pad_at_odd_offset() {
|
||||
// Odd block start: already at the target compressor phase, no pad.
|
||||
let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2];
|
||||
let prepared = vec![deepseek_prepared(vec![3])];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
let ranges = &ranges[&Modality::Image];
|
||||
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![1, DS_IMAGE_ID, DS_IMAGE_ID, DS_IMAGE_ID, 2]
|
||||
);
|
||||
assert_eq!(ranges[0].offset, 1);
|
||||
assert_eq!(ranges[0].length, 3);
|
||||
assert_bool_mask(&ranges[0], &[true, true, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_prepends_alignment_pad_at_even_offset() {
|
||||
// Even block start: one pad token aligns the span to phase 1.
|
||||
let mut prompt_token_ids = vec![1, 3, DS_IMAGE_ID, 2];
|
||||
let prepared = vec![deepseek_prepared(vec![3])];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
let ranges = &ranges[&Modality::Image];
|
||||
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![
|
||||
1,
|
||||
3,
|
||||
DS_IMAGE_PAD_ID,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
2
|
||||
]
|
||||
);
|
||||
// The range starts at the pad; pads are not embed positions.
|
||||
assert_eq!(ranges[0].offset, 2);
|
||||
assert_eq!(ranges[0].length, 4);
|
||||
assert_bool_mask(&ranges[0], &[false, true, true, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expand_prompt_tokens_alignment_pad_realigns_each_span() {
|
||||
// Every image span must start at phase 1 (odd offset) regardless of
|
||||
// where the previous span landed.
|
||||
let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2, DS_IMAGE_ID, 3];
|
||||
let prepared = vec![deepseek_prepared(vec![2, 2])];
|
||||
|
||||
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
|
||||
let ranges = &ranges[&Modality::Image];
|
||||
|
||||
// First span at odd offset 1: no pad (block [1,3)).
|
||||
// Second marker originally at index 3; after the first expansion the
|
||||
// block starts at offset 4 (even): one pad, span at odd offset 5.
|
||||
assert_eq!(
|
||||
prompt_token_ids,
|
||||
vec![
|
||||
1,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
2,
|
||||
DS_IMAGE_PAD_ID,
|
||||
DS_IMAGE_ID,
|
||||
DS_IMAGE_ID,
|
||||
3,
|
||||
]
|
||||
);
|
||||
assert_eq!((ranges[0].offset, ranges[0].length), (1, 2));
|
||||
assert_bool_mask(&ranges[0], &[true, true]);
|
||||
assert_eq!((ranges[1].offset, ranges[1].length), (4, 3));
|
||||
assert_bool_mask(&ranges[1], &[false, true, true]);
|
||||
// Both spans (past the pad) start at odd offsets.
|
||||
assert_eq!((ranges[1].offset + 1) % 2, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,11 @@ use std::sync::{Arc, LazyLock};
|
||||
|
||||
pub use vllm_parser::reasoning::{
|
||||
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
|
||||
DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser,
|
||||
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
|
||||
NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
|
||||
ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
|
||||
DeepSeekV4ReasoningParser, DeepSeekV41ReasoningParser, Glm45ReasoningParser,
|
||||
Glm47ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser, MiniMaxM2ReasoningParser,
|
||||
MiniMaxM3ReasoningParser, NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta,
|
||||
ReasoningError, ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser,
|
||||
Step3p5ReasoningParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
@@ -22,6 +23,7 @@ pub mod names {
|
||||
pub const DEEPSEEK_R1: &str = "deepseek_r1";
|
||||
pub const DEEPSEEK_V3: &str = "deepseek_v3";
|
||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||
pub const DEEPSEEK_V41: &str = "deepseek_v41";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const GLM47: &str = "glm47";
|
||||
pub const KIMI: &str = "kimi";
|
||||
@@ -62,6 +64,7 @@ impl ReasoningParserFactory {
|
||||
.register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
|
||||
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
|
||||
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<DeepSeekV41ReasoningParser>(names::DEEPSEEK_V41)
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_parser::<Glm47ReasoningParser>(names::GLM47)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
@@ -76,6 +79,7 @@ impl ReasoningParserFactory {
|
||||
|
||||
factory
|
||||
.register_pattern("deepseek-r1", names::DEEPSEEK_R1)
|
||||
.register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
|
||||
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek-v3", names::DEEPSEEK_V3)
|
||||
|
||||
@@ -33,6 +33,10 @@ fn factory_resolves_deepseek_v4() {
|
||||
factory.resolve_name_for_model("deepseek_v4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
|
||||
Some(names::DEEPSEEK_V41)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock};
|
||||
|
||||
pub use vllm_parser::tool::{
|
||||
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
|
||||
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
|
||||
Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
|
||||
MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
|
||||
Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
|
||||
DeepSeekV41ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser,
|
||||
HermesToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
|
||||
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
|
||||
Qwen3CoderToolParser, Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
|
||||
};
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
@@ -22,6 +22,7 @@ pub mod names {
|
||||
pub const DEEPSEEK_V31: &str = "deepseek_v31";
|
||||
pub const DEEPSEEK_V32: &str = "deepseek_v32";
|
||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||
pub const DEEPSEEK_V41: &str = "deepseek_v41";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const GLM47: &str = "glm47";
|
||||
pub const GRANITE4: &str = "granite4";
|
||||
@@ -66,6 +67,7 @@ impl ToolParserFactory {
|
||||
.register_parser::<DeepSeekV31ToolParser>(names::DEEPSEEK_V31)
|
||||
.register_parser::<DeepSeekV32ToolParser>(names::DEEPSEEK_V32)
|
||||
.register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<DeepSeekV41ToolParser>(names::DEEPSEEK_V41)
|
||||
.register_parser::<Glm45MoeToolParser>(names::GLM45)
|
||||
.register_parser::<Glm47MoeToolParser>(names::GLM47)
|
||||
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
||||
@@ -101,6 +103,7 @@ impl ToolParserFactory {
|
||||
.register_pattern("llama-3.2", names::LLAMA3_JSON)
|
||||
.register_pattern("llama-3.1", names::LLAMA3_JSON)
|
||||
.register_pattern("deepseek-r1", names::DEEPSEEK_V3)
|
||||
.register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
|
||||
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
||||
.register_pattern("deepseek-v3.2", names::DEEPSEEK_V32)
|
||||
|
||||
@@ -147,6 +147,10 @@ fn factory_new_resolves_default_patterns() {
|
||||
factory.resolve_name_for_model("deepseek_v4"),
|
||||
Some(names::DEEPSEEK_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
|
||||
Some(names::DEEPSEEK_V41)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("deepseek-ai/DeepSeek-R1-0528"),
|
||||
Some(names::DEEPSEEK_V3)
|
||||
|
||||
+230
-59
@@ -1,11 +1,12 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! DeepSeek V4 prompt renderer.
|
||||
//! Shared DeepSeek V4 and V4.1 prompt rendering.
|
||||
//!
|
||||
//! Official Python reference:
|
||||
//! <https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py>
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
@@ -13,8 +14,12 @@ use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use serde_json_fmt::JsonFormat;
|
||||
|
||||
use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, ReasoningEffort};
|
||||
use crate::request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort,
|
||||
};
|
||||
use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
|
||||
|
||||
const BOS_TOKEN: &str = "<|begin▁of▁sentence|>";
|
||||
@@ -23,6 +28,7 @@ const THINKING_START_TOKEN: &str = "<think>";
|
||||
const THINKING_END_TOKEN: &str = "</think>";
|
||||
const DSML_TOKEN: &str = "|DSML|";
|
||||
const USER_SP_TOKEN: &str = "<|User|>";
|
||||
const SYSTEM_SP_TOKEN: &str = "<|System|>";
|
||||
const ASSISTANT_SP_TOKEN: &str = "<|Assistant|>";
|
||||
const REASONING_EFFORT_HIGH: &str = concat!(
|
||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n",
|
||||
@@ -41,6 +47,35 @@ enum ThinkingMode {
|
||||
Thinking,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum DsDialect {
|
||||
V4,
|
||||
V41,
|
||||
}
|
||||
|
||||
impl DsDialect {
|
||||
fn tool_calls_tag(self) -> &'static str {
|
||||
match self {
|
||||
Self::V4 => "tool_calls",
|
||||
Self::V41 => " calls",
|
||||
}
|
||||
}
|
||||
|
||||
fn invoke_tag(self) -> &'static str {
|
||||
match self {
|
||||
Self::V4 => "invoke",
|
||||
Self::V41 => " invoke",
|
||||
}
|
||||
}
|
||||
|
||||
fn parameter_tag(self) -> &'static str {
|
||||
match self {
|
||||
Self::V4 => "parameter",
|
||||
Self::V41 => " parameter",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RenderedToolSchema<'a> {
|
||||
@@ -51,25 +86,47 @@ struct RenderedToolSchema<'a> {
|
||||
}
|
||||
|
||||
/// Render one chat request into the final prompt string.
|
||||
pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
let (thinking_mode, reasoning_effort_prompt) = resolve_thinking_options(request)?;
|
||||
pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result<String> {
|
||||
let (thinking_mode, reasoning_effort_prompt) = match dialect {
|
||||
DsDialect::V4 => {
|
||||
resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))?
|
||||
}
|
||||
DsDialect::V41 => resolve_v41_thinking_options(request)?,
|
||||
};
|
||||
let request_tools = request_tools(request);
|
||||
let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools);
|
||||
let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true)
|
||||
&& !rendered_tools_present(request, request_tools);
|
||||
let drop_historical_developers = thinking_mode == ThinkingMode::Thinking && drop_thinking;
|
||||
let last_user_like_message_index = request.messages.iter().rposition(is_user_like_entry);
|
||||
let last_user_render_index =
|
||||
find_last_user_render_index(request.messages.as_slice(), synthetic_tool_system);
|
||||
let last_user_like_message_index = request
|
||||
.messages
|
||||
.iter()
|
||||
.enumerate()
|
||||
.rfind(|(index, message)| is_user_like_entry(message, *index, dialect))
|
||||
.map(|(index, _)| index);
|
||||
let last_user_render_index = find_last_user_render_index(
|
||||
request.messages.as_slice(),
|
||||
synthetic_tool_system,
|
||||
drop_historical_developers,
|
||||
last_user_like_message_index,
|
||||
dialect,
|
||||
);
|
||||
let mut out = String::from(BOS_TOKEN);
|
||||
if dialect == DsDialect::V41
|
||||
&& (thinking_mode == ThinkingMode::Thinking
|
||||
|| synthetic_tool_system
|
||||
|| matches!(request.messages.first(), Some(ChatMessage::System { .. })))
|
||||
{
|
||||
out.push_str(SYSTEM_SP_TOKEN);
|
||||
}
|
||||
if thinking_mode == ThinkingMode::Thinking {
|
||||
out.push_str(reasoning_effort_prompt);
|
||||
out.push_str(&reasoning_effort_prompt);
|
||||
}
|
||||
|
||||
let mut request_tools_attached = false;
|
||||
let mut render_index = 0isize;
|
||||
if synthetic_tool_system {
|
||||
render_system_message(&mut out, None, request_tools)?;
|
||||
render_system_message(&mut out, None, request_tools, dialect)?;
|
||||
request_tools_attached = true;
|
||||
render_index += 1;
|
||||
}
|
||||
@@ -80,27 +137,35 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
continue;
|
||||
}
|
||||
|
||||
let current_render_index = render_index;
|
||||
render_index += 1;
|
||||
|
||||
if drop_historical_developers
|
||||
&& is_historical_developer(message, message_index, last_user_like_message_index)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let current_render_index = render_index;
|
||||
render_index += 1;
|
||||
|
||||
match message {
|
||||
ChatMessage::System { content } => {
|
||||
if dialect == DsDialect::V41 && current_render_index > 0 {
|
||||
out.push_str(SYSTEM_SP_TOKEN);
|
||||
}
|
||||
let tools = if !request_tools_attached {
|
||||
request_tools_attached = true;
|
||||
request_tools
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
render_system_message(&mut out, Some(content), tools)?;
|
||||
render_system_message(&mut out, Some(content), tools, dialect)?;
|
||||
}
|
||||
ChatMessage::Developer { content, tools } => {
|
||||
render_developer_message(&mut out, content, tools.as_deref().unwrap_or(&[]))?;
|
||||
render_developer_message(
|
||||
&mut out,
|
||||
content,
|
||||
tools.as_deref().unwrap_or(&[]),
|
||||
dialect,
|
||||
)?;
|
||||
}
|
||||
ChatMessage::User { .. } | ChatMessage::ToolResponse { .. } => {
|
||||
render_user_content_block(
|
||||
@@ -108,6 +173,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
request.messages.as_slice(),
|
||||
message_index,
|
||||
&last_tool_call_order,
|
||||
dialect,
|
||||
)?;
|
||||
}
|
||||
ChatMessage::Assistant { content } => {
|
||||
@@ -119,7 +185,13 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
&& (!drop_thinking || current_render_index > last_user_render_index);
|
||||
let append_eos = !(message_index + 1 == request.messages.len()
|
||||
&& request.chat_options.continue_final_message());
|
||||
render_assistant_message(&mut out, emit_thinking_block, append_eos, content)?;
|
||||
render_assistant_message(
|
||||
&mut out,
|
||||
emit_thinking_block,
|
||||
append_eos,
|
||||
content,
|
||||
dialect,
|
||||
)?;
|
||||
|
||||
if content.has_tool_calls() {
|
||||
last_tool_call_order.clear();
|
||||
@@ -133,7 +205,8 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
}
|
||||
}
|
||||
|
||||
if (is_user_like_entry(message) || matches!(message, ChatMessage::System { .. }))
|
||||
if (is_user_like_entry(message, current_render_index as usize, dialect)
|
||||
|| (dialect == DsDialect::V4 && matches!(message, ChatMessage::System { .. })))
|
||||
&& next_rendered_entry_is_assistant_or_end(
|
||||
request.messages.as_slice(),
|
||||
message_index,
|
||||
@@ -181,6 +254,57 @@ fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'st
|
||||
Ok((thinking_mode, reasoning_effort_prompt))
|
||||
}
|
||||
|
||||
/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs.
|
||||
fn resolve_v41_thinking_options(
|
||||
request: &ChatRequest,
|
||||
) -> Result<(ThinkingMode, Cow<'static, str>)> {
|
||||
let mut thinking = request.enable_thinking()?.unwrap_or(true);
|
||||
let budget = match request.chat_options.reasoning_effort {
|
||||
Some(ReasoningEffort::None) => {
|
||||
thinking = false;
|
||||
50
|
||||
}
|
||||
Some(ReasoningEffort::Low) => 25,
|
||||
Some(ReasoningEffort::High) => 50,
|
||||
Some(ReasoningEffort::XHigh) => 75,
|
||||
Some(ReasoningEffort::Max) => 100,
|
||||
Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => {
|
||||
return Err(invalid_v41_reasoning_effort());
|
||||
}
|
||||
None => match request.chat_options.template_kwargs.get("reasoning_effort") {
|
||||
None | Some(Value::Null) => 50,
|
||||
Some(Value::String(effort)) => match effort.as_str() {
|
||||
"low" => 25,
|
||||
"high" => 50,
|
||||
"xhigh" => 75,
|
||||
"max" => 100,
|
||||
_ => return Err(invalid_v41_reasoning_effort()),
|
||||
},
|
||||
Some(Value::Number(budget)) => budget
|
||||
.as_u64()
|
||||
.filter(|budget| (1..=100).contains(budget))
|
||||
.ok_or_else(invalid_v41_reasoning_effort)?,
|
||||
Some(_) => return Err(invalid_v41_reasoning_effort()),
|
||||
},
|
||||
};
|
||||
if thinking {
|
||||
Ok((
|
||||
ThinkingMode::Thinking,
|
||||
Cow::Owned(format!(
|
||||
"Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
|
||||
)),
|
||||
))
|
||||
} else {
|
||||
Ok((ThinkingMode::Chat, Cow::Borrowed("")))
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_v41_reasoning_effort() -> Error {
|
||||
Error::InvalidReasoningEffort(
|
||||
"DeepSeek V4.1 reasoning_effort must be low, high, xhigh, max, or an integer within [1, 100] in chat_template_kwargs".to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Return request-level tools only when native tool parsing is enabled.
|
||||
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
|
||||
if request.tool_parsing_enabled() {
|
||||
@@ -214,16 +338,25 @@ fn rendered_tools_present(request: &ChatRequest, request_tools: &[ChatTool]) ->
|
||||
}
|
||||
|
||||
/// Find the last user-like turn after inline tool-response merging.
|
||||
fn find_last_user_render_index(messages: &[ChatMessage], synthetic_tool_system: bool) -> isize {
|
||||
fn find_last_user_render_index(
|
||||
messages: &[ChatMessage],
|
||||
synthetic_tool_system: bool,
|
||||
drop_historical_developers: bool,
|
||||
last_user_like_message_index: Option<usize>,
|
||||
dialect: DsDialect,
|
||||
) -> isize {
|
||||
let mut render_index = isize::from(synthetic_tool_system);
|
||||
let mut last_user_index = -1;
|
||||
|
||||
for (message_index, message) in messages.iter().enumerate() {
|
||||
if is_following_user_content(messages, message_index) {
|
||||
if is_following_user_content(messages, message_index)
|
||||
|| (drop_historical_developers
|
||||
&& is_historical_developer(message, message_index, last_user_like_message_index))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if is_user_like_entry(message) {
|
||||
if is_user_like_entry(message, render_index as usize, dialect) {
|
||||
last_user_index = render_index;
|
||||
}
|
||||
render_index += 1;
|
||||
@@ -249,11 +382,13 @@ fn is_user_content_entry(message: &ChatMessage) -> bool {
|
||||
}
|
||||
|
||||
/// Return whether one rendered entry should be treated as user-like.
|
||||
fn is_user_like_entry(message: &ChatMessage) -> bool {
|
||||
fn is_user_like_entry(message: &ChatMessage, message_index: usize, dialect: DsDialect) -> bool {
|
||||
matches!(
|
||||
message,
|
||||
ChatMessage::Developer { .. } | ChatMessage::User { .. } | ChatMessage::ToolResponse { .. }
|
||||
)
|
||||
) || (dialect == DsDialect::V41
|
||||
&& message_index > 0
|
||||
&& matches!(message, ChatMessage::System { .. }))
|
||||
}
|
||||
|
||||
/// Return whether a developer entry precedes another user-like turn.
|
||||
@@ -293,22 +428,26 @@ fn next_rendered_entry_is_assistant_or_end(
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
/// Render the tool preamble shown to the model, V4 flavor.
|
||||
fn render_tools(out: &mut String, tools: &[ChatTool]) -> Result<()> {
|
||||
out.push_str(
|
||||
/// Render the tool preamble shown to the model for one DeepSeek dialect.
|
||||
fn render_tools(out: &mut String, tools: &[ChatTool], dialect: DsDialect) -> Result<()> {
|
||||
let tool_calls_tag = dialect.tool_calls_tag();
|
||||
let invoke_tag = dialect.invoke_tag();
|
||||
let parameter_tag = dialect.parameter_tag();
|
||||
write!(
|
||||
out,
|
||||
r#"## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|{tool_calls_tag}>" block like the following:
|
||||
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="$TOOL_NAME">
|
||||
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
|
||||
<|DSML|{tool_calls_tag}>
|
||||
<|DSML|{invoke_tag} name="$TOOL_NAME">
|
||||
<|DSML|{parameter_tag} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|{parameter_tag}>
|
||||
...
|
||||
</|DSML|invoke>
|
||||
<|DSML|invoke name="$TOOL_NAME2">
|
||||
</|DSML|{invoke_tag}>
|
||||
<|DSML|{invoke_tag} name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls>
|
||||
</|DSML|{invoke_tag}>
|
||||
</|DSML|{tool_calls_tag}>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
@@ -319,7 +458,8 @@ Otherwise, output directly after </think> with tool calls or final response.
|
||||
### Available Tool Schemas
|
||||
|
||||
"#,
|
||||
);
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
|
||||
for (index, tool) in tools.iter().enumerate() {
|
||||
if index > 0 {
|
||||
@@ -350,13 +490,14 @@ fn render_system_message(
|
||||
out: &mut String,
|
||||
content: Option<&ChatContent>,
|
||||
tools: &[ChatTool],
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
if let Some(content) = content {
|
||||
write_chat_content(out, content)?;
|
||||
write_chat_content(out, content, dialect)?;
|
||||
}
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
render_tools(out, tools, dialect)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -366,18 +507,19 @@ fn render_developer_message(
|
||||
out: &mut String,
|
||||
content: &ChatContent,
|
||||
tools: &[ChatTool],
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
if content.is_empty() {
|
||||
return Err(Error::ChatTemplate(
|
||||
"invalid DeepSeek V4 developer message: empty content".to_string(),
|
||||
"invalid DeepSeek developer message: empty content".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
out.push_str(USER_SP_TOKEN);
|
||||
write_chat_content(out, content)?;
|
||||
write_chat_content(out, content, dialect)?;
|
||||
if !tools.is_empty() {
|
||||
out.push_str("\n\n");
|
||||
render_tools(out, tools)?;
|
||||
render_tools(out, tools, dialect)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -388,6 +530,7 @@ fn render_user_content_block(
|
||||
messages: &[ChatMessage],
|
||||
message_index: usize,
|
||||
tool_call_order: &HashMap<String, usize>,
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let (block_start, block_end) = user_content_block_bounds(messages, message_index);
|
||||
let mut sorted_tool_indices =
|
||||
@@ -399,7 +542,7 @@ fn render_user_content_block(
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
match &messages[message_index] {
|
||||
ChatMessage::User { content } => write_chat_content(out, content)?,
|
||||
ChatMessage::User { content } => write_chat_content(out, content, dialect)?,
|
||||
ChatMessage::ToolResponse { .. } => {
|
||||
let sorted_index = sorted_tool_indices
|
||||
.next()
|
||||
@@ -407,7 +550,7 @@ fn render_user_content_block(
|
||||
let ChatMessage::ToolResponse { content, .. } = &messages[sorted_index] else {
|
||||
unreachable!("sorted tool response index should reference a tool message");
|
||||
};
|
||||
write_tool_result(out, content)?;
|
||||
write_tool_result(out, content, dialect)?;
|
||||
}
|
||||
_ => unreachable!("user content block should only contain user content messages"),
|
||||
}
|
||||
@@ -454,9 +597,9 @@ fn sorted_tool_response_indices(
|
||||
}
|
||||
|
||||
/// Render one tool response payload inside a V4 `<tool_result>` block.
|
||||
fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
fn write_tool_result(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
|
||||
out.push_str("<tool_result>");
|
||||
write_chat_content(out, content)?;
|
||||
write_chat_content(out, content, dialect)?;
|
||||
out.push_str("</tool_result>");
|
||||
Ok(())
|
||||
}
|
||||
@@ -483,6 +626,7 @@ fn render_assistant_message(
|
||||
emit_thinking_block: bool,
|
||||
append_eos: bool,
|
||||
content: &[AssistantContentBlock],
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let has_tool_calls = content.has_tool_calls();
|
||||
|
||||
@@ -496,14 +640,15 @@ fn render_assistant_message(
|
||||
write_assistant_text(out, content);
|
||||
|
||||
if has_tool_calls {
|
||||
out.push_str("\n\n<|DSML|tool_calls>\n");
|
||||
let tool_calls_tag = dialect.tool_calls_tag();
|
||||
writeln!(out, "\n\n<{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
|
||||
for (index, tool_call) in content.tool_calls().enumerate() {
|
||||
if index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
render_tool_call(out, tool_call)?;
|
||||
render_tool_call(out, tool_call, dialect)?;
|
||||
}
|
||||
out.push_str("\n</|DSML|tool_calls>");
|
||||
write!(out, "\n</{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
|
||||
}
|
||||
|
||||
if append_eos {
|
||||
@@ -513,11 +658,20 @@ fn render_assistant_message(
|
||||
}
|
||||
|
||||
/// Render one assistant tool call in DSML XML-like format.
|
||||
fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
writeln!(out, "<{DSML_TOKEN}invoke name=\"{}\">", tool_call.name)
|
||||
.expect("writing to String cannot fail");
|
||||
encode_arguments_to_dsml(out, tool_call)?;
|
||||
write!(out, "\n</{DSML_TOKEN}invoke>").expect("writing to String cannot fail");
|
||||
fn render_tool_call(
|
||||
out: &mut String,
|
||||
tool_call: &AssistantToolCall,
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let invoke_tag = dialect.invoke_tag();
|
||||
writeln!(
|
||||
out,
|
||||
"<{DSML_TOKEN}{invoke_tag} name=\"{}\">",
|
||||
tool_call.name
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
encode_arguments_to_dsml(out, tool_call, dialect)?;
|
||||
write!(out, "\n</{DSML_TOKEN}{invoke_tag}>").expect("writing to String cannot fail");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -525,15 +679,20 @@ fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<(
|
||||
///
|
||||
/// String values are emitted raw with `string="true"`, while all other JSON
|
||||
/// values are rendered with JSON syntax and `string="false"`.
|
||||
fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
|
||||
fn encode_arguments_to_dsml(
|
||||
out: &mut String,
|
||||
tool_call: &AssistantToolCall,
|
||||
dialect: DsDialect,
|
||||
) -> Result<()> {
|
||||
let parameter_tag = dialect.parameter_tag();
|
||||
let arguments: Value = serde_json::from_str(&tool_call.arguments).map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"assistant tool call has invalid JSON arguments for DeepSeek V4: {error}"
|
||||
"assistant tool call has invalid JSON arguments for DeepSeek: {error}"
|
||||
))
|
||||
})?;
|
||||
let Some(arguments) = arguments.as_object() else {
|
||||
return Err(Error::ChatTemplate(
|
||||
"assistant tool call arguments for DeepSeek V4 must be a JSON object".to_string(),
|
||||
"assistant tool call arguments for DeepSeek must be a JSON object".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
@@ -546,7 +705,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
|
||||
let is_string = matches!(value, Value::String(_));
|
||||
write!(
|
||||
out,
|
||||
"<{DSML_TOKEN}parameter name=\"{key}\" string=\"{}\">",
|
||||
"<{DSML_TOKEN}{parameter_tag} name=\"{key}\" string=\"{}\">",
|
||||
if is_string { "true" } else { "false" }
|
||||
)
|
||||
.expect("writing to String cannot fail");
|
||||
@@ -556,7 +715,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
|
||||
value => out.push_str(&json_dumps(value)?),
|
||||
}
|
||||
|
||||
write!(out, "</{DSML_TOKEN}parameter>").expect("writing to String cannot fail");
|
||||
write!(out, "</{DSML_TOKEN}{parameter_tag}>").expect("writing to String cannot fail");
|
||||
wrote_parameter = true;
|
||||
}
|
||||
|
||||
@@ -565,12 +724,24 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
|
||||
|
||||
/// Write chat content directly into the destination buffer without flattening
|
||||
/// it into an intermediate `String`.
|
||||
fn write_chat_content(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
///
|
||||
/// The V4.1 dialect inlines the image placeholder at each image part's
|
||||
/// position unconditionally (matching the Python encoding's
|
||||
/// `IMAGE_PLACEHOLDER`); other dialects reject multimodal parts.
|
||||
fn write_chat_content(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
|
||||
match content {
|
||||
ChatContent::Text(text) => out.push_str(text),
|
||||
ChatContent::Parts(parts) => {
|
||||
for part in parts {
|
||||
out.push_str(part.as_text()?);
|
||||
for (index, part) in parts.iter().enumerate() {
|
||||
if index > 0 && dialect == DsDialect::V41 {
|
||||
out.push_str("\n\n");
|
||||
}
|
||||
match part {
|
||||
ChatContentPart::ImageUrl { .. } if dialect == DsDialect::V41 => {
|
||||
out.push_str(DEEPSEEK_V41_IMAGE_PLACEHOLDER);
|
||||
}
|
||||
_ => out.push_str(part.as_text()?),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -606,7 +777,7 @@ fn json_dumps<T: Serialize>(value: &T) -> Result<String> {
|
||||
.format_to_string(value)
|
||||
.map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"failed to serialize DeepSeek V4 JSON payload: {error}"
|
||||
"failed to serialize DeepSeek JSON payload: {error}"
|
||||
))
|
||||
})
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
mod encoding;
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::deepseek::{self, DsDialect};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
@@ -24,7 +23,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
|
||||
request.validate()?;
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(encoding::render_request(request)?),
|
||||
prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="$TOOL_NAME">
|
||||
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
|
||||
...
|
||||
</|DSML| invoke>
|
||||
<|DSML| invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
|
||||
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="get_weather">
|
||||
<|DSML| parameter name="location" string="true">Beijing</|DSML| parameter>
|
||||
<|DSML| parameter name="unit" string="true">celsius</|DSML| parameter>
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,3 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,30 @@
|
||||
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
|
||||
|
||||
<|User|>question<|User|>policy
|
||||
|
||||
## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
|
||||
|
||||
<|DSML| calls>
|
||||
<|DSML| invoke name="$TOOL_NAME">
|
||||
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
|
||||
...
|
||||
</|DSML| invoke>
|
||||
<|DSML| invoke name="$TOOL_NAME2">
|
||||
...
|
||||
</|DSML| invoke>
|
||||
</|DSML| calls>
|
||||
|
||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
||||
|
||||
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
|
||||
|
||||
Otherwise, output directly after </think> with tool calls or final response.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
{"name": "lookup", "parameters": {"type": "object"}}
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
||||
<|Assistant|><think></think>answer<|end▁of▁sentence|>
|
||||
@@ -0,0 +1,33 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use super::deepseek::{self, DsDialect};
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::Result;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
/// Dedicated DeepSeek V4.1 renderer.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct DeepSeekV41ChatRenderer;
|
||||
|
||||
impl DeepSeekV41ChatRenderer {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for DeepSeekV41ChatRenderer {
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
request.validate()?;
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V41)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,223 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use expect_test::{expect, expect_file};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::DeepSeekV41ChatRenderer;
|
||||
use crate::ChatRenderer;
|
||||
use crate::event::AssistantContentBlock;
|
||||
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ReasoningEffort};
|
||||
|
||||
fn render(request: &ChatRequest) -> String {
|
||||
DeepSeekV41ChatRenderer::new()
|
||||
.render(request)
|
||||
.unwrap()
|
||||
.prompt
|
||||
.into_text()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn request() -> ChatRequest {
|
||||
ChatRequest {
|
||||
messages: vec![ChatMessage::user("question")],
|
||||
..ChatRequest::for_test()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_shared_deepseek_fixtures_with_v41_reference_encoding() {
|
||||
let fixture_dir =
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/renderer/deepseek_v4/fixtures");
|
||||
for (input, expected) in [
|
||||
(
|
||||
"test_input_1.json",
|
||||
expect_file!["fixtures/test_output_1.txt"],
|
||||
),
|
||||
(
|
||||
"test_input_2.json",
|
||||
expect_file!["fixtures/test_output_2.txt"],
|
||||
),
|
||||
(
|
||||
"test_input_developer_tools.json",
|
||||
expect_file!["fixtures/test_output_developer_tools.txt"],
|
||||
),
|
||||
] {
|
||||
let request = fixture_chat_request(
|
||||
&fixture_dir.join(input),
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: Some(true),
|
||||
no_generation_prompt_when_last_assistant: true,
|
||||
},
|
||||
);
|
||||
expected.assert_eq(&render(&request));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inlines_image_placeholder_at_image_part_positions() {
|
||||
// The placeholder is unconditional (matching the Python encoding's
|
||||
// `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later,
|
||||
// at media preparation.
|
||||
let renderer = DeepSeekV41ChatRenderer::new();
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::User {
|
||||
content: ChatContent::Parts(vec![
|
||||
ChatContentPart::image_url("https://example.com/carrots.jpeg"),
|
||||
ChatContentPart::text("what is in the image?"),
|
||||
]),
|
||||
}],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let rendered = renderer.render(&request).unwrap().prompt.into_text().unwrap();
|
||||
|
||||
// Image parts inline the placeholder at their position; parts stay
|
||||
// separated by "\n\n" (the V4.1 reference encoding's part separator).
|
||||
assert!(
|
||||
rendered.contains("<|User|><|deepseek_image|>\n\nwhat is in the image?"),
|
||||
"unexpected prompt: {rendered:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_reasoning_effort_to_reference_numeric_budget() {
|
||||
for (effort, budget) in [
|
||||
(None, 50),
|
||||
(Some(ReasoningEffort::Low), 25),
|
||||
(Some(ReasoningEffort::High), 50),
|
||||
(Some(ReasoningEffort::XHigh), 75),
|
||||
(Some(ReasoningEffort::Max), 100),
|
||||
] {
|
||||
let mut request = request();
|
||||
request.chat_options.reasoning_effort = effort;
|
||||
assert_eq!(
|
||||
render(&request),
|
||||
format!(
|
||||
"<|begin▁of▁sentence|><|System|>Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>question<|Assistant|><think>"
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_numeric_template_effort_with_top_level_precedence() {
|
||||
for (effort, budget) in [
|
||||
(json!(1), 1),
|
||||
(json!(42), 42),
|
||||
(json!(100), 100),
|
||||
(json!("low"), 25),
|
||||
] {
|
||||
let mut request = request();
|
||||
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
|
||||
assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100")));
|
||||
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
|
||||
assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
|
||||
for effort in [
|
||||
json!("medium"),
|
||||
json!("minimal"),
|
||||
json!(-1),
|
||||
json!(0),
|
||||
json!(101),
|
||||
json!(true),
|
||||
json!(1.5),
|
||||
] {
|
||||
let mut request = request();
|
||||
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
|
||||
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
|
||||
assert!(error.is_request_validation_error());
|
||||
}
|
||||
for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
|
||||
let mut request = request();
|
||||
request.chat_options.reasoning_effort = Some(effort);
|
||||
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
|
||||
assert!(error.is_request_validation_error());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_mode_and_none_effort_use_closed_thinking_prefix() {
|
||||
let mut request = request();
|
||||
request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
|
||||
let expected = expect!["<|begin▁of▁sentence|><|User|>question<|Assistant|></think>"];
|
||||
expected.assert_eq(&render(&request));
|
||||
|
||||
request.chat_options.template_kwargs.insert("thinking".into(), json!(true));
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
|
||||
expected.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leading_system_in_chat_mode_uses_system_token() {
|
||||
let mut request = request();
|
||||
request.messages.insert(0, ChatMessage::system("policy"));
|
||||
request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
|
||||
expect!["<|begin▁of▁sentence|><|System|>policy<|User|>question<|Assistant|></think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_after_dropped_developer_becomes_leading_system() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::developer("old policy", None),
|
||||
ChatMessage::system("sys"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys"]
|
||||
.assert_eq(&render(&request));
|
||||
|
||||
request.messages.push(ChatMessage::user("question"));
|
||||
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>question<|Assistant|><think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_system_advances_last_user_boundary_and_drops_old_reasoning() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::system("sys"),
|
||||
ChatMessage::user("q1"),
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning { text: "r1".into() },
|
||||
AssistantContentBlock::Text { text: "a1".into() },
|
||||
]),
|
||||
ChatMessage::developer("old policy", None),
|
||||
ChatMessage::system("mid sys"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("reasoning_effort".into(), json!(88));
|
||||
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 88 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>q1<|Assistant|></think>a1<|end▁of▁sentence|><|System|>mid sys<|Assistant|><think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn joins_text_parts_with_reference_separator_and_preserves_literal_tags() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![ChatMessage::user(ChatContent::Parts(vec![
|
||||
ChatContentPart::text("first"),
|
||||
ChatContentPart::text("<|DSML|invoke name=\"literal\">"),
|
||||
]))],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("thinking".into(), Value::Bool(false));
|
||||
expect!["<|begin▁of▁sentence|><|User|>first\n\n<|DSML|invoke name=\"literal\"><|Assistant|></think>"]
|
||||
.assert_eq(&render(&request));
|
||||
}
|
||||
@@ -10,8 +10,10 @@ use vllm_text::Prompt;
|
||||
use crate::error::Result;
|
||||
use crate::request::{ChatRequest, ReasoningEffort};
|
||||
|
||||
mod deepseek;
|
||||
pub mod deepseek_v32;
|
||||
pub mod deepseek_v4;
|
||||
pub mod deepseek_v41;
|
||||
pub mod harmony;
|
||||
pub mod hf;
|
||||
mod inkling;
|
||||
@@ -22,6 +24,7 @@ mod test_utils;
|
||||
|
||||
pub use deepseek_v4::DeepSeekV4ChatRenderer;
|
||||
pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use deepseek_v41::DeepSeekV41ChatRenderer;
|
||||
pub use harmony::HarmonyChatRenderer;
|
||||
pub use inkling::InklingChatRenderer;
|
||||
pub use kimi_k3::KimiK3ChatRenderer;
|
||||
|
||||
@@ -22,6 +22,8 @@ pub enum RendererSelection {
|
||||
DeepSeekV32,
|
||||
/// Force the DeepSeek V4 renderer.
|
||||
DeepSeekV4,
|
||||
/// Force the DeepSeek V4.1 renderer.
|
||||
DeepSeekV41,
|
||||
/// Force the GPT-OSS Harmony renderer.
|
||||
Harmony,
|
||||
/// Force the Inkling native token renderer.
|
||||
@@ -34,6 +36,7 @@ impl RendererSelection {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32";
|
||||
pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4";
|
||||
pub const DEEPSEEK_V41_LITERAL: &str = "deepseek_v41";
|
||||
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
|
||||
pub const HARMONY_LITERAL: &str = "harmony";
|
||||
pub const HF_LITERAL: &str = "hf";
|
||||
@@ -48,6 +51,7 @@ impl RendererSelection {
|
||||
Self::Auto => match model_type {
|
||||
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
|
||||
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
|
||||
Self::DEEPSEEK_V41_LITERAL => Self::DeepSeekV41,
|
||||
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
|
||||
Self::INKLING_MODEL_TYPE => Self::Inkling,
|
||||
Self::KIMI_K3_LITERAL => Self::KimiK3,
|
||||
@@ -70,6 +74,8 @@ impl FromStr for RendererSelection {
|
||||
Ok(Self::DeepSeekV32)
|
||||
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) {
|
||||
Ok(Self::DeepSeekV4)
|
||||
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V41_LITERAL) {
|
||||
Ok(Self::DeepSeekV41)
|
||||
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
|
||||
Ok(Self::Harmony)
|
||||
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
|
||||
@@ -92,6 +98,7 @@ impl fmt::Display for RendererSelection {
|
||||
Self::Hf => f.write_str(Self::HF_LITERAL),
|
||||
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
|
||||
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
|
||||
Self::DeepSeekV41 => f.write_str(Self::DEEPSEEK_V41_LITERAL),
|
||||
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
|
||||
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
|
||||
Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL),
|
||||
@@ -121,7 +128,7 @@ mod tests {
|
||||
fn renderer_selection_expected_error_message() {
|
||||
let err = RendererSelection::from_str("unknown").unwrap_err();
|
||||
expect_test::expect![
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)"
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)"
|
||||
]
|
||||
.assert_eq(&err);
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ const TOKEN_COMPLETION_CHUNK_TOKENS: usize = 1;
|
||||
/// One model/parser configuration used to run the fixed roundtrip fixtures.
|
||||
#[derive(Clone)]
|
||||
struct RoundtripCase {
|
||||
/// Hugging Face model id resolved through the production backend loader.
|
||||
/// Hugging Face model id or local checkpoint resolved through the production backend loader.
|
||||
model_id: &'static str,
|
||||
/// Final assistant-history suffix rendered by the chat template but not
|
||||
/// generated by the model body consumed by the output processor.
|
||||
@@ -180,6 +180,19 @@ impl RoundtripCase {
|
||||
}
|
||||
}
|
||||
|
||||
/// DeepSeek V4.1 Flash DSML tool-call format.
|
||||
fn deepseek_v41() -> Self {
|
||||
Self {
|
||||
model_id: "deepseek-ai/DeepSeek-V4.1-Flash",
|
||||
assistant_stop_suffix: "<|end▁of▁sentence|>",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// DeepSeek V3.2 DSML tool-call format.
|
||||
fn deepseek_v32() -> Self {
|
||||
Self {
|
||||
@@ -385,6 +398,7 @@ roundtrip_tests! {
|
||||
minimax_m25 => [reasoning_and_content, tool_call_mix],
|
||||
minimax_m3 => [reasoning_and_content, tool_call_mix],
|
||||
deepseek_v4 => [reasoning_and_content, tool_call_mix],
|
||||
deepseek_v41 => [reasoning_and_content, tool_call_mix],
|
||||
deepseek_v32 => [tool_call_mix],
|
||||
glm45 => [reasoning_and_content, tool_call_mix],
|
||||
glm47 => [reasoning_and_content, tool_call_mix],
|
||||
|
||||
@@ -903,7 +903,7 @@ fn serve_args_reject_unknown_renderer_value() {
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
|
||||
@@ -53,6 +53,8 @@ pub type Step3ReasoningParser = DeepSeekR1ReasoningParser;
|
||||
|
||||
/// DeepSeek V4 currently shares the DeepSeek V3 reasoning parser.
|
||||
pub type DeepSeekV4ReasoningParser = DeepSeekV3ReasoningParser;
|
||||
/// DeepSeek V4.1 uses bare `<think>...</think>` delimiters.
|
||||
pub type DeepSeekV41ReasoningParser = DeepSeekV3ReasoningParser;
|
||||
/// GLM-4.7 and GLM-5 use bare reasoning delimiters.
|
||||
pub type Glm47ReasoningParser = DeepSeekV3ReasoningParser;
|
||||
/// Kimi K2 currently shares the DeepSeek V3 reasoning parser.
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use super::{DeepSeekDsmlToolParser, DsmlTokens};
|
||||
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
|
||||
|
||||
mod structural_tag;
|
||||
|
||||
/// Tool parser for DeepSeek V4.1's spaced DSML tags.
|
||||
///
|
||||
/// Arguments are emitted only after a full `invoke` block is parsed.
|
||||
pub struct DeepSeekV41ToolParser(DeepSeekDsmlToolParser);
|
||||
|
||||
impl ToolParser for DeepSeekV41ToolParser {
|
||||
fn create(tools: &[Tool]) -> Result<Box<dyn ToolParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Ok(Box::new(Self(DeepSeekDsmlToolParser::new(
|
||||
tools,
|
||||
DsmlTokens::V41,
|
||||
))))
|
||||
}
|
||||
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
Some(&structural_tag::DeepSeekV41StructuralTagBuilder)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
|
||||
self.0.parse_into(chunk, output)
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<ToolParserOutput> {
|
||||
self.0.finish()
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
self.0.reset()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,108 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Visible-text grammar for DeepSeek V4.1 tool calls.
|
||||
//!
|
||||
//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with
|
||||
//! V4.1's spaced DSML tags and schema-unconstrained parameter values.
|
||||
|
||||
use serde_json::Value;
|
||||
use xgrammar_structural_tag::Result;
|
||||
use xgrammar_structural_tag::builders::{
|
||||
StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
|
||||
};
|
||||
use xgrammar_structural_tag::format::{
|
||||
Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat,
|
||||
};
|
||||
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
|
||||
|
||||
use super::DsmlTokens;
|
||||
|
||||
const TOKENS: DsmlTokens = DsmlTokens::V41;
|
||||
|
||||
pub(super) struct DeepSeekV41StructuralTagBuilder;
|
||||
|
||||
impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder {
|
||||
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
|
||||
let tags = ctx
|
||||
.function_tools
|
||||
.iter()
|
||||
.map(|tool| call_tag(tool, ctx.options))
|
||||
.collect::<Vec<_>>();
|
||||
let excludes = if ctx.options.exclude_special_tokens {
|
||||
&["<think>", "</think>"][..]
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
let format = match ctx.tool_choice {
|
||||
BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes),
|
||||
BuilderToolChoice::Auto => Format::TriggeredTags(
|
||||
TriggeredTagsFormat::new(
|
||||
&[TOKENS.tool_calls_start],
|
||||
vec![TagFormat::new(
|
||||
format!("{}\n", TOKENS.tool_calls_start),
|
||||
Format::tags_with_separator(tags, "", true, false),
|
||||
TOKENS.tool_calls_end,
|
||||
)],
|
||||
)
|
||||
.with_excludes(excludes),
|
||||
),
|
||||
BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![
|
||||
Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)),
|
||||
Format::tags_with_separator(
|
||||
tags,
|
||||
"",
|
||||
true,
|
||||
ctx.tool_choice == BuilderToolChoice::Forced,
|
||||
),
|
||||
Format::const_string(TOKENS.tool_calls_end),
|
||||
]),
|
||||
};
|
||||
// Reasoning boundaries remain owned by the engine reasoner; serving
|
||||
// calls this visible-text builder with reasoning=false.
|
||||
Ok(StructuralTag::new(format))
|
||||
}
|
||||
}
|
||||
|
||||
fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
|
||||
// TODO: enforce tool.parameters and strict schema semantics once V4.1
|
||||
// argument conversion is available. This builder intentionally ignores
|
||||
// parameters and strict: it constrains tool names and DSML/value syntax
|
||||
// only. Parameter names, presence, uniqueness and value schemas remain
|
||||
// unconstrained, including for strict=true. Serving still uses strict to
|
||||
// decide whether auto tool choice activates a grammar.
|
||||
TagFormat::new(
|
||||
format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name),
|
||||
Format::star(parameter(options)),
|
||||
format!("{}\n", TOKENS.invoke_end),
|
||||
)
|
||||
}
|
||||
|
||||
fn parameter(options: StructuralTagOptions) -> Format {
|
||||
Format::tag(
|
||||
format!("{} name=\"", TOKENS.parameter_start),
|
||||
Format::sequence(vec![
|
||||
Format::regex(r#"[^"]+"#),
|
||||
Format::const_string("\" string=\""),
|
||||
Format::or(vec![
|
||||
Format::sequence(vec![
|
||||
Format::const_string("true\">"),
|
||||
Format::any_text_excluding(&[
|
||||
TOKENS.parameter_end,
|
||||
TOKENS.invoke_end,
|
||||
TOKENS.tool_calls_end,
|
||||
]),
|
||||
]),
|
||||
Format::sequence(vec![
|
||||
Format::const_string("false\">"),
|
||||
Format::JsonSchema(
|
||||
JsonSchemaFormat::new(Value::Bool(true))
|
||||
.with_any_order(options.any_order)
|
||||
.with_max_whitespace_cnt(options.max_whitespace_cnt),
|
||||
),
|
||||
]),
|
||||
]),
|
||||
]),
|
||||
format!("{}\n", TOKENS.parameter_end),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use expect_test::expect;
|
||||
use serde_json::json;
|
||||
use xgrammar_structural_tag::builders::StructuralTagOptions;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
};
|
||||
|
||||
use super::DeepSeekV41ToolParser;
|
||||
use crate::tool::test_utils::collect_stream;
|
||||
use crate::tool::{Tool, ToolParser};
|
||||
|
||||
fn tools() -> Vec<Tool> {
|
||||
vec![Tool {
|
||||
name: "lookup".into(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"limit": {"type": "integer"},
|
||||
"enabled": {"type": "boolean"},
|
||||
"data": {"type": "object"},
|
||||
"items": {"type": "array"},
|
||||
"label": {"type": "string"}
|
||||
}
|
||||
}),
|
||||
strict: None,
|
||||
}]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spaced_dsml_streaming_preserves_shared_schema_coercion() {
|
||||
let wire = concat!(
|
||||
"summary\n\n<|DSML| calls>\n",
|
||||
"<|DSML| invoke name=\"lookup\">\n",
|
||||
"<|DSML| parameter name=\"query\" string=\"true\"> value\n</|DSML| parameter>\n",
|
||||
"<|DSML| parameter name=\"limit\" string=\"false\">2</|DSML| parameter>\n",
|
||||
"<|DSML| parameter name=\"enabled\" string=\"false\">true</|DSML| parameter>\n",
|
||||
"<|DSML| parameter name=\"data\" string=\"false\">{\"x\":1}</|DSML| parameter>\n",
|
||||
"<|DSML| parameter name=\"items\" string=\"false\">[1,2]</|DSML| parameter>\n",
|
||||
"<|DSML| parameter name=\"label\" string=\"false\">42</|DSML| parameter>\n",
|
||||
"<|DSML| parameter name=\"unknown\" string=\"false\">42</|DSML| parameter>\n",
|
||||
"<|DSML| parameter name=\"nullish\" string=\"false\">null</|DSML| parameter>\n",
|
||||
"</|DSML| invoke>\n",
|
||||
"<|DSML| invoke name=\"lookup\">\n",
|
||||
"<|DSML| parameter name=\"query\" string=\"true\">second</|DSML| parameter>\n",
|
||||
"</|DSML| invoke>\n</|DSML| calls>",
|
||||
);
|
||||
let expected = [
|
||||
json!({"query":" value\n","limit":2,"enabled":true,"data":{"x":1},"items":[1,2],"label":"42","unknown":"42","nullish":null}),
|
||||
json!({"query":"second"}),
|
||||
];
|
||||
for split in wire.char_indices().map(|(i, _)| i).chain(std::iter::once(wire.len())) {
|
||||
let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
|
||||
let output = collect_stream(parser.as_mut(), &[&wire[..split], &wire[split..]]);
|
||||
// Protocol framing follows the current shared DSML behavior.
|
||||
assert_eq!(output.normal_text(), "summary", "split {split}");
|
||||
assert_eq!(output.calls().len(), 2, "split {split}");
|
||||
for (index, call) in output.calls().iter().enumerate() {
|
||||
assert_eq!(call.tool_index, index, "split {split}");
|
||||
assert_eq!(call.name.as_deref(), Some("lookup"), "split {split}");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&call.arguments).unwrap(),
|
||||
expected[index],
|
||||
"split {split}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v4_tags_remain_text_in_v41_dialect() {
|
||||
let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
|
||||
let output = collect_stream(parser.as_mut(), &["before<|DSML|tool_calls>after"]);
|
||||
expect!["before<|DSML|tool_calls>after"].assert_eq(&output.normal_text());
|
||||
assert!(output.calls().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structural_tag_ignores_parameter_schemas_and_strict() {
|
||||
let parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
|
||||
let builder = parser.structural_tag_builder().unwrap();
|
||||
let options = StructuralTagOptions::default().with_reasoning(false);
|
||||
for choice in [
|
||||
ToolChoice::auto(),
|
||||
ToolChoice::required(),
|
||||
ToolChoice::function("lookup"),
|
||||
] {
|
||||
let build = |function| {
|
||||
build_structural_tag(
|
||||
builder,
|
||||
&[ToolParam::Function(FunctionToolParam::new(function))],
|
||||
choice.clone(),
|
||||
options,
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let expected = build(FunctionDefinition::new("lookup"));
|
||||
for strict in [None, Some(false), Some(true)] {
|
||||
for parameters in [
|
||||
json!(false),
|
||||
json!({"type":"object", "properties":{"query":{"const":"fixed"}}, "required":["query"]}),
|
||||
] {
|
||||
let mut function = FunctionDefinition::new("lookup").with_parameters(parameters);
|
||||
function.strict = strict;
|
||||
assert_eq!(build(function), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,14 +14,11 @@ use crate::tool::Tool;
|
||||
|
||||
mod deepseek_v32;
|
||||
mod deepseek_v4;
|
||||
mod deepseek_v41;
|
||||
|
||||
pub use deepseek_v4::DeepSeekV4ToolParser;
|
||||
pub use deepseek_v32::DeepSeekV32ToolParser;
|
||||
|
||||
const INVOKE_START: &str = "<|DSML|invoke";
|
||||
const INVOKE_END: &str = "</|DSML|invoke>";
|
||||
const PARAMETER_START: &str = "<|DSML|parameter";
|
||||
const PARAMETER_END: &str = "</|DSML|parameter>";
|
||||
pub use deepseek_v41::DeepSeekV41ToolParser;
|
||||
|
||||
type DsmlInput<'i> = Partial<&'i str>;
|
||||
|
||||
@@ -30,6 +27,10 @@ struct DsmlTokens {
|
||||
tool_calls_start: &'static str,
|
||||
framed_tool_calls_start: &'static str,
|
||||
tool_calls_end: &'static str,
|
||||
invoke_start: &'static str,
|
||||
invoke_end: &'static str,
|
||||
parameter_start: &'static str,
|
||||
parameter_end: &'static str,
|
||||
}
|
||||
|
||||
impl DsmlTokens {
|
||||
@@ -37,11 +38,25 @@ impl DsmlTokens {
|
||||
tool_calls_start: "<|DSML|function_calls>",
|
||||
framed_tool_calls_start: "\n\n<|DSML|function_calls>",
|
||||
tool_calls_end: "</|DSML|function_calls>",
|
||||
invoke_start: "<|DSML|invoke",
|
||||
invoke_end: "</|DSML|invoke>",
|
||||
parameter_start: "<|DSML|parameter",
|
||||
parameter_end: "</|DSML|parameter>",
|
||||
};
|
||||
const V4: Self = Self {
|
||||
tool_calls_start: "<|DSML|tool_calls>",
|
||||
framed_tool_calls_start: "\n\n<|DSML|tool_calls>",
|
||||
tool_calls_end: "</|DSML|tool_calls>",
|
||||
..Self::V32
|
||||
};
|
||||
const V41: Self = Self {
|
||||
tool_calls_start: "<|DSML| calls>",
|
||||
framed_tool_calls_start: "\n\n<|DSML| calls>",
|
||||
tool_calls_end: "</|DSML| calls>",
|
||||
invoke_start: "<|DSML| invoke",
|
||||
invoke_end: "</|DSML| invoke>",
|
||||
parameter_start: "<|DSML| parameter",
|
||||
parameter_end: "</|DSML| parameter>",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -205,7 +220,7 @@ fn parse_tool_block_event(
|
||||
) -> ModalResult<DsmlEvent> {
|
||||
ws0.void().parse_next(input)?;
|
||||
alt((
|
||||
|input: &mut DsmlInput<'_>| invoke_event(input, invoke_end_scan),
|
||||
|input: &mut DsmlInput<'_>| invoke_event(input, tokens, invoke_end_scan),
|
||||
|input: &mut DsmlInput<'_>| tool_calls_end_event(input, tokens),
|
||||
))
|
||||
.parse_next(input)
|
||||
@@ -243,19 +258,20 @@ fn safe_text_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult
|
||||
/// Parse a DSML invoke block.
|
||||
fn invoke_event(
|
||||
input: &mut DsmlInput<'_>,
|
||||
tokens: DsmlTokens,
|
||||
invoke_end_scan: &mut MarkerScanState,
|
||||
) -> ModalResult<DsmlEvent> {
|
||||
let (name, body) = seq!(
|
||||
_: literal(INVOKE_START),
|
||||
_: literal(tokens.invoke_start),
|
||||
_: ws1,
|
||||
dsml_name_attr,
|
||||
_: ws0,
|
||||
_: ">",
|
||||
take_until_marker(INVOKE_END, invoke_end_scan),
|
||||
_: literal(INVOKE_END),
|
||||
take_until_marker(tokens.invoke_end, invoke_end_scan),
|
||||
_: literal(tokens.invoke_end),
|
||||
)
|
||||
.parse_next(input)?;
|
||||
let raw_params = parse_invoke_params(body)?;
|
||||
let raw_params = parse_invoke_params(body, tokens)?;
|
||||
Ok(DsmlEvent::Invoke {
|
||||
name: name.to_string(),
|
||||
raw_params,
|
||||
@@ -263,23 +279,31 @@ fn invoke_event(
|
||||
}
|
||||
|
||||
/// Parse a DSML invoke body.
|
||||
fn parse_invoke_params(invoke_body: &str) -> ModalResult<Vec<DsmlParameter>> {
|
||||
fn parse_invoke_params(invoke_body: &str, tokens: DsmlTokens) -> ModalResult<Vec<DsmlParameter>> {
|
||||
let mut input = invoke_body;
|
||||
delimited(ws0, repeat(0.., terminated(parse_parameter, ws0)), eof).parse_next(&mut input)
|
||||
delimited(
|
||||
ws0,
|
||||
repeat(
|
||||
0..,
|
||||
terminated(|input: &mut &str| parse_parameter(input, tokens), ws0),
|
||||
),
|
||||
eof,
|
||||
)
|
||||
.parse_next(&mut input)
|
||||
}
|
||||
|
||||
/// Parse a DSML parameter block.
|
||||
fn parse_parameter(input: &mut &str) -> ModalResult<DsmlParameter> {
|
||||
fn parse_parameter(input: &mut &str, tokens: DsmlTokens) -> ModalResult<DsmlParameter> {
|
||||
seq! {DsmlParameter {
|
||||
_: literal(PARAMETER_START),
|
||||
_: literal(tokens.parameter_start),
|
||||
_: ws1,
|
||||
name: name_attr.map(|name: &str| name.to_string()),
|
||||
_: ws1,
|
||||
is_string: string_attr.map(|value| value == "true"),
|
||||
_: ws0,
|
||||
_: ">",
|
||||
value: take_until(0.., PARAMETER_END).map(str::to_string),
|
||||
_: literal(PARAMETER_END),
|
||||
value: take_until(0.., tokens.parameter_end).map(str::to_string),
|
||||
_: literal(tokens.parameter_end),
|
||||
}}
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ mod seed_oss;
|
||||
pub mod test_utils;
|
||||
use std::collections::{BTreeMap, btree_map};
|
||||
|
||||
pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
|
||||
pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser, DeepSeekV41ToolParser};
|
||||
pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
|
||||
pub use error::{Result, ToolParserError};
|
||||
pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
|
||||
|
||||
@@ -167,6 +167,19 @@ mod tests {
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_reasoning_effort_maps_to_invalid_request() {
|
||||
let error = vllm_chat::Error::InvalidReasoningEffort(
|
||||
"DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(),
|
||||
);
|
||||
let api_error = chat_submit_error("failed to submit chat request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
assert_eq!(
|
||||
api_error.to_error_response().error.error_type,
|
||||
"invalid_request_error"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn llm_wrapped_empty_prompt_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds {
|
||||
|
||||
Reference in New Issue
Block a user