[Model][Frontend] Support DeepSeek-V4.1-Flash in Rust and Python frontends (#56208)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Bugen Zhao
2026-09-10 01:15:06 -07:00
committed by GitHub
co-authored by Isotr0py
parent 9521c60bdc
commit b47b01cf33
52 changed files with 2986 additions and 113 deletions
+3 -2
View File
@@ -19,8 +19,8 @@ use crate::output::{
};
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
use crate::renderer::{
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
InklingChatRenderer, KimiK3ChatRenderer,
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer,
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer,
};
use crate::request::ChatRequest;
use crate::{DynChatOutputProcessor, RendererSelection};
@@ -72,6 +72,7 @@ impl HfChatBackend {
)?),
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()),
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
+3
View File
@@ -17,6 +17,8 @@ pub enum Error {
MissingChatTemplate,
#[error("chat template error: {0}")]
ChatTemplate(String),
#[error("{0}")]
InvalidReasoningEffort(String),
#[error("multimodal input is not supported by this chat renderer")]
UnsupportedMultimodalRenderer,
#[error("unsupported multimodal content: {0}")]
@@ -96,6 +98,7 @@ impl Error {
pub fn is_request_validation_error(&self) -> bool {
match self {
Self::PromptTooLong { .. }
| Self::InvalidReasoningEffort(_)
| Self::DuplicateToolName { .. }
| Self::ToolChoiceRequiresTools
| Self::ToolChoiceFunctionNotFound { .. } => true,
+4 -4
View File
@@ -33,8 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use parser::{ParserSelection, validate_parser_overrides};
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
RendererSelection,
};
pub use request::{
@@ -363,7 +363,7 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
}
#[test]
@@ -374,6 +374,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
+39 -1
View File
@@ -153,6 +153,10 @@ struct ResolvedMultimodalSpec {
modality: Modality,
field_layouts: EncoderFieldLayouts,
keep_on_cpu_keys: HashSet<String>,
/// Spec-declared wire key for the primary encoder input tensor, when it
/// differs from the per-modality default (e.g. `"patches"` for
/// DeepSeek-V4.1 images).
encoder_input_key: Option<String>,
}
impl ResolvedMultimodalSpec {
@@ -162,10 +166,14 @@ impl ResolvedMultimodalSpec {
modality,
field_layouts: raw.encoder_field_layouts_for(modality),
keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
encoder_input_key: raw.encoder_input_key_for(modality),
}
}
fn primary_key(&self) -> &'static str {
fn primary_key(&self) -> &str {
if let Some(key) = &self.encoder_input_key {
return key;
}
match self.modality {
Modality::Image => image::IMAGE_PRIMARY_KEY,
Modality::Video => video::VIDEO_PRIMARY_KEY,
@@ -841,6 +849,9 @@ mod tests {
pub(super) const QWEN3_IMAGE_PAD_ID: u32 = 151655;
pub(super) const QWEN3_VIDEO_PAD_ID: u32 = 151656;
pub(super) const DEEPSEEK_V41_IMAGE_ID: u32 = 129264;
pub(super) const DEEPSEEK_V41_IMAGE_PAD_ID: u32 = 129265;
fn llama4_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
@@ -954,6 +965,33 @@ mod tests {
assert_eq!(info.placeholder_token(Modality::Video), None);
}
fn deepseek_v41_info() -> MultimodalModelInfo {
let config = serde_json::json!({
"model_type": "deepseek_v41",
"image_token_id": DEEPSEEK_V41_IMAGE_ID,
});
let tokenizer = TestTokenizer::new()
.with_regular_token("<|deepseek_image|>", DEEPSEEK_V41_IMAGE_ID)
.with_regular_token("<|place_holder_mm_span_0436|>", DEEPSEEK_V41_IMAGE_PAD_ID);
test_info("deepseek_v41", config, tokenizer)
}
#[test]
fn deepseek_v41_resolves_image_support_only() {
let info = deepseek_v41_info();
assert_eq!(
info.placeholder_token(Modality::Image),
Some("<|deepseek_image|>")
);
assert_eq!(info.placeholder_token(Modality::Video), None);
let image = info.image.as_ref().expect("image support");
assert_eq!(image.placeholder.marker_token_id, DEEPSEEK_V41_IMAGE_ID);
assert_eq!(image.placeholder.embed_token_id, DEEPSEEK_V41_IMAGE_ID);
// The engine's forward kwargs pop `patches` (not `pixel_values`).
assert_eq!(image.spec.primary_key(), "patches");
}
#[test]
fn input_audio_uses_connector_data_urls() {
assert_eq!(
+136 -8
View File
@@ -45,6 +45,14 @@ impl<'a> ExpansionLane<'a> {
/// matching the original media-part order within that modality; markers of
/// different modalities may interleave freely.
///
/// Replacements may carry an [`AlignmentPad`](llm_multimodal::AlignmentPad)
/// (e.g. DeepSeek-V4.1's compressor-alignment pad): `period - 1 - offset %
/// period` pad tokens are
/// prepended at splice time, where `offset` is the position the replacement
/// lands at in the expanded prompt. The pads are part of the reported range
/// and count as embed positions only if the pad token id equals the
/// modality's embed token id.
///
/// The returned ranges point into the already-expanded prompt, grouped per
/// modality in item order.
pub(super) fn expand_prompt_token_ids(
@@ -56,12 +64,14 @@ pub(super) fn expand_prompt_token_ids(
return Ok(HashMap::new());
}
let replacement_growth = lanes
.iter()
.flat_map(|lane| lane.replacements.iter())
.fold(0usize, |total, replacement| {
total.saturating_add(replacement.tokens.len().saturating_sub(1))
});
let replacement_growth = lanes.iter().flat_map(|lane| lane.replacements.iter()).fold(
0usize,
|total, replacement| {
// Alignment pads are position-dependent; reserve the worst case.
let max_pads = replacement.alignment_pad.map_or(0, |pad| pad.period - 1);
total.saturating_add(max_pads + replacement.tokens.len().saturating_sub(1))
},
);
let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth);
let mut expanded = Vec::with_capacity(expanded_len);
@@ -95,10 +105,20 @@ pub(super) fn expand_prompt_token_ids(
expanded.len()
);
}
let range_len = replacement_len.saturating_add(structural_prefix);
// Position-dependent alignment padding, computed from the offset the
// replacement block lands at in the expanded prompt (mirroring
// vLLM's `_apply_token_matches_with_placeholders` splice-time pad).
let pad_count = replacement.alignment_pad.map_or(0, |pad| pad.count_at(expanded.len()));
let range_len = structural_prefix + pad_count + replacement_len;
let is_embed = {
let mut mask = Vec::with_capacity(range_len);
mask.extend(std::iter::repeat_n(false, structural_prefix));
if let Some(pad) = &replacement.alignment_pad {
mask.extend(std::iter::repeat_n(
pad.token_id as u32 == lane.embed_token_id,
pad_count,
));
}
mask.extend(
replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
);
@@ -106,6 +126,9 @@ pub(super) fn expand_prompt_token_ids(
};
let expanded_offset = expanded.len() - structural_prefix;
if let Some(pad) = &replacement.alignment_pad {
expanded.extend(std::iter::repeat_n(pad.token_id as u32, pad_count));
}
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
offset: expanded_offset,
@@ -454,7 +477,6 @@ mod tests {
}
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
prepared_media(
Modality::Audio,
@@ -542,4 +564,110 @@ mod tests {
));
assert_eq!(prompt_token_ids, original_prompt_token_ids);
}
// DeepSeek-V4.1 image spans: every span position carries the image token
// id; a compressor-alignment pad (id 129265, period 2) is prepended at
// splice time so spans always start at the same compressor phase.
const DS_IMAGE_ID: u32 = 129_264;
const DS_IMAGE_PAD_ID: u32 = 129_265;
fn deepseek_prepared(span_lens: Vec<usize>) -> PreparedMedia {
prepared_media(
Modality::Image,
"<|deepseek_image|>",
DS_IMAGE_ID,
DS_IMAGE_ID,
span_lens
.into_iter()
.map(|len| {
PromptReplacement::repeated(
Modality::Image,
"<|deepseek_image|>",
DS_IMAGE_ID as TokenId,
len,
)
.with_alignment_pad(DS_IMAGE_PAD_ID as TokenId, 2)
})
.collect(),
)
}
#[test]
fn expand_prompt_tokens_skips_alignment_pad_at_odd_offset() {
// Odd block start: already at the target compressor phase, no pad.
let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2];
let prepared = vec![deepseek_prepared(vec![3])];
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
let ranges = &ranges[&Modality::Image];
assert_eq!(
prompt_token_ids,
vec![1, DS_IMAGE_ID, DS_IMAGE_ID, DS_IMAGE_ID, 2]
);
assert_eq!(ranges[0].offset, 1);
assert_eq!(ranges[0].length, 3);
assert_bool_mask(&ranges[0], &[true, true, true]);
}
#[test]
fn expand_prompt_tokens_prepends_alignment_pad_at_even_offset() {
// Even block start: one pad token aligns the span to phase 1.
let mut prompt_token_ids = vec![1, 3, DS_IMAGE_ID, 2];
let prepared = vec![deepseek_prepared(vec![3])];
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
let ranges = &ranges[&Modality::Image];
assert_eq!(
prompt_token_ids,
vec![
1,
3,
DS_IMAGE_PAD_ID,
DS_IMAGE_ID,
DS_IMAGE_ID,
DS_IMAGE_ID,
2
]
);
// The range starts at the pad; pads are not embed positions.
assert_eq!(ranges[0].offset, 2);
assert_eq!(ranges[0].length, 4);
assert_bool_mask(&ranges[0], &[false, true, true, true]);
}
#[test]
fn expand_prompt_tokens_alignment_pad_realigns_each_span() {
// Every image span must start at phase 1 (odd offset) regardless of
// where the previous span landed.
let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2, DS_IMAGE_ID, 3];
let prepared = vec![deepseek_prepared(vec![2, 2])];
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
let ranges = &ranges[&Modality::Image];
// First span at odd offset 1: no pad (block [1,3)).
// Second marker originally at index 3; after the first expansion the
// block starts at offset 4 (even): one pad, span at odd offset 5.
assert_eq!(
prompt_token_ids,
vec![
1,
DS_IMAGE_ID,
DS_IMAGE_ID,
2,
DS_IMAGE_PAD_ID,
DS_IMAGE_ID,
DS_IMAGE_ID,
3,
]
);
assert_eq!((ranges[0].offset, ranges[0].length), (1, 2));
assert_bool_mask(&ranges[0], &[true, true]);
assert_eq!((ranges[1].offset, ranges[1].length), (4, 3));
assert_bool_mask(&ranges[1], &[false, true, true]);
// Both spans (past the pad) start at odd offsets.
assert_eq!((ranges[1].offset + 1) % 2, 1);
}
}
+8 -4
View File
@@ -7,10 +7,11 @@ use std::sync::{Arc, LazyLock};
pub use vllm_parser::reasoning::{
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser,
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
DeepSeekV4ReasoningParser, DeepSeekV41ReasoningParser, Glm45ReasoningParser,
Glm47ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser, MiniMaxM2ReasoningParser,
MiniMaxM3ReasoningParser, NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta,
ReasoningError, ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser,
Step3p5ReasoningParser,
};
use vllm_tokenizer::DynTokenizer;
@@ -22,6 +23,7 @@ pub mod names {
pub const DEEPSEEK_R1: &str = "deepseek_r1";
pub const DEEPSEEK_V3: &str = "deepseek_v3";
pub const DEEPSEEK_V4: &str = "deepseek_v4";
pub const DEEPSEEK_V41: &str = "deepseek_v41";
pub const GLM45: &str = "glm45";
pub const GLM47: &str = "glm47";
pub const KIMI: &str = "kimi";
@@ -62,6 +64,7 @@ impl ReasoningParserFactory {
.register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
.register_parser::<DeepSeekV41ReasoningParser>(names::DEEPSEEK_V41)
.register_parser::<Glm45ReasoningParser>(names::GLM45)
.register_parser::<Glm47ReasoningParser>(names::GLM47)
.register_parser::<KimiReasoningParser>(names::KIMI)
@@ -76,6 +79,7 @@ impl ReasoningParserFactory {
factory
.register_pattern("deepseek-r1", names::DEEPSEEK_R1)
.register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
.register_pattern("deepseek-v3", names::DEEPSEEK_V3)
@@ -33,6 +33,10 @@ fn factory_resolves_deepseek_v4() {
factory.resolve_name_for_model("deepseek_v4"),
Some(names::DEEPSEEK_V4)
);
assert_eq!(
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
Some(names::DEEPSEEK_V41)
);
}
#[test]
+7 -4
View File
@@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock};
pub use vllm_parser::tool::{
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
DeepSeekV41ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser,
HermesToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
Qwen3CoderToolParser, Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
};
use crate::parser::ParserFactory;
@@ -22,6 +22,7 @@ pub mod names {
pub const DEEPSEEK_V31: &str = "deepseek_v31";
pub const DEEPSEEK_V32: &str = "deepseek_v32";
pub const DEEPSEEK_V4: &str = "deepseek_v4";
pub const DEEPSEEK_V41: &str = "deepseek_v41";
pub const GLM45: &str = "glm45";
pub const GLM47: &str = "glm47";
pub const GRANITE4: &str = "granite4";
@@ -66,6 +67,7 @@ impl ToolParserFactory {
.register_parser::<DeepSeekV31ToolParser>(names::DEEPSEEK_V31)
.register_parser::<DeepSeekV32ToolParser>(names::DEEPSEEK_V32)
.register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
.register_parser::<DeepSeekV41ToolParser>(names::DEEPSEEK_V41)
.register_parser::<Glm45MoeToolParser>(names::GLM45)
.register_parser::<Glm47MoeToolParser>(names::GLM47)
.register_parser::<Granite4ToolParser>(names::GRANITE4)
@@ -101,6 +103,7 @@ impl ToolParserFactory {
.register_pattern("llama-3.2", names::LLAMA3_JSON)
.register_pattern("llama-3.1", names::LLAMA3_JSON)
.register_pattern("deepseek-r1", names::DEEPSEEK_V3)
.register_pattern("deepseek-v4.1", names::DEEPSEEK_V41)
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
.register_pattern("deepseek-v3.2", names::DEEPSEEK_V32)
+4
View File
@@ -147,6 +147,10 @@ fn factory_new_resolves_default_patterns() {
factory.resolve_name_for_model("deepseek_v4"),
Some(names::DEEPSEEK_V4)
);
assert_eq!(
factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"),
Some(names::DEEPSEEK_V41)
);
assert_eq!(
factory.resolve_name_for_model("deepseek-ai/DeepSeek-R1-0528"),
Some(names::DEEPSEEK_V3)
@@ -1,11 +1,12 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! DeepSeek V4 prompt renderer.
//! Shared DeepSeek V4 and V4.1 prompt rendering.
//!
//! Official Python reference:
//! <https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py>
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::Write as _;
@@ -13,8 +14,12 @@ use serde::Serialize;
use serde_json::Value;
use serde_json_fmt::JsonFormat;
use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER;
use crate::error::{Error, Result};
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, ReasoningEffort};
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort,
};
use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
const BOS_TOKEN: &str = "<|begin▁of▁sentence|>";
@@ -23,6 +28,7 @@ const THINKING_START_TOKEN: &str = "<think>";
const THINKING_END_TOKEN: &str = "</think>";
const DSML_TOKEN: &str = "|DSML|";
const USER_SP_TOKEN: &str = "<|User|>";
const SYSTEM_SP_TOKEN: &str = "<|System|>";
const ASSISTANT_SP_TOKEN: &str = "<|Assistant|>";
const REASONING_EFFORT_HIGH: &str = concat!(
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n",
@@ -41,6 +47,35 @@ enum ThinkingMode {
Thinking,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum DsDialect {
V4,
V41,
}
impl DsDialect {
fn tool_calls_tag(self) -> &'static str {
match self {
Self::V4 => "tool_calls",
Self::V41 => " calls",
}
}
fn invoke_tag(self) -> &'static str {
match self {
Self::V4 => "invoke",
Self::V41 => " invoke",
}
}
fn parameter_tag(self) -> &'static str {
match self {
Self::V4 => "parameter",
Self::V41 => " parameter",
}
}
}
#[serde_with::skip_serializing_none]
#[derive(Debug, Serialize)]
struct RenderedToolSchema<'a> {
@@ -51,25 +86,47 @@ struct RenderedToolSchema<'a> {
}
/// Render one chat request into the final prompt string.
pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
let (thinking_mode, reasoning_effort_prompt) = resolve_thinking_options(request)?;
pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result<String> {
let (thinking_mode, reasoning_effort_prompt) = match dialect {
DsDialect::V4 => {
resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))?
}
DsDialect::V41 => resolve_v41_thinking_options(request)?,
};
let request_tools = request_tools(request);
let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools);
let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true)
&& !rendered_tools_present(request, request_tools);
let drop_historical_developers = thinking_mode == ThinkingMode::Thinking && drop_thinking;
let last_user_like_message_index = request.messages.iter().rposition(is_user_like_entry);
let last_user_render_index =
find_last_user_render_index(request.messages.as_slice(), synthetic_tool_system);
let last_user_like_message_index = request
.messages
.iter()
.enumerate()
.rfind(|(index, message)| is_user_like_entry(message, *index, dialect))
.map(|(index, _)| index);
let last_user_render_index = find_last_user_render_index(
request.messages.as_slice(),
synthetic_tool_system,
drop_historical_developers,
last_user_like_message_index,
dialect,
);
let mut out = String::from(BOS_TOKEN);
if dialect == DsDialect::V41
&& (thinking_mode == ThinkingMode::Thinking
|| synthetic_tool_system
|| matches!(request.messages.first(), Some(ChatMessage::System { .. })))
{
out.push_str(SYSTEM_SP_TOKEN);
}
if thinking_mode == ThinkingMode::Thinking {
out.push_str(reasoning_effort_prompt);
out.push_str(&reasoning_effort_prompt);
}
let mut request_tools_attached = false;
let mut render_index = 0isize;
if synthetic_tool_system {
render_system_message(&mut out, None, request_tools)?;
render_system_message(&mut out, None, request_tools, dialect)?;
request_tools_attached = true;
render_index += 1;
}
@@ -80,27 +137,35 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
continue;
}
let current_render_index = render_index;
render_index += 1;
if drop_historical_developers
&& is_historical_developer(message, message_index, last_user_like_message_index)
{
continue;
}
let current_render_index = render_index;
render_index += 1;
match message {
ChatMessage::System { content } => {
if dialect == DsDialect::V41 && current_render_index > 0 {
out.push_str(SYSTEM_SP_TOKEN);
}
let tools = if !request_tools_attached {
request_tools_attached = true;
request_tools
} else {
&[]
};
render_system_message(&mut out, Some(content), tools)?;
render_system_message(&mut out, Some(content), tools, dialect)?;
}
ChatMessage::Developer { content, tools } => {
render_developer_message(&mut out, content, tools.as_deref().unwrap_or(&[]))?;
render_developer_message(
&mut out,
content,
tools.as_deref().unwrap_or(&[]),
dialect,
)?;
}
ChatMessage::User { .. } | ChatMessage::ToolResponse { .. } => {
render_user_content_block(
@@ -108,6 +173,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
request.messages.as_slice(),
message_index,
&last_tool_call_order,
dialect,
)?;
}
ChatMessage::Assistant { content } => {
@@ -119,7 +185,13 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
&& (!drop_thinking || current_render_index > last_user_render_index);
let append_eos = !(message_index + 1 == request.messages.len()
&& request.chat_options.continue_final_message());
render_assistant_message(&mut out, emit_thinking_block, append_eos, content)?;
render_assistant_message(
&mut out,
emit_thinking_block,
append_eos,
content,
dialect,
)?;
if content.has_tool_calls() {
last_tool_call_order.clear();
@@ -133,7 +205,8 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
}
}
if (is_user_like_entry(message) || matches!(message, ChatMessage::System { .. }))
if (is_user_like_entry(message, current_render_index as usize, dialect)
|| (dialect == DsDialect::V4 && matches!(message, ChatMessage::System { .. })))
&& next_rendered_entry_is_assistant_or_end(
request.messages.as_slice(),
message_index,
@@ -181,6 +254,57 @@ fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'st
Ok((thinking_mode, reasoning_effort_prompt))
}
/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs.
fn resolve_v41_thinking_options(
request: &ChatRequest,
) -> Result<(ThinkingMode, Cow<'static, str>)> {
let mut thinking = request.enable_thinking()?.unwrap_or(true);
let budget = match request.chat_options.reasoning_effort {
Some(ReasoningEffort::None) => {
thinking = false;
50
}
Some(ReasoningEffort::Low) => 25,
Some(ReasoningEffort::High) => 50,
Some(ReasoningEffort::XHigh) => 75,
Some(ReasoningEffort::Max) => 100,
Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => {
return Err(invalid_v41_reasoning_effort());
}
None => match request.chat_options.template_kwargs.get("reasoning_effort") {
None | Some(Value::Null) => 50,
Some(Value::String(effort)) => match effort.as_str() {
"low" => 25,
"high" => 50,
"xhigh" => 75,
"max" => 100,
_ => return Err(invalid_v41_reasoning_effort()),
},
Some(Value::Number(budget)) => budget
.as_u64()
.filter(|budget| (1..=100).contains(budget))
.ok_or_else(invalid_v41_reasoning_effort)?,
Some(_) => return Err(invalid_v41_reasoning_effort()),
},
};
if thinking {
Ok((
ThinkingMode::Thinking,
Cow::Owned(format!(
"Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
)),
))
} else {
Ok((ThinkingMode::Chat, Cow::Borrowed("")))
}
}
fn invalid_v41_reasoning_effort() -> Error {
Error::InvalidReasoningEffort(
"DeepSeek V4.1 reasoning_effort must be low, high, xhigh, max, or an integer within [1, 100] in chat_template_kwargs".to_string(),
)
}
/// Return request-level tools only when native tool parsing is enabled.
fn request_tools(request: &ChatRequest) -> &[ChatTool] {
if request.tool_parsing_enabled() {
@@ -214,16 +338,25 @@ fn rendered_tools_present(request: &ChatRequest, request_tools: &[ChatTool]) ->
}
/// Find the last user-like turn after inline tool-response merging.
fn find_last_user_render_index(messages: &[ChatMessage], synthetic_tool_system: bool) -> isize {
fn find_last_user_render_index(
messages: &[ChatMessage],
synthetic_tool_system: bool,
drop_historical_developers: bool,
last_user_like_message_index: Option<usize>,
dialect: DsDialect,
) -> isize {
let mut render_index = isize::from(synthetic_tool_system);
let mut last_user_index = -1;
for (message_index, message) in messages.iter().enumerate() {
if is_following_user_content(messages, message_index) {
if is_following_user_content(messages, message_index)
|| (drop_historical_developers
&& is_historical_developer(message, message_index, last_user_like_message_index))
{
continue;
}
if is_user_like_entry(message) {
if is_user_like_entry(message, render_index as usize, dialect) {
last_user_index = render_index;
}
render_index += 1;
@@ -249,11 +382,13 @@ fn is_user_content_entry(message: &ChatMessage) -> bool {
}
/// Return whether one rendered entry should be treated as user-like.
fn is_user_like_entry(message: &ChatMessage) -> bool {
fn is_user_like_entry(message: &ChatMessage, message_index: usize, dialect: DsDialect) -> bool {
matches!(
message,
ChatMessage::Developer { .. } | ChatMessage::User { .. } | ChatMessage::ToolResponse { .. }
)
) || (dialect == DsDialect::V41
&& message_index > 0
&& matches!(message, ChatMessage::System { .. }))
}
/// Return whether a developer entry precedes another user-like turn.
@@ -293,22 +428,26 @@ fn next_rendered_entry_is_assistant_or_end(
.unwrap_or(true)
}
/// Render the tool preamble shown to the model, V4 flavor.
fn render_tools(out: &mut String, tools: &[ChatTool]) -> Result<()> {
out.push_str(
/// Render the tool preamble shown to the model for one DeepSeek dialect.
fn render_tools(out: &mut String, tools: &[ChatTool], dialect: DsDialect) -> Result<()> {
let tool_calls_tag = dialect.tool_calls_tag();
let invoke_tag = dialect.invoke_tag();
let parameter_tag = dialect.parameter_tag();
write!(
out,
r#"## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|{tool_calls_tag}>" block like the following:
<|DSML|tool_calls>
<|DSML|invoke name="$TOOL_NAME">
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
<|DSML|{tool_calls_tag}>
<|DSML|{invoke_tag} name="$TOOL_NAME">
<|DSML|{parameter_tag} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|{parameter_tag}>
...
</|DSML|invoke>
<|DSML|invoke name="$TOOL_NAME2">
</|DSML|{invoke_tag}>
<|DSML|{invoke_tag} name="$TOOL_NAME2">
...
</|DSML|invoke>
</|DSML|tool_calls>
</|DSML|{invoke_tag}>
</|DSML|{tool_calls_tag}>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
@@ -319,7 +458,8 @@ Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
"#,
);
)
.expect("writing to String cannot fail");
for (index, tool) in tools.iter().enumerate() {
if index > 0 {
@@ -350,13 +490,14 @@ fn render_system_message(
out: &mut String,
content: Option<&ChatContent>,
tools: &[ChatTool],
dialect: DsDialect,
) -> Result<()> {
if let Some(content) = content {
write_chat_content(out, content)?;
write_chat_content(out, content, dialect)?;
}
if !tools.is_empty() {
out.push_str("\n\n");
render_tools(out, tools)?;
render_tools(out, tools, dialect)?;
}
Ok(())
}
@@ -366,18 +507,19 @@ fn render_developer_message(
out: &mut String,
content: &ChatContent,
tools: &[ChatTool],
dialect: DsDialect,
) -> Result<()> {
if content.is_empty() {
return Err(Error::ChatTemplate(
"invalid DeepSeek V4 developer message: empty content".to_string(),
"invalid DeepSeek developer message: empty content".to_string(),
));
}
out.push_str(USER_SP_TOKEN);
write_chat_content(out, content)?;
write_chat_content(out, content, dialect)?;
if !tools.is_empty() {
out.push_str("\n\n");
render_tools(out, tools)?;
render_tools(out, tools, dialect)?;
}
Ok(())
}
@@ -388,6 +530,7 @@ fn render_user_content_block(
messages: &[ChatMessage],
message_index: usize,
tool_call_order: &HashMap<String, usize>,
dialect: DsDialect,
) -> Result<()> {
let (block_start, block_end) = user_content_block_bounds(messages, message_index);
let mut sorted_tool_indices =
@@ -399,7 +542,7 @@ fn render_user_content_block(
out.push_str("\n\n");
}
match &messages[message_index] {
ChatMessage::User { content } => write_chat_content(out, content)?,
ChatMessage::User { content } => write_chat_content(out, content, dialect)?,
ChatMessage::ToolResponse { .. } => {
let sorted_index = sorted_tool_indices
.next()
@@ -407,7 +550,7 @@ fn render_user_content_block(
let ChatMessage::ToolResponse { content, .. } = &messages[sorted_index] else {
unreachable!("sorted tool response index should reference a tool message");
};
write_tool_result(out, content)?;
write_tool_result(out, content, dialect)?;
}
_ => unreachable!("user content block should only contain user content messages"),
}
@@ -454,9 +597,9 @@ fn sorted_tool_response_indices(
}
/// Render one tool response payload inside a V4 `<tool_result>` block.
fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> {
fn write_tool_result(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
out.push_str("<tool_result>");
write_chat_content(out, content)?;
write_chat_content(out, content, dialect)?;
out.push_str("</tool_result>");
Ok(())
}
@@ -483,6 +626,7 @@ fn render_assistant_message(
emit_thinking_block: bool,
append_eos: bool,
content: &[AssistantContentBlock],
dialect: DsDialect,
) -> Result<()> {
let has_tool_calls = content.has_tool_calls();
@@ -496,14 +640,15 @@ fn render_assistant_message(
write_assistant_text(out, content);
if has_tool_calls {
out.push_str("\n\n<|DSML|tool_calls>\n");
let tool_calls_tag = dialect.tool_calls_tag();
writeln!(out, "\n\n<{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
for (index, tool_call) in content.tool_calls().enumerate() {
if index > 0 {
out.push('\n');
}
render_tool_call(out, tool_call)?;
render_tool_call(out, tool_call, dialect)?;
}
out.push_str("\n</|DSML|tool_calls>");
write!(out, "\n</{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail");
}
if append_eos {
@@ -513,11 +658,20 @@ fn render_assistant_message(
}
/// Render one assistant tool call in DSML XML-like format.
fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
writeln!(out, "<{DSML_TOKEN}invoke name=\"{}\">", tool_call.name)
.expect("writing to String cannot fail");
encode_arguments_to_dsml(out, tool_call)?;
write!(out, "\n</{DSML_TOKEN}invoke>").expect("writing to String cannot fail");
fn render_tool_call(
out: &mut String,
tool_call: &AssistantToolCall,
dialect: DsDialect,
) -> Result<()> {
let invoke_tag = dialect.invoke_tag();
writeln!(
out,
"<{DSML_TOKEN}{invoke_tag} name=\"{}\">",
tool_call.name
)
.expect("writing to String cannot fail");
encode_arguments_to_dsml(out, tool_call, dialect)?;
write!(out, "\n</{DSML_TOKEN}{invoke_tag}>").expect("writing to String cannot fail");
Ok(())
}
@@ -525,15 +679,20 @@ fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<(
///
/// String values are emitted raw with `string="true"`, while all other JSON
/// values are rendered with JSON syntax and `string="false"`.
fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> {
fn encode_arguments_to_dsml(
out: &mut String,
tool_call: &AssistantToolCall,
dialect: DsDialect,
) -> Result<()> {
let parameter_tag = dialect.parameter_tag();
let arguments: Value = serde_json::from_str(&tool_call.arguments).map_err(|error| {
Error::ChatTemplate(format!(
"assistant tool call has invalid JSON arguments for DeepSeek V4: {error}"
"assistant tool call has invalid JSON arguments for DeepSeek: {error}"
))
})?;
let Some(arguments) = arguments.as_object() else {
return Err(Error::ChatTemplate(
"assistant tool call arguments for DeepSeek V4 must be a JSON object".to_string(),
"assistant tool call arguments for DeepSeek must be a JSON object".to_string(),
));
};
@@ -546,7 +705,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
let is_string = matches!(value, Value::String(_));
write!(
out,
"<{DSML_TOKEN}parameter name=\"{key}\" string=\"{}\">",
"<{DSML_TOKEN}{parameter_tag} name=\"{key}\" string=\"{}\">",
if is_string { "true" } else { "false" }
)
.expect("writing to String cannot fail");
@@ -556,7 +715,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
value => out.push_str(&json_dumps(value)?),
}
write!(out, "</{DSML_TOKEN}parameter>").expect("writing to String cannot fail");
write!(out, "</{DSML_TOKEN}{parameter_tag}>").expect("writing to String cannot fail");
wrote_parameter = true;
}
@@ -565,12 +724,24 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) ->
/// Write chat content directly into the destination buffer without flattening
/// it into an intermediate `String`.
fn write_chat_content(out: &mut String, content: &ChatContent) -> Result<()> {
///
/// The V4.1 dialect inlines the image placeholder at each image part's
/// position unconditionally (matching the Python encoding's
/// `IMAGE_PLACEHOLDER`); other dialects reject multimodal parts.
fn write_chat_content(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
match content {
ChatContent::Text(text) => out.push_str(text),
ChatContent::Parts(parts) => {
for part in parts {
out.push_str(part.as_text()?);
for (index, part) in parts.iter().enumerate() {
if index > 0 && dialect == DsDialect::V41 {
out.push_str("\n\n");
}
match part {
ChatContentPart::ImageUrl { .. } if dialect == DsDialect::V41 => {
out.push_str(DEEPSEEK_V41_IMAGE_PLACEHOLDER);
}
_ => out.push_str(part.as_text()?),
}
}
}
}
@@ -606,7 +777,7 @@ fn json_dumps<T: Serialize>(value: &T) -> Result<String> {
.format_to_string(value)
.map_err(|error| {
Error::ChatTemplate(format!(
"failed to serialize DeepSeek V4 JSON payload: {error}"
"failed to serialize DeepSeek JSON payload: {error}"
))
})
}
@@ -1,10 +1,9 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
mod encoding;
use vllm_text::Prompt;
use super::deepseek::{self, DsDialect};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::Result;
use crate::request::ChatRequest;
@@ -24,7 +23,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
request.validate()?;
Ok(RenderedPrompt {
prompt: Prompt::Text(encoding::render_request(request)?),
prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?),
effective_template_kwargs: request_template_kwargs(request),
})
}
@@ -0,0 +1,38 @@
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
You are a helpful assistant.
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
<|DSML| calls>
<|DSML| invoke name="$TOOL_NAME">
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
...
</|DSML| invoke>
<|DSML| invoke name="$TOOL_NAME2">
...
</|DSML| invoke>
</|DSML| calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
<|DSML| calls>
<|DSML| invoke name="get_weather">
<|DSML| parameter name="location" string="true">Beijing</|DSML| parameter>
<|DSML| parameter name="unit" string="true">celsius</|DSML| parameter>
</|DSML| invoke>
</|DSML| calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
@@ -0,0 +1,3 @@
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
@@ -0,0 +1,30 @@
<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)
<|User|>question<|User|>policy
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
<|DSML| calls>
<|DSML| invoke name="$TOOL_NAME">
<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
...
</|DSML| invoke>
<|DSML| invoke name="$TOOL_NAME2">
...
</|DSML| invoke>
</|DSML| calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "lookup", "parameters": {"type": "object"}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
<|Assistant|><think></think>answer<|end▁of▁sentence|>
@@ -0,0 +1,33 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_text::Prompt;
use super::deepseek::{self, DsDialect};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::Result;
use crate::request::ChatRequest;
/// Dedicated DeepSeek V4.1 renderer.
#[derive(Debug, Clone, Copy, Default)]
pub struct DeepSeekV41ChatRenderer;
impl DeepSeekV41ChatRenderer {
pub fn new() -> Self {
Self
}
}
impl ChatRenderer for DeepSeekV41ChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
Ok(RenderedPrompt {
prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V41)?),
effective_template_kwargs: request_template_kwargs(request),
})
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,223 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::path::PathBuf;
use expect_test::{expect, expect_file};
use serde_json::{Value, json};
use super::DeepSeekV41ChatRenderer;
use crate::ChatRenderer;
use crate::event::AssistantContentBlock;
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ReasoningEffort};
fn render(request: &ChatRequest) -> String {
DeepSeekV41ChatRenderer::new()
.render(request)
.unwrap()
.prompt
.into_text()
.unwrap()
}
fn request() -> ChatRequest {
ChatRequest {
messages: vec![ChatMessage::user("question")],
..ChatRequest::for_test()
}
}
#[test]
fn renders_shared_deepseek_fixtures_with_v41_reference_encoding() {
let fixture_dir =
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/renderer/deepseek_v4/fixtures");
for (input, expected) in [
(
"test_input_1.json",
expect_file!["fixtures/test_output_1.txt"],
),
(
"test_input_2.json",
expect_file!["fixtures/test_output_2.txt"],
),
(
"test_input_developer_tools.json",
expect_file!["fixtures/test_output_developer_tools.txt"],
),
] {
let request = fixture_chat_request(
&fixture_dir.join(input),
FixtureRequestOptions {
enable_thinking: Some(true),
no_generation_prompt_when_last_assistant: true,
},
);
expected.assert_eq(&render(&request));
}
}
#[test]
fn inlines_image_placeholder_at_image_part_positions() {
// The placeholder is unconditional (matching the Python encoding's
// `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later,
// at media preparation.
let renderer = DeepSeekV41ChatRenderer::new();
let request = ChatRequest {
messages: vec![ChatMessage::User {
content: ChatContent::Parts(vec![
ChatContentPart::image_url("https://example.com/carrots.jpeg"),
ChatContentPart::text("what is in the image?"),
]),
}],
..ChatRequest::for_test()
};
let rendered = renderer.render(&request).unwrap().prompt.into_text().unwrap();
// Image parts inline the placeholder at their position; parts stay
// separated by "\n\n" (the V4.1 reference encoding's part separator).
assert!(
rendered.contains("<|User|><|deepseek_image|>\n\nwhat is in the image?"),
"unexpected prompt: {rendered:?}"
);
}
#[test]
fn maps_reasoning_effort_to_reference_numeric_budget() {
for (effort, budget) in [
(None, 50),
(Some(ReasoningEffort::Low), 25),
(Some(ReasoningEffort::High), 50),
(Some(ReasoningEffort::XHigh), 75),
(Some(ReasoningEffort::Max), 100),
] {
let mut request = request();
request.chat_options.reasoning_effort = effort;
assert_eq!(
render(&request),
format!(
"<|begin▁of▁sentence|><|System|>Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n<|User|>question<|Assistant|><think>"
)
);
}
}
#[test]
fn accepts_numeric_template_effort_with_top_level_precedence() {
for (effort, budget) in [
(json!(1), 1),
(json!(42), 42),
(json!(100), 100),
(json!("low"), 25),
] {
let mut request = request();
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100")));
request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100"));
}
}
#[test]
fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
for effort in [
json!("medium"),
json!("minimal"),
json!(-1),
json!(0),
json!(101),
json!(true),
json!(1.5),
] {
let mut request = request();
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
assert!(error.is_request_validation_error());
}
for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
let mut request = request();
request.chat_options.reasoning_effort = Some(effort);
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
assert!(error.is_request_validation_error());
}
}
#[test]
fn chat_mode_and_none_effort_use_closed_thinking_prefix() {
let mut request = request();
request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
let expected = expect!["<|begin▁of▁sentence|><|User|>question<|Assistant|></think>"];
expected.assert_eq(&render(&request));
request.chat_options.template_kwargs.insert("thinking".into(), json!(true));
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
expected.assert_eq(&render(&request));
}
#[test]
fn leading_system_in_chat_mode_uses_system_token() {
let mut request = request();
request.messages.insert(0, ChatMessage::system("policy"));
request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
expect!["<|begin▁of▁sentence|><|System|>policy<|User|>question<|Assistant|></think>"]
.assert_eq(&render(&request));
}
#[test]
fn system_after_dropped_developer_becomes_leading_system() {
let mut request = ChatRequest {
messages: vec![
ChatMessage::developer("old policy", None),
ChatMessage::system("sys"),
],
..ChatRequest::for_test()
};
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys"]
.assert_eq(&render(&request));
request.messages.push(ChatMessage::user("question"));
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>question<|Assistant|><think>"]
.assert_eq(&render(&request));
}
#[test]
fn mid_system_advances_last_user_boundary_and_drops_old_reasoning() {
let mut request = ChatRequest {
messages: vec![
ChatMessage::system("sys"),
ChatMessage::user("q1"),
ChatMessage::assistant_blocks(vec![
AssistantContentBlock::Reasoning { text: "r1".into() },
AssistantContentBlock::Text { text: "a1".into() },
]),
ChatMessage::developer("old policy", None),
ChatMessage::system("mid sys"),
],
..ChatRequest::for_test()
};
request
.chat_options
.template_kwargs
.insert("reasoning_effort".into(), json!(88));
expect!["<|begin▁of▁sentence|><|System|>Reasoning Effort: 88 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<|User|>q1<|Assistant|></think>a1<|end▁of▁sentence|><|System|>mid sys<|Assistant|><think>"]
.assert_eq(&render(&request));
}
#[test]
fn joins_text_parts_with_reference_separator_and_preserves_literal_tags() {
let mut request = ChatRequest {
messages: vec![ChatMessage::user(ChatContent::Parts(vec![
ChatContentPart::text("first"),
ChatContentPart::text("<|DSML|invoke name=\"literal\">"),
]))],
..ChatRequest::for_test()
};
request
.chat_options
.template_kwargs
.insert("thinking".into(), Value::Bool(false));
expect!["<|begin▁of▁sentence|><|User|>first\n\n<|DSML|invoke name=\"literal\"><|Assistant|></think>"]
.assert_eq(&render(&request));
}
+3
View File
@@ -10,8 +10,10 @@ use vllm_text::Prompt;
use crate::error::Result;
use crate::request::{ChatRequest, ReasoningEffort};
mod deepseek;
pub mod deepseek_v32;
pub mod deepseek_v4;
pub mod deepseek_v41;
pub mod harmony;
pub mod hf;
mod inkling;
@@ -22,6 +24,7 @@ mod test_utils;
pub use deepseek_v4::DeepSeekV4ChatRenderer;
pub use deepseek_v32::DeepSeekV32ChatRenderer;
pub use deepseek_v41::DeepSeekV41ChatRenderer;
pub use harmony::HarmonyChatRenderer;
pub use inkling::InklingChatRenderer;
pub use kimi_k3::KimiK3ChatRenderer;
+8 -1
View File
@@ -22,6 +22,8 @@ pub enum RendererSelection {
DeepSeekV32,
/// Force the DeepSeek V4 renderer.
DeepSeekV4,
/// Force the DeepSeek V4.1 renderer.
DeepSeekV41,
/// Force the GPT-OSS Harmony renderer.
Harmony,
/// Force the Inkling native token renderer.
@@ -34,6 +36,7 @@ impl RendererSelection {
pub const AUTO_LITERAL: &str = "auto";
pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32";
pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4";
pub const DEEPSEEK_V41_LITERAL: &str = "deepseek_v41";
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
pub const HARMONY_LITERAL: &str = "harmony";
pub const HF_LITERAL: &str = "hf";
@@ -48,6 +51,7 @@ impl RendererSelection {
Self::Auto => match model_type {
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
Self::DEEPSEEK_V41_LITERAL => Self::DeepSeekV41,
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
Self::INKLING_MODEL_TYPE => Self::Inkling,
Self::KIMI_K3_LITERAL => Self::KimiK3,
@@ -70,6 +74,8 @@ impl FromStr for RendererSelection {
Ok(Self::DeepSeekV32)
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) {
Ok(Self::DeepSeekV4)
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V41_LITERAL) {
Ok(Self::DeepSeekV41)
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
Ok(Self::Harmony)
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
@@ -92,6 +98,7 @@ impl fmt::Display for RendererSelection {
Self::Hf => f.write_str(Self::HF_LITERAL),
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
Self::DeepSeekV41 => f.write_str(Self::DEEPSEEK_V41_LITERAL),
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL),
@@ -121,7 +128,7 @@ mod tests {
fn renderer_selection_expected_error_message() {
let err = RendererSelection::from_str("unknown").unwrap_err();
expect_test::expect![
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)"
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)"
]
.assert_eq(&err);
}
+15 -1
View File
@@ -29,7 +29,7 @@ const TOKEN_COMPLETION_CHUNK_TOKENS: usize = 1;
/// One model/parser configuration used to run the fixed roundtrip fixtures.
#[derive(Clone)]
struct RoundtripCase {
/// Hugging Face model id resolved through the production backend loader.
/// Hugging Face model id or local checkpoint resolved through the production backend loader.
model_id: &'static str,
/// Final assistant-history suffix rendered by the chat template but not
/// generated by the model body consumed by the output processor.
@@ -180,6 +180,19 @@ impl RoundtripCase {
}
}
/// DeepSeek V4.1 Flash DSML tool-call format.
fn deepseek_v41() -> Self {
Self {
model_id: "deepseek-ai/DeepSeek-V4.1-Flash",
assistant_stop_suffix: "<|end▁of▁sentence|>",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
json_fmt: compact_json_fmt(),
sort_json_keys: false,
}
}
/// DeepSeek V3.2 DSML tool-call format.
fn deepseek_v32() -> Self {
Self {
@@ -385,6 +398,7 @@ roundtrip_tests! {
minimax_m25 => [reasoning_and_content, tool_call_mix],
minimax_m3 => [reasoning_and_content, tool_call_mix],
deepseek_v4 => [reasoning_and_content, tool_call_mix],
deepseek_v41 => [reasoning_and_content, tool_call_mix],
deepseek_v32 => [tool_call_mix],
glm45 => [reasoning_and_content, tool_call_mix],
glm47 => [reasoning_and_content, tool_call_mix],
+1 -1
View File
@@ -903,7 +903,7 @@ fn serve_args_reject_unknown_renderer_value() {
.unwrap_err();
expect![[r#"
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)
For more information, try '--help'.
"#]]
+2
View File
@@ -53,6 +53,8 @@ pub type Step3ReasoningParser = DeepSeekR1ReasoningParser;
/// DeepSeek V4 currently shares the DeepSeek V3 reasoning parser.
pub type DeepSeekV4ReasoningParser = DeepSeekV3ReasoningParser;
/// DeepSeek V4.1 uses bare `<think>...</think>` delimiters.
pub type DeepSeekV41ReasoningParser = DeepSeekV3ReasoningParser;
/// GLM-4.7 and GLM-5 use bare reasoning delimiters.
pub type Glm47ReasoningParser = DeepSeekV3ReasoningParser;
/// Kimi K2 currently shares the DeepSeek V3 reasoning parser.
@@ -0,0 +1,47 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use super::{DeepSeekDsmlToolParser, DsmlTokens};
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
mod structural_tag;
/// Tool parser for DeepSeek V4.1's spaced DSML tags.
///
/// Arguments are emitted only after a full `invoke` block is parsed.
pub struct DeepSeekV41ToolParser(DeepSeekDsmlToolParser);
impl ToolParser for DeepSeekV41ToolParser {
fn create(tools: &[Tool]) -> Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Ok(Box::new(Self(DeepSeekDsmlToolParser::new(
tools,
DsmlTokens::V41,
))))
}
fn preserve_special_tokens(&self) -> bool {
true
}
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
Some(&structural_tag::DeepSeekV41StructuralTagBuilder)
}
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
self.0.parse_into(chunk, output)
}
fn finish(&mut self) -> Result<ToolParserOutput> {
self.0.finish()
}
fn reset(&mut self) -> String {
self.0.reset()
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,108 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Visible-text grammar for DeepSeek V4.1 tool calls.
//!
//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with
//! V4.1's spaced DSML tags and schema-unconstrained parameter values.
use serde_json::Value;
use xgrammar_structural_tag::Result;
use xgrammar_structural_tag::builders::{
StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
};
use xgrammar_structural_tag::format::{
Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat,
};
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
use super::DsmlTokens;
const TOKENS: DsmlTokens = DsmlTokens::V41;
pub(super) struct DeepSeekV41StructuralTagBuilder;
impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder {
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
let tags = ctx
.function_tools
.iter()
.map(|tool| call_tag(tool, ctx.options))
.collect::<Vec<_>>();
let excludes = if ctx.options.exclude_special_tokens {
&["<think>", "</think>"][..]
} else {
&[]
};
let format = match ctx.tool_choice {
BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes),
BuilderToolChoice::Auto => Format::TriggeredTags(
TriggeredTagsFormat::new(
&[TOKENS.tool_calls_start],
vec![TagFormat::new(
format!("{}\n", TOKENS.tool_calls_start),
Format::tags_with_separator(tags, "", true, false),
TOKENS.tool_calls_end,
)],
)
.with_excludes(excludes),
),
BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![
Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)),
Format::tags_with_separator(
tags,
"",
true,
ctx.tool_choice == BuilderToolChoice::Forced,
),
Format::const_string(TOKENS.tool_calls_end),
]),
};
// Reasoning boundaries remain owned by the engine reasoner; serving
// calls this visible-text builder with reasoning=false.
Ok(StructuralTag::new(format))
}
}
fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
// TODO: enforce tool.parameters and strict schema semantics once V4.1
// argument conversion is available. This builder intentionally ignores
// parameters and strict: it constrains tool names and DSML/value syntax
// only. Parameter names, presence, uniqueness and value schemas remain
// unconstrained, including for strict=true. Serving still uses strict to
// decide whether auto tool choice activates a grammar.
TagFormat::new(
format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name),
Format::star(parameter(options)),
format!("{}\n", TOKENS.invoke_end),
)
}
fn parameter(options: StructuralTagOptions) -> Format {
Format::tag(
format!("{} name=\"", TOKENS.parameter_start),
Format::sequence(vec![
Format::regex(r#"[^"]+"#),
Format::const_string("\" string=\""),
Format::or(vec![
Format::sequence(vec![
Format::const_string("true\">"),
Format::any_text_excluding(&[
TOKENS.parameter_end,
TOKENS.invoke_end,
TOKENS.tool_calls_end,
]),
]),
Format::sequence(vec![
Format::const_string("false\">"),
Format::JsonSchema(
JsonSchemaFormat::new(Value::Bool(true))
.with_any_order(options.any_order)
.with_max_whitespace_cnt(options.max_whitespace_cnt),
),
]),
]),
]),
format!("{}\n", TOKENS.parameter_end),
)
}
@@ -0,0 +1,113 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use expect_test::expect;
use serde_json::json;
use xgrammar_structural_tag::builders::StructuralTagOptions;
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
};
use super::DeepSeekV41ToolParser;
use crate::tool::test_utils::collect_stream;
use crate::tool::{Tool, ToolParser};
fn tools() -> Vec<Tool> {
vec![Tool {
name: "lookup".into(),
description: None,
parameters: json!({
"type": "object",
"properties": {
"query": {"type": "string"},
"limit": {"type": "integer"},
"enabled": {"type": "boolean"},
"data": {"type": "object"},
"items": {"type": "array"},
"label": {"type": "string"}
}
}),
strict: None,
}]
}
#[test]
fn spaced_dsml_streaming_preserves_shared_schema_coercion() {
let wire = concat!(
"summary\n\n<|DSML| calls>\n",
"<|DSML| invoke name=\"lookup\">\n",
"<|DSML| parameter name=\"query\" string=\"true\"> value\n</|DSML| parameter>\n",
"<|DSML| parameter name=\"limit\" string=\"false\">2</|DSML| parameter>\n",
"<|DSML| parameter name=\"enabled\" string=\"false\">true</|DSML| parameter>\n",
"<|DSML| parameter name=\"data\" string=\"false\">{\"x\":1}</|DSML| parameter>\n",
"<|DSML| parameter name=\"items\" string=\"false\">[1,2]</|DSML| parameter>\n",
"<|DSML| parameter name=\"label\" string=\"false\">42</|DSML| parameter>\n",
"<|DSML| parameter name=\"unknown\" string=\"false\">42</|DSML| parameter>\n",
"<|DSML| parameter name=\"nullish\" string=\"false\">null</|DSML| parameter>\n",
"</|DSML| invoke>\n",
"<|DSML| invoke name=\"lookup\">\n",
"<|DSML| parameter name=\"query\" string=\"true\">second</|DSML| parameter>\n",
"</|DSML| invoke>\n</|DSML| calls>",
);
let expected = [
json!({"query":" value\n","limit":2,"enabled":true,"data":{"x":1},"items":[1,2],"label":"42","unknown":"42","nullish":null}),
json!({"query":"second"}),
];
for split in wire.char_indices().map(|(i, _)| i).chain(std::iter::once(wire.len())) {
let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
let output = collect_stream(parser.as_mut(), &[&wire[..split], &wire[split..]]);
// Protocol framing follows the current shared DSML behavior.
assert_eq!(output.normal_text(), "summary", "split {split}");
assert_eq!(output.calls().len(), 2, "split {split}");
for (index, call) in output.calls().iter().enumerate() {
assert_eq!(call.tool_index, index, "split {split}");
assert_eq!(call.name.as_deref(), Some("lookup"), "split {split}");
assert_eq!(
serde_json::from_str::<serde_json::Value>(&call.arguments).unwrap(),
expected[index],
"split {split}"
);
}
}
}
#[test]
fn v4_tags_remain_text_in_v41_dialect() {
let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
let output = collect_stream(parser.as_mut(), &["before<|DSML|tool_calls>after"]);
expect!["before<|DSML|tool_calls>after"].assert_eq(&output.normal_text());
assert!(output.calls().is_empty());
}
#[test]
fn structural_tag_ignores_parameter_schemas_and_strict() {
let parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
let builder = parser.structural_tag_builder().unwrap();
let options = StructuralTagOptions::default().with_reasoning(false);
for choice in [
ToolChoice::auto(),
ToolChoice::required(),
ToolChoice::function("lookup"),
] {
let build = |function| {
build_structural_tag(
builder,
&[ToolParam::Function(FunctionToolParam::new(function))],
choice.clone(),
options,
)
.unwrap()
};
let expected = build(FunctionDefinition::new("lookup"));
for strict in [None, Some(false), Some(true)] {
for parameters in [
json!(false),
json!({"type":"object", "properties":{"query":{"const":"fixed"}}, "required":["query"]}),
] {
let mut function = FunctionDefinition::new("lookup").with_parameters(parameters);
function.strict = strict;
assert_eq!(build(function), expected);
}
}
}
}
+40 -16
View File
@@ -14,14 +14,11 @@ use crate::tool::Tool;
mod deepseek_v32;
mod deepseek_v4;
mod deepseek_v41;
pub use deepseek_v4::DeepSeekV4ToolParser;
pub use deepseek_v32::DeepSeekV32ToolParser;
const INVOKE_START: &str = "<|DSML|invoke";
const INVOKE_END: &str = "</|DSML|invoke>";
const PARAMETER_START: &str = "<|DSML|parameter";
const PARAMETER_END: &str = "</|DSML|parameter>";
pub use deepseek_v41::DeepSeekV41ToolParser;
type DsmlInput<'i> = Partial<&'i str>;
@@ -30,6 +27,10 @@ struct DsmlTokens {
tool_calls_start: &'static str,
framed_tool_calls_start: &'static str,
tool_calls_end: &'static str,
invoke_start: &'static str,
invoke_end: &'static str,
parameter_start: &'static str,
parameter_end: &'static str,
}
impl DsmlTokens {
@@ -37,11 +38,25 @@ impl DsmlTokens {
tool_calls_start: "<|DSML|function_calls>",
framed_tool_calls_start: "\n\n<|DSML|function_calls>",
tool_calls_end: "</|DSML|function_calls>",
invoke_start: "<|DSML|invoke",
invoke_end: "</|DSML|invoke>",
parameter_start: "<|DSML|parameter",
parameter_end: "</|DSML|parameter>",
};
const V4: Self = Self {
tool_calls_start: "<|DSML|tool_calls>",
framed_tool_calls_start: "\n\n<|DSML|tool_calls>",
tool_calls_end: "</|DSML|tool_calls>",
..Self::V32
};
const V41: Self = Self {
tool_calls_start: "<|DSML| calls>",
framed_tool_calls_start: "\n\n<|DSML| calls>",
tool_calls_end: "</|DSML| calls>",
invoke_start: "<|DSML| invoke",
invoke_end: "</|DSML| invoke>",
parameter_start: "<|DSML| parameter",
parameter_end: "</|DSML| parameter>",
};
}
@@ -205,7 +220,7 @@ fn parse_tool_block_event(
) -> ModalResult<DsmlEvent> {
ws0.void().parse_next(input)?;
alt((
|input: &mut DsmlInput<'_>| invoke_event(input, invoke_end_scan),
|input: &mut DsmlInput<'_>| invoke_event(input, tokens, invoke_end_scan),
|input: &mut DsmlInput<'_>| tool_calls_end_event(input, tokens),
))
.parse_next(input)
@@ -243,19 +258,20 @@ fn safe_text_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult
/// Parse a DSML invoke block.
fn invoke_event(
input: &mut DsmlInput<'_>,
tokens: DsmlTokens,
invoke_end_scan: &mut MarkerScanState,
) -> ModalResult<DsmlEvent> {
let (name, body) = seq!(
_: literal(INVOKE_START),
_: literal(tokens.invoke_start),
_: ws1,
dsml_name_attr,
_: ws0,
_: ">",
take_until_marker(INVOKE_END, invoke_end_scan),
_: literal(INVOKE_END),
take_until_marker(tokens.invoke_end, invoke_end_scan),
_: literal(tokens.invoke_end),
)
.parse_next(input)?;
let raw_params = parse_invoke_params(body)?;
let raw_params = parse_invoke_params(body, tokens)?;
Ok(DsmlEvent::Invoke {
name: name.to_string(),
raw_params,
@@ -263,23 +279,31 @@ fn invoke_event(
}
/// Parse a DSML invoke body.
fn parse_invoke_params(invoke_body: &str) -> ModalResult<Vec<DsmlParameter>> {
fn parse_invoke_params(invoke_body: &str, tokens: DsmlTokens) -> ModalResult<Vec<DsmlParameter>> {
let mut input = invoke_body;
delimited(ws0, repeat(0.., terminated(parse_parameter, ws0)), eof).parse_next(&mut input)
delimited(
ws0,
repeat(
0..,
terminated(|input: &mut &str| parse_parameter(input, tokens), ws0),
),
eof,
)
.parse_next(&mut input)
}
/// Parse a DSML parameter block.
fn parse_parameter(input: &mut &str) -> ModalResult<DsmlParameter> {
fn parse_parameter(input: &mut &str, tokens: DsmlTokens) -> ModalResult<DsmlParameter> {
seq! {DsmlParameter {
_: literal(PARAMETER_START),
_: literal(tokens.parameter_start),
_: ws1,
name: name_attr.map(|name: &str| name.to_string()),
_: ws1,
is_string: string_attr.map(|value| value == "true"),
_: ws0,
_: ">",
value: take_until(0.., PARAMETER_END).map(str::to_string),
_: literal(PARAMETER_END),
value: take_until(0.., tokens.parameter_end).map(str::to_string),
_: literal(tokens.parameter_end),
}}
.parse_next(input)
}
+1 -1
View File
@@ -20,7 +20,7 @@ mod seed_oss;
pub mod test_utils;
use std::collections::{BTreeMap, btree_map};
pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser, DeepSeekV41ToolParser};
pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
pub use error::{Result, ToolParserError};
pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
+13
View File
@@ -167,6 +167,19 @@ mod tests {
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
}
#[test]
fn invalid_reasoning_effort_maps_to_invalid_request() {
let error = vllm_chat::Error::InvalidReasoningEffort(
"DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(),
);
let api_error = chat_submit_error("failed to submit chat request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
assert_eq!(
api_error.to_error_response().error.error_type,
"invalid_request_error"
);
}
#[test]
fn llm_wrapped_empty_prompt_maps_to_invalid_request() {
let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds {