diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 29b3eb268a..7cd52d6b1c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" version = "1.7.1" -source = "git+https://github.com/smg-project/llm-multimodal?rev=74f64753e040ade87c9d7f4bb0ad2f4dd7764652#74f64753e040ade87c9d7f4bb0ad2f4dd7764652" +source = "git+https://github.com/smg-project/llm-multimodal?rev=24c676dd49d73f05cb4068aade2e60397a5c0afe#24c676dd49d73f05cb4068aade2e60397a5c0afe" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 5c76f09855..a9421e7c4a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -60,7 +60,7 @@ indexmap = "2.13.0" indicatif = "0.18.4" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "74f64753e040ade87c9d7f4bb0ad2f4dd7764652", default-features = false, features = ["native-tls"] } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "24c676dd49d73f05cb4068aade2e60397a5c0afe", default-features = false, features = ["native-tls"] } mimalloc = "0.1.52" minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.24", features = ["pycompat"] } diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 0be33f91a8..717bde2898 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -19,8 +19,8 @@ use crate::output::{ }; use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo}; use crate::renderer::{ - DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer, - InklingChatRenderer, KimiK3ChatRenderer, + DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer, + HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, }; use crate::request::ChatRequest; use crate::{DynChatOutputProcessor, RendererSelection}; @@ -72,6 +72,7 @@ impl HfChatBackend { )?), RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()), RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()), + RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()), RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?), RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?), RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())), diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index 082e665dbb..e3299dfdd4 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -17,6 +17,8 @@ pub enum Error { MissingChatTemplate, #[error("chat template error: {0}")] ChatTemplate(String), + #[error("{0}")] + InvalidReasoningEffort(String), #[error("multimodal input is not supported by this chat renderer")] UnsupportedMultimodalRenderer, #[error("unsupported multimodal content: {0}")] @@ -96,6 +98,7 @@ impl Error { pub fn is_request_validation_error(&self) -> bool { match self { Self::PromptTooLong { .. } + | Self::InvalidReasoningEffort(_) | Self::DuplicateToolName { .. } | Self::ToolChoiceRequiresTools | Self::ToolChoiceFunctionNotFound { .. } => true, diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 6bb87cfec4..6250330e48 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -33,8 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory}; pub use parser::{ParserSelection, validate_parser_overrides}; pub use renderer::hf::ChatTemplateContentFormatOption; pub use renderer::{ - ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, - HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt, + ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, + DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt, RendererSelection, }; pub use request::{ @@ -363,7 +363,7 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); } #[test] @@ -374,6 +374,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 174a0d4bb0..a0743b8769 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -153,6 +153,10 @@ struct ResolvedMultimodalSpec { modality: Modality, field_layouts: EncoderFieldLayouts, keep_on_cpu_keys: HashSet, + /// Spec-declared wire key for the primary encoder input tensor, when it + /// differs from the per-modality default (e.g. `"patches"` for + /// DeepSeek-V4.1 images). + encoder_input_key: Option, } impl ResolvedMultimodalSpec { @@ -162,10 +166,14 @@ impl ResolvedMultimodalSpec { modality, field_layouts: raw.encoder_field_layouts_for(modality), keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(), + encoder_input_key: raw.encoder_input_key_for(modality), } } - fn primary_key(&self) -> &'static str { + fn primary_key(&self) -> &str { + if let Some(key) = &self.encoder_input_key { + return key; + } match self.modality { Modality::Image => image::IMAGE_PRIMARY_KEY, Modality::Video => video::VIDEO_PRIMARY_KEY, @@ -841,6 +849,9 @@ mod tests { pub(super) const QWEN3_IMAGE_PAD_ID: u32 = 151655; pub(super) const QWEN3_VIDEO_PAD_ID: u32 = 151656; + pub(super) const DEEPSEEK_V41_IMAGE_ID: u32 = 129264; + pub(super) const DEEPSEEK_V41_IMAGE_PAD_ID: u32 = 129265; + fn llama4_tokenizer() -> TestTokenizer { TestTokenizer::new() .with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID) @@ -954,6 +965,33 @@ mod tests { assert_eq!(info.placeholder_token(Modality::Video), None); } + fn deepseek_v41_info() -> MultimodalModelInfo { + let config = serde_json::json!({ + "model_type": "deepseek_v41", + "image_token_id": DEEPSEEK_V41_IMAGE_ID, + }); + let tokenizer = TestTokenizer::new() + .with_regular_token("<｜deepseek_image｜>", DEEPSEEK_V41_IMAGE_ID) + .with_regular_token("<|place_holder_mm_span_0436|>", DEEPSEEK_V41_IMAGE_PAD_ID); + test_info("deepseek_v41", config, tokenizer) + } + + #[test] + fn deepseek_v41_resolves_image_support_only() { + let info = deepseek_v41_info(); + + assert_eq!( + info.placeholder_token(Modality::Image), + Some("<｜deepseek_image｜>") + ); + assert_eq!(info.placeholder_token(Modality::Video), None); + let image = info.image.as_ref().expect("image support"); + assert_eq!(image.placeholder.marker_token_id, DEEPSEEK_V41_IMAGE_ID); + assert_eq!(image.placeholder.embed_token_id, DEEPSEEK_V41_IMAGE_ID); + // The engine's forward kwargs pop `patches` (not `pixel_values`). + assert_eq!(image.spec.primary_key(), "patches"); + } + #[test] fn input_audio_uses_connector_data_urls() { assert_eq!( diff --git a/rust/src/chat/src/multimodal/expand.rs b/rust/src/chat/src/multimodal/expand.rs index d9c9c54b34..1172d0ccc0 100644 --- a/rust/src/chat/src/multimodal/expand.rs +++ b/rust/src/chat/src/multimodal/expand.rs @@ -45,6 +45,14 @@ impl<'a> ExpansionLane<'a> { /// matching the original media-part order within that modality; markers of /// different modalities may interleave freely. /// +/// Replacements may carry an [`AlignmentPad`](llm_multimodal::AlignmentPad) +/// (e.g. DeepSeek-V4.1's compressor-alignment pad): `period - 1 - offset % +/// period` pad tokens are +/// prepended at splice time, where `offset` is the position the replacement +/// lands at in the expanded prompt. The pads are part of the reported range +/// and count as embed positions only if the pad token id equals the +/// modality's embed token id. +/// /// The returned ranges point into the already-expanded prompt, grouped per /// modality in item order. pub(super) fn expand_prompt_token_ids( @@ -56,12 +64,14 @@ pub(super) fn expand_prompt_token_ids( return Ok(HashMap::new()); } - let replacement_growth = lanes - .iter() - .flat_map(|lane| lane.replacements.iter()) - .fold(0usize, |total, replacement| { - total.saturating_add(replacement.tokens.len().saturating_sub(1)) - }); + let replacement_growth = lanes.iter().flat_map(|lane| lane.replacements.iter()).fold( + 0usize, + |total, replacement| { + // Alignment pads are position-dependent; reserve the worst case. + let max_pads = replacement.alignment_pad.map_or(0, |pad| pad.period - 1); + total.saturating_add(max_pads + replacement.tokens.len().saturating_sub(1)) + }, + ); let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth); let mut expanded = Vec::with_capacity(expanded_len); @@ -95,10 +105,20 @@ pub(super) fn expand_prompt_token_ids( expanded.len() ); } - let range_len = replacement_len.saturating_add(structural_prefix); + // Position-dependent alignment padding, computed from the offset the + // replacement block lands at in the expanded prompt (mirroring + // vLLM's `_apply_token_matches_with_placeholders` splice-time pad). + let pad_count = replacement.alignment_pad.map_or(0, |pad| pad.count_at(expanded.len())); + let range_len = structural_prefix + pad_count + replacement_len; let is_embed = { let mut mask = Vec::with_capacity(range_len); mask.extend(std::iter::repeat_n(false, structural_prefix)); + if let Some(pad) = &replacement.alignment_pad { + mask.extend(std::iter::repeat_n( + pad.token_id as u32 == lane.embed_token_id, + pad_count, + )); + } mask.extend( replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id), ); @@ -106,6 +126,9 @@ pub(super) fn expand_prompt_token_ids( }; let expanded_offset = expanded.len() - structural_prefix; + if let Some(pad) = &replacement.alignment_pad { + expanded.extend(std::iter::repeat_n(pad.token_id as u32, pad_count)); + } expanded.extend(replacement.tokens.iter().map(|&token| token as u32)); ranges.entry(lane.modality).or_default().push(PlaceholderRange { offset: expanded_offset, @@ -454,7 +477,6 @@ mod tests { } const QWEN3_AUDIO_PAD_ID: u32 = 151_676; - fn qwen3_audio_prepared(replacements: Vec) -> PreparedMedia { prepared_media( Modality::Audio, @@ -542,4 +564,110 @@ mod tests { )); assert_eq!(prompt_token_ids, original_prompt_token_ids); } + + // DeepSeek-V4.1 image spans: every span position carries the image token + // id; a compressor-alignment pad (id 129265, period 2) is prepended at + // splice time so spans always start at the same compressor phase. + const DS_IMAGE_ID: u32 = 129_264; + const DS_IMAGE_PAD_ID: u32 = 129_265; + + fn deepseek_prepared(span_lens: Vec) -> PreparedMedia { + prepared_media( + Modality::Image, + "<｜deepseek_image｜>", + DS_IMAGE_ID, + DS_IMAGE_ID, + span_lens + .into_iter() + .map(|len| { + PromptReplacement::repeated( + Modality::Image, + "<｜deepseek_image｜>", + DS_IMAGE_ID as TokenId, + len, + ) + .with_alignment_pad(DS_IMAGE_PAD_ID as TokenId, 2) + }) + .collect(), + ) + } + + #[test] + fn expand_prompt_tokens_skips_alignment_pad_at_odd_offset() { + // Odd block start: already at the target compressor phase, no pad. + let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2]; + let prepared = vec![deepseek_prepared(vec![3])]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + let ranges = &ranges[&Modality::Image]; + + assert_eq!( + prompt_token_ids, + vec![1, DS_IMAGE_ID, DS_IMAGE_ID, DS_IMAGE_ID, 2] + ); + assert_eq!(ranges[0].offset, 1); + assert_eq!(ranges[0].length, 3); + assert_bool_mask(&ranges[0], &[true, true, true]); + } + + #[test] + fn expand_prompt_tokens_prepends_alignment_pad_at_even_offset() { + // Even block start: one pad token aligns the span to phase 1. + let mut prompt_token_ids = vec![1, 3, DS_IMAGE_ID, 2]; + let prepared = vec![deepseek_prepared(vec![3])]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + let ranges = &ranges[&Modality::Image]; + + assert_eq!( + prompt_token_ids, + vec![ + 1, + 3, + DS_IMAGE_PAD_ID, + DS_IMAGE_ID, + DS_IMAGE_ID, + DS_IMAGE_ID, + 2 + ] + ); + // The range starts at the pad; pads are not embed positions. + assert_eq!(ranges[0].offset, 2); + assert_eq!(ranges[0].length, 4); + assert_bool_mask(&ranges[0], &[false, true, true, true]); + } + + #[test] + fn expand_prompt_tokens_alignment_pad_realigns_each_span() { + // Every image span must start at phase 1 (odd offset) regardless of + // where the previous span landed. + let mut prompt_token_ids = vec![1, DS_IMAGE_ID, 2, DS_IMAGE_ID, 3]; + let prepared = vec![deepseek_prepared(vec![2, 2])]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + let ranges = &ranges[&Modality::Image]; + + // First span at odd offset 1: no pad (block [1,3)). + // Second marker originally at index 3; after the first expansion the + // block starts at offset 4 (even): one pad, span at odd offset 5. + assert_eq!( + prompt_token_ids, + vec![ + 1, + DS_IMAGE_ID, + DS_IMAGE_ID, + 2, + DS_IMAGE_PAD_ID, + DS_IMAGE_ID, + DS_IMAGE_ID, + 3, + ] + ); + assert_eq!((ranges[0].offset, ranges[0].length), (1, 2)); + assert_bool_mask(&ranges[0], &[true, true]); + assert_eq!((ranges[1].offset, ranges[1].length), (4, 3)); + assert_bool_mask(&ranges[1], &[false, true, true]); + // Both spans (past the pad) start at odd offsets. + assert_eq!((ranges[1].offset + 1) % 2, 1); + } } diff --git a/rust/src/chat/src/parser/reasoning/mod.rs b/rust/src/chat/src/parser/reasoning/mod.rs index e37e92054a..c283b3056f 100644 --- a/rust/src/chat/src/parser/reasoning/mod.rs +++ b/rust/src/chat/src/parser/reasoning/mod.rs @@ -7,10 +7,11 @@ use std::sync::{Arc, LazyLock}; pub use vllm_parser::reasoning::{ CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser, - DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser, - KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser, - NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError, - ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser, + DeepSeekV4ReasoningParser, DeepSeekV41ReasoningParser, Glm45ReasoningParser, + Glm47ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser, MiniMaxM2ReasoningParser, + MiniMaxM3ReasoningParser, NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, + ReasoningError, ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, + Step3p5ReasoningParser, }; use vllm_tokenizer::DynTokenizer; @@ -22,6 +23,7 @@ pub mod names { pub const DEEPSEEK_R1: &str = "deepseek_r1"; pub const DEEPSEEK_V3: &str = "deepseek_v3"; pub const DEEPSEEK_V4: &str = "deepseek_v4"; + pub const DEEPSEEK_V41: &str = "deepseek_v41"; pub const GLM45: &str = "glm45"; pub const GLM47: &str = "glm47"; pub const KIMI: &str = "kimi"; @@ -62,6 +64,7 @@ impl ReasoningParserFactory { .register_parser::(names::DEEPSEEK_R1) .register_parser::(names::DEEPSEEK_V3) .register_parser::(names::DEEPSEEK_V4) + .register_parser::(names::DEEPSEEK_V41) .register_parser::(names::GLM45) .register_parser::(names::GLM47) .register_parser::(names::KIMI) @@ -76,6 +79,7 @@ impl ReasoningParserFactory { factory .register_pattern("deepseek-r1", names::DEEPSEEK_R1) + .register_pattern("deepseek-v4.1", names::DEEPSEEK_V41) .register_pattern("deepseek-v4", names::DEEPSEEK_V4) .register_pattern("deepseek_v4", names::DEEPSEEK_V4) .register_pattern("deepseek-v3", names::DEEPSEEK_V3) diff --git a/rust/src/chat/src/parser/reasoning/tests.rs b/rust/src/chat/src/parser/reasoning/tests.rs index f2db9b08c7..6b032a233f 100644 --- a/rust/src/chat/src/parser/reasoning/tests.rs +++ b/rust/src/chat/src/parser/reasoning/tests.rs @@ -33,6 +33,10 @@ fn factory_resolves_deepseek_v4() { factory.resolve_name_for_model("deepseek_v4"), Some(names::DEEPSEEK_V4) ); + assert_eq!( + factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"), + Some(names::DEEPSEEK_V41) + ); } #[test] diff --git a/rust/src/chat/src/parser/tool/mod.rs b/rust/src/chat/src/parser/tool/mod.rs index bf76a4b315..26a2ecc4c7 100644 --- a/rust/src/chat/src/parser/tool/mod.rs +++ b/rust/src/chat/src/parser/tool/mod.rs @@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock}; pub use vllm_parser::tool::{ DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser, - Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, - Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser, - MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser, - Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError, + DeepSeekV41ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, + HermesToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, + MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, + Qwen3CoderToolParser, Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError, }; use crate::parser::ParserFactory; @@ -22,6 +22,7 @@ pub mod names { pub const DEEPSEEK_V31: &str = "deepseek_v31"; pub const DEEPSEEK_V32: &str = "deepseek_v32"; pub const DEEPSEEK_V4: &str = "deepseek_v4"; + pub const DEEPSEEK_V41: &str = "deepseek_v41"; pub const GLM45: &str = "glm45"; pub const GLM47: &str = "glm47"; pub const GRANITE4: &str = "granite4"; @@ -66,6 +67,7 @@ impl ToolParserFactory { .register_parser::(names::DEEPSEEK_V31) .register_parser::(names::DEEPSEEK_V32) .register_parser::(names::DEEPSEEK_V4) + .register_parser::(names::DEEPSEEK_V41) .register_parser::(names::GLM45) .register_parser::(names::GLM47) .register_parser::(names::GRANITE4) @@ -101,6 +103,7 @@ impl ToolParserFactory { .register_pattern("llama-3.2", names::LLAMA3_JSON) .register_pattern("llama-3.1", names::LLAMA3_JSON) .register_pattern("deepseek-r1", names::DEEPSEEK_V3) + .register_pattern("deepseek-v4.1", names::DEEPSEEK_V41) .register_pattern("deepseek-v4", names::DEEPSEEK_V4) .register_pattern("deepseek_v4", names::DEEPSEEK_V4) .register_pattern("deepseek-v3.2", names::DEEPSEEK_V32) diff --git a/rust/src/chat/src/parser/tool/tests.rs b/rust/src/chat/src/parser/tool/tests.rs index 381a3bbf3b..24984c0731 100644 --- a/rust/src/chat/src/parser/tool/tests.rs +++ b/rust/src/chat/src/parser/tool/tests.rs @@ -147,6 +147,10 @@ fn factory_new_resolves_default_patterns() { factory.resolve_name_for_model("deepseek_v4"), Some(names::DEEPSEEK_V4) ); + assert_eq!( + factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4.1-Flash"), + Some(names::DEEPSEEK_V41) + ); assert_eq!( factory.resolve_name_for_model("deepseek-ai/DeepSeek-R1-0528"), Some(names::DEEPSEEK_V3) diff --git a/rust/src/chat/src/renderer/deepseek_v4/encoding.rs b/rust/src/chat/src/renderer/deepseek.rs similarity index 67% rename from rust/src/chat/src/renderer/deepseek_v4/encoding.rs rename to rust/src/chat/src/renderer/deepseek.rs index 263edba831..ab435028eb 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/encoding.rs +++ b/rust/src/chat/src/renderer/deepseek.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -//! DeepSeek V4 prompt renderer. +//! Shared DeepSeek V4 and V4.1 prompt rendering. //! //! Official Python reference: //! +use std::borrow::Cow; use std::collections::HashMap; use std::fmt::Write as _; @@ -13,8 +14,12 @@ use serde::Serialize; use serde_json::Value; use serde_json_fmt::JsonFormat; +use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER; + use crate::error::{Error, Result}; -use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, ReasoningEffort}; +use crate::request::{ + ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort, +}; use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall}; const BOS_TOKEN: &str = "<｜begin▁of▁sentence｜>"; @@ -23,6 +28,7 @@ const THINKING_START_TOKEN: &str = ""; const THINKING_END_TOKEN: &str = ""; const DSML_TOKEN: &str = "｜DSML｜"; const USER_SP_TOKEN: &str = "<｜User｜>"; +const SYSTEM_SP_TOKEN: &str = "<｜System｜>"; const ASSISTANT_SP_TOKEN: &str = "<｜Assistant｜>"; const REASONING_EFFORT_HIGH: &str = concat!( "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n", @@ -41,6 +47,35 @@ enum ThinkingMode { Thinking, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DsDialect { + V4, + V41, +} + +impl DsDialect { + fn tool_calls_tag(self) -> &'static str { + match self { + Self::V4 => "tool_calls", + Self::V41 => " calls", + } + } + + fn invoke_tag(self) -> &'static str { + match self { + Self::V4 => "invoke", + Self::V41 => " invoke", + } + } + + fn parameter_tag(self) -> &'static str { + match self { + Self::V4 => "parameter", + Self::V41 => " parameter", + } + } +} + #[serde_with::skip_serializing_none] #[derive(Debug, Serialize)] struct RenderedToolSchema<'a> { @@ -51,25 +86,47 @@ struct RenderedToolSchema<'a> { } /// Render one chat request into the final prompt string. -pub(super) fn render_request(request: &ChatRequest) -> Result { - let (thinking_mode, reasoning_effort_prompt) = resolve_thinking_options(request)?; +pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result { + let (thinking_mode, reasoning_effort_prompt) = match dialect { + DsDialect::V4 => { + resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))? + } + DsDialect::V41 => resolve_v41_thinking_options(request)?, + }; let request_tools = request_tools(request); let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools); let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true) && !rendered_tools_present(request, request_tools); let drop_historical_developers = thinking_mode == ThinkingMode::Thinking && drop_thinking; - let last_user_like_message_index = request.messages.iter().rposition(is_user_like_entry); - let last_user_render_index = - find_last_user_render_index(request.messages.as_slice(), synthetic_tool_system); + let last_user_like_message_index = request + .messages + .iter() + .enumerate() + .rfind(|(index, message)| is_user_like_entry(message, *index, dialect)) + .map(|(index, _)| index); + let last_user_render_index = find_last_user_render_index( + request.messages.as_slice(), + synthetic_tool_system, + drop_historical_developers, + last_user_like_message_index, + dialect, + ); let mut out = String::from(BOS_TOKEN); + if dialect == DsDialect::V41 + && (thinking_mode == ThinkingMode::Thinking + || synthetic_tool_system + || matches!(request.messages.first(), Some(ChatMessage::System { .. }))) + { + out.push_str(SYSTEM_SP_TOKEN); + } if thinking_mode == ThinkingMode::Thinking { - out.push_str(reasoning_effort_prompt); + out.push_str(&reasoning_effort_prompt); } let mut request_tools_attached = false; let mut render_index = 0isize; if synthetic_tool_system { - render_system_message(&mut out, None, request_tools)?; + render_system_message(&mut out, None, request_tools, dialect)?; request_tools_attached = true; render_index += 1; } @@ -80,27 +137,35 @@ pub(super) fn render_request(request: &ChatRequest) -> Result { continue; } - let current_render_index = render_index; - render_index += 1; - if drop_historical_developers && is_historical_developer(message, message_index, last_user_like_message_index) { continue; } + let current_render_index = render_index; + render_index += 1; + match message { ChatMessage::System { content } => { + if dialect == DsDialect::V41 && current_render_index > 0 { + out.push_str(SYSTEM_SP_TOKEN); + } let tools = if !request_tools_attached { request_tools_attached = true; request_tools } else { &[] }; - render_system_message(&mut out, Some(content), tools)?; + render_system_message(&mut out, Some(content), tools, dialect)?; } ChatMessage::Developer { content, tools } => { - render_developer_message(&mut out, content, tools.as_deref().unwrap_or(&[]))?; + render_developer_message( + &mut out, + content, + tools.as_deref().unwrap_or(&[]), + dialect, + )?; } ChatMessage::User { .. } | ChatMessage::ToolResponse { .. } => { render_user_content_block( @@ -108,6 +173,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result { request.messages.as_slice(), message_index, &last_tool_call_order, + dialect, )?; } ChatMessage::Assistant { content } => { @@ -119,7 +185,13 @@ pub(super) fn render_request(request: &ChatRequest) -> Result { && (!drop_thinking || current_render_index > last_user_render_index); let append_eos = !(message_index + 1 == request.messages.len() && request.chat_options.continue_final_message()); - render_assistant_message(&mut out, emit_thinking_block, append_eos, content)?; + render_assistant_message( + &mut out, + emit_thinking_block, + append_eos, + content, + dialect, + )?; if content.has_tool_calls() { last_tool_call_order.clear(); @@ -133,7 +205,8 @@ pub(super) fn render_request(request: &ChatRequest) -> Result { } } - if (is_user_like_entry(message) || matches!(message, ChatMessage::System { .. })) + if (is_user_like_entry(message, current_render_index as usize, dialect) + || (dialect == DsDialect::V4 && matches!(message, ChatMessage::System { .. }))) && next_rendered_entry_is_assistant_or_end( request.messages.as_slice(), message_index, @@ -181,6 +254,57 @@ fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'st Ok((thinking_mode, reasoning_effort_prompt)) } +/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs. +fn resolve_v41_thinking_options( + request: &ChatRequest, +) -> Result<(ThinkingMode, Cow<'static, str>)> { + let mut thinking = request.enable_thinking()?.unwrap_or(true); + let budget = match request.chat_options.reasoning_effort { + Some(ReasoningEffort::None) => { + thinking = false; + 50 + } + Some(ReasoningEffort::Low) => 25, + Some(ReasoningEffort::High) => 50, + Some(ReasoningEffort::XHigh) => 75, + Some(ReasoningEffort::Max) => 100, + Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => { + return Err(invalid_v41_reasoning_effort()); + } + None => match request.chat_options.template_kwargs.get("reasoning_effort") { + None | Some(Value::Null) => 50, + Some(Value::String(effort)) => match effort.as_str() { + "low" => 25, + "high" => 50, + "xhigh" => 75, + "max" => 100, + _ => return Err(invalid_v41_reasoning_effort()), + }, + Some(Value::Number(budget)) => budget + .as_u64() + .filter(|budget| (1..=100).contains(budget)) + .ok_or_else(invalid_v41_reasoning_effort)?, + Some(_) => return Err(invalid_v41_reasoning_effort()), + }, + }; + if thinking { + Ok(( + ThinkingMode::Thinking, + Cow::Owned(format!( + "Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n" + )), + )) + } else { + Ok((ThinkingMode::Chat, Cow::Borrowed(""))) + } +} + +fn invalid_v41_reasoning_effort() -> Error { + Error::InvalidReasoningEffort( + "DeepSeek V4.1 reasoning_effort must be low, high, xhigh, max, or an integer within [1, 100] in chat_template_kwargs".to_string(), + ) +} + /// Return request-level tools only when native tool parsing is enabled. fn request_tools(request: &ChatRequest) -> &[ChatTool] { if request.tool_parsing_enabled() { @@ -214,16 +338,25 @@ fn rendered_tools_present(request: &ChatRequest, request_tools: &[ChatTool]) -> } /// Find the last user-like turn after inline tool-response merging. -fn find_last_user_render_index(messages: &[ChatMessage], synthetic_tool_system: bool) -> isize { +fn find_last_user_render_index( + messages: &[ChatMessage], + synthetic_tool_system: bool, + drop_historical_developers: bool, + last_user_like_message_index: Option, + dialect: DsDialect, +) -> isize { let mut render_index = isize::from(synthetic_tool_system); let mut last_user_index = -1; for (message_index, message) in messages.iter().enumerate() { - if is_following_user_content(messages, message_index) { + if is_following_user_content(messages, message_index) + || (drop_historical_developers + && is_historical_developer(message, message_index, last_user_like_message_index)) + { continue; } - if is_user_like_entry(message) { + if is_user_like_entry(message, render_index as usize, dialect) { last_user_index = render_index; } render_index += 1; @@ -249,11 +382,13 @@ fn is_user_content_entry(message: &ChatMessage) -> bool { } /// Return whether one rendered entry should be treated as user-like. -fn is_user_like_entry(message: &ChatMessage) -> bool { +fn is_user_like_entry(message: &ChatMessage, message_index: usize, dialect: DsDialect) -> bool { matches!( message, ChatMessage::Developer { .. } | ChatMessage::User { .. } | ChatMessage::ToolResponse { .. } - ) + ) || (dialect == DsDialect::V41 + && message_index > 0 + && matches!(message, ChatMessage::System { .. })) } /// Return whether a developer entry precedes another user-like turn. @@ -293,22 +428,26 @@ fn next_rendered_entry_is_assistant_or_end( .unwrap_or(true) } -/// Render the tool preamble shown to the model, V4 flavor. -fn render_tools(out: &mut String, tools: &[ChatTool]) -> Result<()> { - out.push_str( +/// Render the tool preamble shown to the model for one DeepSeek dialect. +fn render_tools(out: &mut String, tools: &[ChatTool], dialect: DsDialect) -> Result<()> { + let tool_calls_tag = dialect.tool_calls_tag(); + let invoke_tag = dialect.invoke_tag(); + let parameter_tag = dialect.parameter_tag(); + write!( + out, r#"## Tools -You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following: +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜{tool_calls_tag}>" block like the following: -<｜DSML｜tool_calls> -<｜DSML｜invoke name="$TOOL_NAME"> -<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +<｜DSML｜{tool_calls_tag}> +<｜DSML｜{invoke_tag} name="$TOOL_NAME"> +<｜DSML｜{parameter_tag} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE ... - -<｜DSML｜invoke name="$TOOL_NAME2"> + +<｜DSML｜{invoke_tag} name="$TOOL_NAME2"> ... - - + + String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. @@ -319,7 +458,8 @@ Otherwise, output directly after with tool calls or final response. ### Available Tool Schemas "#, - ); + ) + .expect("writing to String cannot fail"); for (index, tool) in tools.iter().enumerate() { if index > 0 { @@ -350,13 +490,14 @@ fn render_system_message( out: &mut String, content: Option<&ChatContent>, tools: &[ChatTool], + dialect: DsDialect, ) -> Result<()> { if let Some(content) = content { - write_chat_content(out, content)?; + write_chat_content(out, content, dialect)?; } if !tools.is_empty() { out.push_str("\n\n"); - render_tools(out, tools)?; + render_tools(out, tools, dialect)?; } Ok(()) } @@ -366,18 +507,19 @@ fn render_developer_message( out: &mut String, content: &ChatContent, tools: &[ChatTool], + dialect: DsDialect, ) -> Result<()> { if content.is_empty() { return Err(Error::ChatTemplate( - "invalid DeepSeek V4 developer message: empty content".to_string(), + "invalid DeepSeek developer message: empty content".to_string(), )); } out.push_str(USER_SP_TOKEN); - write_chat_content(out, content)?; + write_chat_content(out, content, dialect)?; if !tools.is_empty() { out.push_str("\n\n"); - render_tools(out, tools)?; + render_tools(out, tools, dialect)?; } Ok(()) } @@ -388,6 +530,7 @@ fn render_user_content_block( messages: &[ChatMessage], message_index: usize, tool_call_order: &HashMap, + dialect: DsDialect, ) -> Result<()> { let (block_start, block_end) = user_content_block_bounds(messages, message_index); let mut sorted_tool_indices = @@ -399,7 +542,7 @@ fn render_user_content_block( out.push_str("\n\n"); } match &messages[message_index] { - ChatMessage::User { content } => write_chat_content(out, content)?, + ChatMessage::User { content } => write_chat_content(out, content, dialect)?, ChatMessage::ToolResponse { .. } => { let sorted_index = sorted_tool_indices .next() @@ -407,7 +550,7 @@ fn render_user_content_block( let ChatMessage::ToolResponse { content, .. } = &messages[sorted_index] else { unreachable!("sorted tool response index should reference a tool message"); }; - write_tool_result(out, content)?; + write_tool_result(out, content, dialect)?; } _ => unreachable!("user content block should only contain user content messages"), } @@ -454,9 +597,9 @@ fn sorted_tool_response_indices( } /// Render one tool response payload inside a V4 `` block. -fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> { +fn write_tool_result(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> { out.push_str(""); - write_chat_content(out, content)?; + write_chat_content(out, content, dialect)?; out.push_str(""); Ok(()) } @@ -483,6 +626,7 @@ fn render_assistant_message( emit_thinking_block: bool, append_eos: bool, content: &[AssistantContentBlock], + dialect: DsDialect, ) -> Result<()> { let has_tool_calls = content.has_tool_calls(); @@ -496,14 +640,15 @@ fn render_assistant_message( write_assistant_text(out, content); if has_tool_calls { - out.push_str("\n\n<｜DSML｜tool_calls>\n"); + let tool_calls_tag = dialect.tool_calls_tag(); + writeln!(out, "\n\n<{DSML_TOKEN}{tool_calls_tag}>").expect("writing to String cannot fail"); for (index, tool_call) in content.tool_calls().enumerate() { if index > 0 { out.push('\n'); } - render_tool_call(out, tool_call)?; + render_tool_call(out, tool_call, dialect)?; } - out.push_str("\n"); + write!(out, "\n").expect("writing to String cannot fail"); } if append_eos { @@ -513,11 +658,20 @@ fn render_assistant_message( } /// Render one assistant tool call in DSML XML-like format. -fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> { - writeln!(out, "<{DSML_TOKEN}invoke name=\"{}\">", tool_call.name) - .expect("writing to String cannot fail"); - encode_arguments_to_dsml(out, tool_call)?; - write!(out, "\n").expect("writing to String cannot fail"); +fn render_tool_call( + out: &mut String, + tool_call: &AssistantToolCall, + dialect: DsDialect, +) -> Result<()> { + let invoke_tag = dialect.invoke_tag(); + writeln!( + out, + "<{DSML_TOKEN}{invoke_tag} name=\"{}\">", + tool_call.name + ) + .expect("writing to String cannot fail"); + encode_arguments_to_dsml(out, tool_call, dialect)?; + write!(out, "\n").expect("writing to String cannot fail"); Ok(()) } @@ -525,15 +679,20 @@ fn render_tool_call(out: &mut String, tool_call: &AssistantToolCall) -> Result<( /// /// String values are emitted raw with `string="true"`, while all other JSON /// values are rendered with JSON syntax and `string="false"`. -fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> Result<()> { +fn encode_arguments_to_dsml( + out: &mut String, + tool_call: &AssistantToolCall, + dialect: DsDialect, +) -> Result<()> { + let parameter_tag = dialect.parameter_tag(); let arguments: Value = serde_json::from_str(&tool_call.arguments).map_err(|error| { Error::ChatTemplate(format!( - "assistant tool call has invalid JSON arguments for DeepSeek V4: {error}" + "assistant tool call has invalid JSON arguments for DeepSeek: {error}" )) })?; let Some(arguments) = arguments.as_object() else { return Err(Error::ChatTemplate( - "assistant tool call arguments for DeepSeek V4 must be a JSON object".to_string(), + "assistant tool call arguments for DeepSeek must be a JSON object".to_string(), )); }; @@ -546,7 +705,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> let is_string = matches!(value, Value::String(_)); write!( out, - "<{DSML_TOKEN}parameter name=\"{key}\" string=\"{}\">", + "<{DSML_TOKEN}{parameter_tag} name=\"{key}\" string=\"{}\">", if is_string { "true" } else { "false" } ) .expect("writing to String cannot fail"); @@ -556,7 +715,7 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> value => out.push_str(&json_dumps(value)?), } - write!(out, "").expect("writing to String cannot fail"); + write!(out, "").expect("writing to String cannot fail"); wrote_parameter = true; } @@ -565,12 +724,24 @@ fn encode_arguments_to_dsml(out: &mut String, tool_call: &AssistantToolCall) -> /// Write chat content directly into the destination buffer without flattening /// it into an intermediate `String`. -fn write_chat_content(out: &mut String, content: &ChatContent) -> Result<()> { +/// +/// The V4.1 dialect inlines the image placeholder at each image part's +/// position unconditionally (matching the Python encoding's +/// `IMAGE_PLACEHOLDER`); other dialects reject multimodal parts. +fn write_chat_content(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> { match content { ChatContent::Text(text) => out.push_str(text), ChatContent::Parts(parts) => { - for part in parts { - out.push_str(part.as_text()?); + for (index, part) in parts.iter().enumerate() { + if index > 0 && dialect == DsDialect::V41 { + out.push_str("\n\n"); + } + match part { + ChatContentPart::ImageUrl { .. } if dialect == DsDialect::V41 => { + out.push_str(DEEPSEEK_V41_IMAGE_PLACEHOLDER); + } + _ => out.push_str(part.as_text()?), + } } } } @@ -606,7 +777,7 @@ fn json_dumps(value: &T) -> Result { .format_to_string(value) .map_err(|error| { Error::ChatTemplate(format!( - "failed to serialize DeepSeek V4 JSON payload: {error}" + "failed to serialize DeepSeek JSON payload: {error}" )) }) } diff --git a/rust/src/chat/src/renderer/deepseek_v4/mod.rs b/rust/src/chat/src/renderer/deepseek_v4/mod.rs index 2762f1546f..155a6ee140 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/mod.rs +++ b/rust/src/chat/src/renderer/deepseek_v4/mod.rs @@ -1,10 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -mod encoding; - use vllm_text::Prompt; +use super::deepseek::{self, DsDialect}; use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; use crate::Result; use crate::request::ChatRequest; @@ -24,7 +23,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer { request.validate()?; Ok(RenderedPrompt { - prompt: Prompt::Text(encoding::render_request(request)?), + prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?), effective_template_kwargs: request_template_kwargs(request), }) } diff --git a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt new file mode 100644 index 0000000000..0897086688 --- /dev/null +++ b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_1.txt @@ -0,0 +1,38 @@ +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜ calls>" block like the following: + +<｜DSML｜ calls> +<｜DSML｜ invoke name="$TOOL_NAME"> +<｜DSML｜ parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<｜DSML｜ invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}} +{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<｜User｜>What's the weather in Beijing?<｜Assistant｜>The user wants to know the weather in Beijing. I should use the get_weather tool. + +<｜DSML｜ calls> +<｜DSML｜ invoke name="get_weather"> +<｜DSML｜ parameter name="location" string="true">Beijing +<｜DSML｜ parameter name="unit" string="true">celsius + +<｜end▁of▁sentence｜><｜User｜>{"temperature": 22, "condition": "sunny", "humidity": 45}<｜Assistant｜>Got the weather data. Let me format a nice response.The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt new file mode 100644 index 0000000000..4e93ce2cbb --- /dev/null +++ b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_2.txt @@ -0,0 +1,3 @@ +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant.<｜User｜>Hello<｜Assistant｜>Hi there! How can I help you?<｜end▁of▁sentence｜><｜User｜>What is the capital of France?<｜Assistant｜>The user asks about the capital of France. It is Paris.The capital of France is Paris.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt new file mode 100644 index 0000000000..99f4af0d0e --- /dev/null +++ b/rust/src/chat/src/renderer/deepseek_v41/fixtures/test_output_developer_tools.txt @@ -0,0 +1,30 @@ +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) + +<｜User｜>question<｜User｜>policy + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜ calls>" block like the following: + +<｜DSML｜ calls> +<｜DSML｜ invoke name="$TOOL_NAME"> +<｜DSML｜ parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<｜DSML｜ invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "lookup", "parameters": {"type": "object"}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<｜Assistant｜>answer<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/rust/src/chat/src/renderer/deepseek_v41/mod.rs b/rust/src/chat/src/renderer/deepseek_v41/mod.rs new file mode 100644 index 0000000000..4e5ddb3725 --- /dev/null +++ b/rust/src/chat/src/renderer/deepseek_v41/mod.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use vllm_text::Prompt; + +use super::deepseek::{self, DsDialect}; +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use crate::Result; +use crate::request::ChatRequest; + +/// Dedicated DeepSeek V4.1 renderer. +#[derive(Debug, Clone, Copy, Default)] +pub struct DeepSeekV41ChatRenderer; + +impl DeepSeekV41ChatRenderer { + pub fn new() -> Self { + Self + } +} + +impl ChatRenderer for DeepSeekV41ChatRenderer { + fn render(&self, request: &ChatRequest) -> Result { + request.validate()?; + + Ok(RenderedPrompt { + prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V41)?), + effective_template_kwargs: request_template_kwargs(request), + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/src/chat/src/renderer/deepseek_v41/tests.rs b/rust/src/chat/src/renderer/deepseek_v41/tests.rs new file mode 100644 index 0000000000..d7b0c538bb --- /dev/null +++ b/rust/src/chat/src/renderer/deepseek_v41/tests.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::path::PathBuf; + +use expect_test::{expect, expect_file}; +use serde_json::{Value, json}; + +use super::DeepSeekV41ChatRenderer; +use crate::ChatRenderer; +use crate::event::AssistantContentBlock; +use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; +use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ReasoningEffort}; + +fn render(request: &ChatRequest) -> String { + DeepSeekV41ChatRenderer::new() + .render(request) + .unwrap() + .prompt + .into_text() + .unwrap() +} + +fn request() -> ChatRequest { + ChatRequest { + messages: vec![ChatMessage::user("question")], + ..ChatRequest::for_test() + } +} + +#[test] +fn renders_shared_deepseek_fixtures_with_v41_reference_encoding() { + let fixture_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/renderer/deepseek_v4/fixtures"); + for (input, expected) in [ + ( + "test_input_1.json", + expect_file!["fixtures/test_output_1.txt"], + ), + ( + "test_input_2.json", + expect_file!["fixtures/test_output_2.txt"], + ), + ( + "test_input_developer_tools.json", + expect_file!["fixtures/test_output_developer_tools.txt"], + ), + ] { + let request = fixture_chat_request( + &fixture_dir.join(input), + FixtureRequestOptions { + enable_thinking: Some(true), + no_generation_prompt_when_last_assistant: true, + }, + ); + expected.assert_eq(&render(&request)); + } +} + +#[test] +fn inlines_image_placeholder_at_image_part_positions() { + // The placeholder is unconditional (matching the Python encoding's + // `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later, + // at media preparation. + let renderer = DeepSeekV41ChatRenderer::new(); + let request = ChatRequest { + messages: vec![ChatMessage::User { + content: ChatContent::Parts(vec![ + ChatContentPart::image_url("https://example.com/carrots.jpeg"), + ChatContentPart::text("what is in the image?"), + ]), + }], + ..ChatRequest::for_test() + }; + + let rendered = renderer.render(&request).unwrap().prompt.into_text().unwrap(); + + // Image parts inline the placeholder at their position; parts stay + // separated by "\n\n" (the V4.1 reference encoding's part separator). + assert!( + rendered.contains("<｜User｜><｜deepseek_image｜>\n\nwhat is in the image?"), + "unexpected prompt: {rendered:?}" + ); +} + +#[test] +fn maps_reasoning_effort_to_reference_numeric_budget() { + for (effort, budget) in [ + (None, 50), + (Some(ReasoningEffort::Low), 25), + (Some(ReasoningEffort::High), 50), + (Some(ReasoningEffort::XHigh), 75), + (Some(ReasoningEffort::Max), 100), + ] { + let mut request = request(); + request.chat_options.reasoning_effort = effort; + assert_eq!( + render(&request), + format!( + "<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n<｜User｜>question<｜Assistant｜>" + ) + ); + } +} + +#[test] +fn accepts_numeric_template_effort_with_top_level_precedence() { + for (effort, budget) in [ + (json!(1), 1), + (json!(42), 42), + (json!(100), 100), + (json!("low"), 25), + ] { + let mut request = request(); + request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort); + assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100"))); + + request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh); + assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100")); + } +} + +#[test] +fn rejects_undefined_effort_names_and_invalid_numeric_budgets() { + for effort in [ + json!("medium"), + json!("minimal"), + json!(-1), + json!(0), + json!(101), + json!(true), + json!(1.5), + ] { + let mut request = request(); + request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort); + let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err(); + assert!(error.is_request_validation_error()); + } + for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] { + let mut request = request(); + request.chat_options.reasoning_effort = Some(effort); + let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err(); + assert!(error.is_request_validation_error()); + } +} + +#[test] +fn chat_mode_and_none_effort_use_closed_thinking_prefix() { + let mut request = request(); + request.chat_options.template_kwargs.insert("thinking".into(), json!(false)); + let expected = expect!["<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜>"]; + expected.assert_eq(&render(&request)); + + request.chat_options.template_kwargs.insert("thinking".into(), json!(true)); + request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + expected.assert_eq(&render(&request)); +} + +#[test] +fn leading_system_in_chat_mode_uses_system_token() { + let mut request = request(); + request.messages.insert(0, ChatMessage::system("policy")); + request.chat_options.template_kwargs.insert("thinking".into(), json!(false)); + expect!["<｜begin▁of▁sentence｜><｜System｜>policy<｜User｜>question<｜Assistant｜>"] + .assert_eq(&render(&request)); +} + +#[test] +fn system_after_dropped_developer_becomes_leading_system() { + let mut request = ChatRequest { + messages: vec![ + ChatMessage::developer("old policy", None), + ChatMessage::system("sys"), + ], + ..ChatRequest::for_test() + }; + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys"] + .assert_eq(&render(&request)); + + request.messages.push(ChatMessage::user("question")); + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<｜User｜>question<｜Assistant｜>"] + .assert_eq(&render(&request)); +} + +#[test] +fn mid_system_advances_last_user_boundary_and_drops_old_reasoning() { + let mut request = ChatRequest { + messages: vec![ + ChatMessage::system("sys"), + ChatMessage::user("q1"), + ChatMessage::assistant_blocks(vec![ + AssistantContentBlock::Reasoning { text: "r1".into() }, + AssistantContentBlock::Text { text: "a1".into() }, + ]), + ChatMessage::developer("old policy", None), + ChatMessage::system("mid sys"), + ], + ..ChatRequest::for_test() + }; + request + .chat_options + .template_kwargs + .insert("reasoning_effort".into(), json!(88)); + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 88 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<｜User｜>q1<｜Assistant｜>a1<｜end▁of▁sentence｜><｜System｜>mid sys<｜Assistant｜>"] + .assert_eq(&render(&request)); +} + +#[test] +fn joins_text_parts_with_reference_separator_and_preserves_literal_tags() { + let mut request = ChatRequest { + messages: vec![ChatMessage::user(ChatContent::Parts(vec![ + ChatContentPart::text("first"), + ChatContentPart::text("<｜DSML｜invoke name=\"literal\">"), + ]))], + ..ChatRequest::for_test() + }; + request + .chat_options + .template_kwargs + .insert("thinking".into(), Value::Bool(false)); + expect!["<｜begin▁of▁sentence｜><｜User｜>first\n\n<｜DSML｜invoke name=\"literal\"><｜Assistant｜>"] + .assert_eq(&render(&request)); +} diff --git a/rust/src/chat/src/renderer/mod.rs b/rust/src/chat/src/renderer/mod.rs index a83953f331..a4649a34f7 100644 --- a/rust/src/chat/src/renderer/mod.rs +++ b/rust/src/chat/src/renderer/mod.rs @@ -10,8 +10,10 @@ use vllm_text::Prompt; use crate::error::Result; use crate::request::{ChatRequest, ReasoningEffort}; +mod deepseek; pub mod deepseek_v32; pub mod deepseek_v4; +pub mod deepseek_v41; pub mod harmony; pub mod hf; mod inkling; @@ -22,6 +24,7 @@ mod test_utils; pub use deepseek_v4::DeepSeekV4ChatRenderer; pub use deepseek_v32::DeepSeekV32ChatRenderer; +pub use deepseek_v41::DeepSeekV41ChatRenderer; pub use harmony::HarmonyChatRenderer; pub use inkling::InklingChatRenderer; pub use kimi_k3::KimiK3ChatRenderer; diff --git a/rust/src/chat/src/renderer/selection.rs b/rust/src/chat/src/renderer/selection.rs index 5b6642c4c0..0ece893762 100644 --- a/rust/src/chat/src/renderer/selection.rs +++ b/rust/src/chat/src/renderer/selection.rs @@ -22,6 +22,8 @@ pub enum RendererSelection { DeepSeekV32, /// Force the DeepSeek V4 renderer. DeepSeekV4, + /// Force the DeepSeek V4.1 renderer. + DeepSeekV41, /// Force the GPT-OSS Harmony renderer. Harmony, /// Force the Inkling native token renderer. @@ -34,6 +36,7 @@ impl RendererSelection { pub const AUTO_LITERAL: &str = "auto"; pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32"; pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4"; + pub const DEEPSEEK_V41_LITERAL: &str = "deepseek_v41"; pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss"; pub const HARMONY_LITERAL: &str = "harmony"; pub const HF_LITERAL: &str = "hf"; @@ -48,6 +51,7 @@ impl RendererSelection { Self::Auto => match model_type { Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32, Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4, + Self::DEEPSEEK_V41_LITERAL => Self::DeepSeekV41, Self::GPT_OSS_MODEL_TYPE => Self::Harmony, Self::INKLING_MODEL_TYPE => Self::Inkling, Self::KIMI_K3_LITERAL => Self::KimiK3, @@ -70,6 +74,8 @@ impl FromStr for RendererSelection { Ok(Self::DeepSeekV32) } else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) { Ok(Self::DeepSeekV4) + } else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V41_LITERAL) { + Ok(Self::DeepSeekV41) } else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) { Ok(Self::Harmony) } else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) { @@ -92,6 +98,7 @@ impl fmt::Display for RendererSelection { Self::Hf => f.write_str(Self::HF_LITERAL), Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL), Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL), + Self::DeepSeekV41 => f.write_str(Self::DEEPSEEK_V41_LITERAL), Self::Harmony => f.write_str(Self::HARMONY_LITERAL), Self::Inkling => f.write_str(Self::INKLING_LITERAL), Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL), @@ -121,7 +128,7 @@ mod tests { fn renderer_selection_expected_error_message() { let err = RendererSelection::from_str("unknown").unwrap_err(); expect_test::expect![ - "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)" + "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3)" ] .assert_eq(&err); } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 9f5577da64..5669e255e0 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -29,7 +29,7 @@ const TOKEN_COMPLETION_CHUNK_TOKENS: usize = 1; /// One model/parser configuration used to run the fixed roundtrip fixtures. #[derive(Clone)] struct RoundtripCase { - /// Hugging Face model id resolved through the production backend loader. + /// Hugging Face model id or local checkpoint resolved through the production backend loader. model_id: &'static str, /// Final assistant-history suffix rendered by the chat template but not /// generated by the model body consumed by the output processor. @@ -180,6 +180,19 @@ impl RoundtripCase { } } + /// DeepSeek V4.1 Flash DSML tool-call format. + fn deepseek_v41() -> Self { + Self { + model_id: "deepseek-ai/DeepSeek-V4.1-Flash", + assistant_stop_suffix: "<｜end▁of▁sentence｜>", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + thinking_behavior: ThinkingBehavior::Toggleable { default: true }, + json_fmt: compact_json_fmt(), + sort_json_keys: false, + } + } + /// DeepSeek V3.2 DSML tool-call format. fn deepseek_v32() -> Self { Self { @@ -385,6 +398,7 @@ roundtrip_tests! { minimax_m25 => [reasoning_and_content, tool_call_mix], minimax_m3 => [reasoning_and_content, tool_call_mix], deepseek_v4 => [reasoning_and_content, tool_call_mix], + deepseek_v41 => [reasoning_and_content, tool_call_mix], deepseek_v32 => [tool_call_mix], glm45 => [reasoning_and_content, tool_call_mix], glm47 => [reasoning_and_content, tool_call_mix], diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 2474a1e142..e9411891e0 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -903,7 +903,7 @@ fn serve_args_reject_unknown_renderer_value() { .unwrap_err(); expect![[r#" - error: invalid value 'definitely_missing' for '--tokenizer-mode ': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3) + error: invalid value 'definitely_missing' for '--tokenizer-mode ': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, deepseek_v41, harmony, inkling, kimi_k3) For more information, try '--help'. "#]] diff --git a/rust/src/parser/src/reasoning/mod.rs b/rust/src/parser/src/reasoning/mod.rs index 9a430f1996..670695f7b0 100644 --- a/rust/src/parser/src/reasoning/mod.rs +++ b/rust/src/parser/src/reasoning/mod.rs @@ -53,6 +53,8 @@ pub type Step3ReasoningParser = DeepSeekR1ReasoningParser; /// DeepSeek V4 currently shares the DeepSeek V3 reasoning parser. pub type DeepSeekV4ReasoningParser = DeepSeekV3ReasoningParser; +/// DeepSeek V4.1 uses bare `...` delimiters. +pub type DeepSeekV41ReasoningParser = DeepSeekV3ReasoningParser; /// GLM-4.7 and GLM-5 use bare reasoning delimiters. pub type Glm47ReasoningParser = DeepSeekV3ReasoningParser; /// Kimi K2 currently shares the DeepSeek V3 reasoning parser. diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs new file mode 100644 index 0000000000..44658b1d44 --- /dev/null +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use super::{DeepSeekDsmlToolParser, DsmlTokens}; +use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput}; + +mod structural_tag; + +/// Tool parser for DeepSeek V4.1's spaced DSML tags. +/// +/// Arguments are emitted only after a full `invoke` block is parsed. +pub struct DeepSeekV41ToolParser(DeepSeekDsmlToolParser); + +impl ToolParser for DeepSeekV41ToolParser { + fn create(tools: &[Tool]) -> Result> + where + Self: Sized + 'static, + { + Ok(Box::new(Self(DeepSeekDsmlToolParser::new( + tools, + DsmlTokens::V41, + )))) + } + + fn preserve_special_tokens(&self) -> bool { + true + } + + fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { + Some(&structural_tag::DeepSeekV41StructuralTagBuilder) + } + + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { + self.0.parse_into(chunk, output) + } + + fn finish(&mut self) -> Result { + self.0.finish() + } + + fn reset(&mut self) -> String { + self.0.reset() + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs new file mode 100644 index 0000000000..d5fb6118fb --- /dev/null +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Visible-text grammar for DeepSeek V4.1 tool calls. +//! +//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with +//! V4.1's spaced DSML tags and schema-unconstrained parameter values. + +use serde_json::Value; +use xgrammar_structural_tag::Result; +use xgrammar_structural_tag::builders::{ + StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, +}; +use xgrammar_structural_tag::format::{ + Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat, +}; +use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam}; + +use super::DsmlTokens; + +const TOKENS: DsmlTokens = DsmlTokens::V41; + +pub(super) struct DeepSeekV41StructuralTagBuilder; + +impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder { + fn build(&self, ctx: StructuralTagContext<'_>) -> Result { + let tags = ctx + .function_tools + .iter() + .map(|tool| call_tag(tool, ctx.options)) + .collect::>(); + let excludes = if ctx.options.exclude_special_tokens { + &["", ""][..] + } else { + &[] + }; + let format = match ctx.tool_choice { + BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes), + BuilderToolChoice::Auto => Format::TriggeredTags( + TriggeredTagsFormat::new( + &[TOKENS.tool_calls_start], + vec![TagFormat::new( + format!("{}\n", TOKENS.tool_calls_start), + Format::tags_with_separator(tags, "", true, false), + TOKENS.tool_calls_end, + )], + ) + .with_excludes(excludes), + ), + BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![ + Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)), + Format::tags_with_separator( + tags, + "", + true, + ctx.tool_choice == BuilderToolChoice::Forced, + ), + Format::const_string(TOKENS.tool_calls_end), + ]), + }; + // Reasoning boundaries remain owned by the engine reasoner; serving + // calls this visible-text builder with reasoning=false. + Ok(StructuralTag::new(format)) + } +} + +fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat { + // TODO: enforce tool.parameters and strict schema semantics once V4.1 + // argument conversion is available. This builder intentionally ignores + // parameters and strict: it constrains tool names and DSML/value syntax + // only. Parameter names, presence, uniqueness and value schemas remain + // unconstrained, including for strict=true. Serving still uses strict to + // decide whether auto tool choice activates a grammar. + TagFormat::new( + format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name), + Format::star(parameter(options)), + format!("{}\n", TOKENS.invoke_end), + ) +} + +fn parameter(options: StructuralTagOptions) -> Format { + Format::tag( + format!("{} name=\"", TOKENS.parameter_start), + Format::sequence(vec![ + Format::regex(r#"[^"]+"#), + Format::const_string("\" string=\""), + Format::or(vec![ + Format::sequence(vec![ + Format::const_string("true\">"), + Format::any_text_excluding(&[ + TOKENS.parameter_end, + TOKENS.invoke_end, + TOKENS.tool_calls_end, + ]), + ]), + Format::sequence(vec![ + Format::const_string("false\">"), + Format::JsonSchema( + JsonSchemaFormat::new(Value::Bool(true)) + .with_any_order(options.any_order) + .with_max_whitespace_cnt(options.max_whitespace_cnt), + ), + ]), + ]), + ]), + format!("{}\n", TOKENS.parameter_end), + ) +} diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs new file mode 100644 index 0000000000..d40f82d533 --- /dev/null +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use expect_test::expect; +use serde_json::json; +use xgrammar_structural_tag::builders::StructuralTagOptions; +use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, +}; + +use super::DeepSeekV41ToolParser; +use crate::tool::test_utils::collect_stream; +use crate::tool::{Tool, ToolParser}; + +fn tools() -> Vec { + vec![Tool { + name: "lookup".into(), + description: None, + parameters: json!({ + "type": "object", + "properties": { + "query": {"type": "string"}, + "limit": {"type": "integer"}, + "enabled": {"type": "boolean"}, + "data": {"type": "object"}, + "items": {"type": "array"}, + "label": {"type": "string"} + } + }), + strict: None, + }] +} + +#[test] +fn spaced_dsml_streaming_preserves_shared_schema_coercion() { + let wire = concat!( + "summary\n\n<｜DSML｜ calls>\n", + "<｜DSML｜ invoke name=\"lookup\">\n", + "<｜DSML｜ parameter name=\"query\" string=\"true\"> value\n\n", + "<｜DSML｜ parameter name=\"limit\" string=\"false\">2\n", + "<｜DSML｜ parameter name=\"enabled\" string=\"false\">true\n", + "<｜DSML｜ parameter name=\"data\" string=\"false\">{\"x\":1}\n", + "<｜DSML｜ parameter name=\"items\" string=\"false\">[1,2]\n", + "<｜DSML｜ parameter name=\"label\" string=\"false\">42\n", + "<｜DSML｜ parameter name=\"unknown\" string=\"false\">42\n", + "<｜DSML｜ parameter name=\"nullish\" string=\"false\">null\n", + "\n", + "<｜DSML｜ invoke name=\"lookup\">\n", + "<｜DSML｜ parameter name=\"query\" string=\"true\">second\n", + "\n", + ); + let expected = [ + json!({"query":" value\n","limit":2,"enabled":true,"data":{"x":1},"items":[1,2],"label":"42","unknown":"42","nullish":null}), + json!({"query":"second"}), + ]; + for split in wire.char_indices().map(|(i, _)| i).chain(std::iter::once(wire.len())) { + let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); + let output = collect_stream(parser.as_mut(), &[&wire[..split], &wire[split..]]); + // Protocol framing follows the current shared DSML behavior. + assert_eq!(output.normal_text(), "summary", "split {split}"); + assert_eq!(output.calls().len(), 2, "split {split}"); + for (index, call) in output.calls().iter().enumerate() { + assert_eq!(call.tool_index, index, "split {split}"); + assert_eq!(call.name.as_deref(), Some("lookup"), "split {split}"); + assert_eq!( + serde_json::from_str::(&call.arguments).unwrap(), + expected[index], + "split {split}" + ); + } + } +} + +#[test] +fn v4_tags_remain_text_in_v41_dialect() { + let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); + let output = collect_stream(parser.as_mut(), &["before<｜DSML｜tool_calls>after"]); + expect!["before<｜DSML｜tool_calls>after"].assert_eq(&output.normal_text()); + assert!(output.calls().is_empty()); +} + +#[test] +fn structural_tag_ignores_parameter_schemas_and_strict() { + let parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); + let builder = parser.structural_tag_builder().unwrap(); + let options = StructuralTagOptions::default().with_reasoning(false); + for choice in [ + ToolChoice::auto(), + ToolChoice::required(), + ToolChoice::function("lookup"), + ] { + let build = |function| { + build_structural_tag( + builder, + &[ToolParam::Function(FunctionToolParam::new(function))], + choice.clone(), + options, + ) + .unwrap() + }; + let expected = build(FunctionDefinition::new("lookup")); + for strict in [None, Some(false), Some(true)] { + for parameters in [ + json!(false), + json!({"type":"object", "properties":{"query":{"const":"fixed"}}, "required":["query"]}), + ] { + let mut function = FunctionDefinition::new("lookup").with_parameters(parameters); + function.strict = strict; + assert_eq!(build(function), expected); + } + } + } +} diff --git a/rust/src/parser/src/tool/deepseek_dsml/mod.rs b/rust/src/parser/src/tool/deepseek_dsml/mod.rs index 6ef5e78d4e..01a3fbe282 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/mod.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/mod.rs @@ -14,14 +14,11 @@ use crate::tool::Tool; mod deepseek_v32; mod deepseek_v4; +mod deepseek_v41; pub use deepseek_v4::DeepSeekV4ToolParser; pub use deepseek_v32::DeepSeekV32ToolParser; - -const INVOKE_START: &str = "<｜DSML｜invoke"; -const INVOKE_END: &str = ""; -const PARAMETER_START: &str = "<｜DSML｜parameter"; -const PARAMETER_END: &str = ""; +pub use deepseek_v41::DeepSeekV41ToolParser; type DsmlInput<'i> = Partial<&'i str>; @@ -30,6 +27,10 @@ struct DsmlTokens { tool_calls_start: &'static str, framed_tool_calls_start: &'static str, tool_calls_end: &'static str, + invoke_start: &'static str, + invoke_end: &'static str, + parameter_start: &'static str, + parameter_end: &'static str, } impl DsmlTokens { @@ -37,11 +38,25 @@ impl DsmlTokens { tool_calls_start: "<｜DSML｜function_calls>", framed_tool_calls_start: "\n\n<｜DSML｜function_calls>", tool_calls_end: "", + invoke_start: "<｜DSML｜invoke", + invoke_end: "", + parameter_start: "<｜DSML｜parameter", + parameter_end: "", }; const V4: Self = Self { tool_calls_start: "<｜DSML｜tool_calls>", framed_tool_calls_start: "\n\n<｜DSML｜tool_calls>", tool_calls_end: "", + ..Self::V32 + }; + const V41: Self = Self { + tool_calls_start: "<｜DSML｜ calls>", + framed_tool_calls_start: "\n\n<｜DSML｜ calls>", + tool_calls_end: "", + invoke_start: "<｜DSML｜ invoke", + invoke_end: "", + parameter_start: "<｜DSML｜ parameter", + parameter_end: "", }; } @@ -205,7 +220,7 @@ fn parse_tool_block_event( ) -> ModalResult { ws0.void().parse_next(input)?; alt(( - |input: &mut DsmlInput<'_>| invoke_event(input, invoke_end_scan), + |input: &mut DsmlInput<'_>| invoke_event(input, tokens, invoke_end_scan), |input: &mut DsmlInput<'_>| tool_calls_end_event(input, tokens), )) .parse_next(input) @@ -243,19 +258,20 @@ fn safe_text_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult /// Parse a DSML invoke block. fn invoke_event( input: &mut DsmlInput<'_>, + tokens: DsmlTokens, invoke_end_scan: &mut MarkerScanState, ) -> ModalResult { let (name, body) = seq!( - _: literal(INVOKE_START), + _: literal(tokens.invoke_start), _: ws1, dsml_name_attr, _: ws0, _: ">", - take_until_marker(INVOKE_END, invoke_end_scan), - _: literal(INVOKE_END), + take_until_marker(tokens.invoke_end, invoke_end_scan), + _: literal(tokens.invoke_end), ) .parse_next(input)?; - let raw_params = parse_invoke_params(body)?; + let raw_params = parse_invoke_params(body, tokens)?; Ok(DsmlEvent::Invoke { name: name.to_string(), raw_params, @@ -263,23 +279,31 @@ fn invoke_event( } /// Parse a DSML invoke body. -fn parse_invoke_params(invoke_body: &str) -> ModalResult> { +fn parse_invoke_params(invoke_body: &str, tokens: DsmlTokens) -> ModalResult> { let mut input = invoke_body; - delimited(ws0, repeat(0.., terminated(parse_parameter, ws0)), eof).parse_next(&mut input) + delimited( + ws0, + repeat( + 0.., + terminated(|input: &mut &str| parse_parameter(input, tokens), ws0), + ), + eof, + ) + .parse_next(&mut input) } /// Parse a DSML parameter block. -fn parse_parameter(input: &mut &str) -> ModalResult { +fn parse_parameter(input: &mut &str, tokens: DsmlTokens) -> ModalResult { seq! {DsmlParameter { - _: literal(PARAMETER_START), + _: literal(tokens.parameter_start), _: ws1, name: name_attr.map(|name: &str| name.to_string()), _: ws1, is_string: string_attr.map(|value| value == "true"), _: ws0, _: ">", - value: take_until(0.., PARAMETER_END).map(str::to_string), - _: literal(PARAMETER_END), + value: take_until(0.., tokens.parameter_end).map(str::to_string), + _: literal(tokens.parameter_end), }} .parse_next(input) } diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index 83510aeaf0..57bbb59528 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -20,7 +20,7 @@ mod seed_oss; pub mod test_utils; use std::collections::{BTreeMap, btree_map}; -pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser}; +pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser, DeepSeekV41ToolParser}; pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser}; pub use error::{Result, ToolParserError}; pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser}; diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index 17b1024943..b658388cbd 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -167,6 +167,19 @@ mod tests { assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); } + #[test] + fn invalid_reasoning_effort_maps_to_invalid_request() { + let error = vllm_chat::Error::InvalidReasoningEffort( + "DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(), + ); + let api_error = chat_submit_error("failed to submit chat request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + assert_eq!( + api_error.to_error_response().error.error_type, + "invalid_request_error" + ); + } + #[test] fn llm_wrapped_empty_prompt_maps_to_invalid_request() { let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { diff --git a/tests/parser/engine/test_deepseek_v41.py b/tests/parser/engine/test_deepseek_v41.py new file mode 100644 index 0000000000..cc7d6e0aed --- /dev/null +++ b/tests/parser/engine/test_deepseek_v41.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json + +import pytest + +from tests.parser.engine.replay_harness import ( + DUMMY_TOOLS, + MockTokenizer, + _test_request, + collect_output, + replay_streaming, +) +from vllm.parser.deepseek_v41 import deepseek_v41_config +from vllm.parser.parser_manager import ParserManager + +CALLS = ( + '\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n' + '<｜DSML｜ parameter name="city" string="true">杭州\n' + '<｜DSML｜ parameter name="count" string="false">42\n' + '\n<｜DSML｜ invoke name="add">\n' + '<｜DSML｜ parameter name="x" string="false">1.5\n' + '<｜DSML｜ parameter name="y" string="false">2.25\n' + "\n" +) + + +def tokenizer_for(text, special_calls): + vocab = {"": 50, "": 51} + if special_calls: + vocab |= {"<｜DSML｜ calls>": 52, "": 53} + tokens: list[tuple[int, str]] = [] + while text: + special = next((marker for marker in vocab if text.startswith(marker)), None) + piece = special or text[0] + tokens.append((vocab[piece] if special else 100 + len(tokens), piece)) + text = text[len(piece) :] + return MockTokenizer(vocab, tokens), tokens + + +def parser_for(tokenizer, controls): + cls = ParserManager.get_parser( + tool_parser_name="deepseek_v41", + reasoning_parser_name="deepseek_v41", + enable_auto_tools=True, + ) + return cls(tokenizer, chat_template_kwargs=controls) + + +@pytest.mark.parametrize("thinking", [False, True]) +@pytest.mark.parametrize("special_calls", [False, True]) +@pytest.mark.parametrize("chunk_size", [1, 7, 10000]) +def test_registered_adapters_parse_parallel_calls_across_chunks( + thinking, + special_calls, + chunk_size, +): + text = ("Plan." if thinking else "") + "Checking." + CALLS + tokenizer, tokens = tokenizer_for(text, special_calls) + parser = parser_for(tokenizer, {"thinking": thinking}) + output = collect_output( + replay_streaming( + parser, + tokens, + chunk_size=chunk_size, + finished_on_last=True, + tools=DUMMY_TOOLS, + prompt_token_ids=[50 if thinking else 51], + ) + ) + assert output.reasoning == ("Plan." if thinking else "") + assert output.content.strip() == "Checking." + assert [call["name"] for call in output.tool_calls] == ["get_weather", "add"] + assert [json.loads(call["arguments"]) for call in output.tool_calls] == [ + {"city": "杭州", "count": 42}, + {"x": 1.5, "y": 2.25}, + ] + + +@pytest.mark.parametrize("thinking", [False, True]) +def test_registered_adapters_parse_complete_output(thinking): + text = ("Plan." if thinking else "") + CALLS + tokenizer, tokens = tokenizer_for(text, False) + parser = parser_for(tokenizer, {"thinking": thinking}) + reasoning, content, calls = parser.parse( + text, + _test_request(tools=DUMMY_TOOLS), + enable_auto_tools=True, + model_output_token_ids=[tid for tid, _ in tokens], + ) + assert (reasoning or "") == ("Plan." if thinking else "") + assert not (content or "").strip() + assert [call.name for call in calls] == ["get_weather", "add"] + assert json.loads(calls[0].arguments) == {"city": "杭州", "count": 42} + + +@pytest.mark.parametrize( + ("controls", "text", "expected_reasoning", "reasoning_tokens"), + [ + ({"thinking": False}, "12", "", 0), + ({"enable_thinking": False}, "12", "", 0), + ({"thinking": True, "reasoning_effort": "none"}, "12", "", 0), + ({}, "12", "", 0), + ({}, "Plan.12", "Plan.", 5), + ], +) +def test_reasoning_adapter_controls_and_usage( + controls, + text, + expected_reasoning, + reasoning_tokens, +): + tokenizer, tokens = tokenizer_for(text, False) + parser = parser_for(tokenizer, controls) + output = collect_output( + replay_streaming( + parser, + tokens, + chunk_size=1, + finished_on_last=True, + ) + ) + assert output.reasoning == expected_reasoning + assert output.content == "12" + assert ( + parser.reasoning_parser.count_reasoning_tokens([tid for tid, _ in tokens]) + == reasoning_tokens + ) + assert parser.is_reasoning_end([50, 100, 51]) + assert not parser.is_reasoning_end([51, 100, 50]) + + +def test_python_argument_conversion_and_partial_values(): + converter = deepseek_v41_config().arg_converter + raw = ( + '<｜DSML｜ parameter name="object" string="false">' + '{"a": [true, null]}' + '<｜DSML｜ parameter name="bad" string="false">[broken' + '<｜DSML｜ parameter name="text" string="true"> a<｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜ calls>" block like the following: + +<｜DSML｜ calls> +<｜DSML｜ invoke name="$TOOL_NAME"> +<｜DSML｜ parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<｜DSML｜ invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}} +{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<｜User｜>What's the weather in Beijing?<｜Assistant｜>The user wants to know the weather in Beijing. I should use the get_weather tool. + +<｜DSML｜ calls> +<｜DSML｜ invoke name="get_weather"> +<｜DSML｜ parameter name="location" string="true">Beijing +<｜DSML｜ parameter name="unit" string="true">celsius + +<｜end▁of▁sentence｜><｜User｜>{"temperature": 22, "condition": "sunny", "humidity": 45}<｜Assistant｜>Got the weather data. Let me format a nice response.The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt b/tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt new file mode 100644 index 0000000000..4e93ce2cbb --- /dev/null +++ b/tests/tokenizers_/fixtures/deepseek_v41/test_output_2.txt @@ -0,0 +1,3 @@ +<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant.<｜User｜>Hello<｜Assistant｜>Hi there! How can I help you?<｜end▁of▁sentence｜><｜User｜>What is the capital of France?<｜Assistant｜>The user asks about the capital of France. It is Paris.The capital of France is Paris.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/tests/tokenizers_/test_deepseek_v41.py b/tests/tokenizers_/test_deepseek_v41.py new file mode 100644 index 0000000000..493fdd88ba --- /dev/null +++ b/tests/tokenizers_/test_deepseek_v41.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import copy +import json +from pathlib import Path + +import pytest + +from tests.tokenizers_.test_deepseek_v4 import FakeHfTokenizer +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.tokenizers.deepseek_v41 import get_deepseek_v41_tokenizer + +FIXTURES = Path(__file__).parent / "fixtures" + + +def render(messages, **kwargs): + return get_deepseek_v41_tokenizer(FakeHfTokenizer()).apply_chat_template( + messages, tokenize=False, **kwargs + ) + + +@pytest.mark.parametrize("case_id", [1, 2]) +def test_reference_encoder_fixtures(case_id): + # Expected prompts come from the DeepSeek V4.1 reference encoder. + data = json.loads( + (FIXTURES / "deepseek_v4" / f"test_input_{case_id}.json").read_text() + ) + messages = data["messages"] if isinstance(data, dict) else data + tools = data.get("tools") if isinstance(data, dict) else None + expected = (FIXTURES / "deepseek_v41" / f"test_output_{case_id}.txt").read_text() + assert render(messages, tools=tools) == expected + + +@pytest.mark.parametrize( + ("effort", "budget"), + [ + (None, 50), + ("low", 25), + ("high", 50), + ("xhigh", 75), + ("max", 100), + (1, 1), + (42, 42), + (100, 100), + ], +) +def test_numeric_reasoning_effort(effort, budget): + assert render( + [{"role": "user", "content": "question"}], reasoning_effort=effort + ) == ( + "<｜begin▁of▁sentence｜><｜System｜>" + f"Reasoning Effort: {budget} (range 1-100, the higher the value, " + "the more thorough the reasoning)\n\n" + "<｜User｜>question<｜Assistant｜>" + ) + + +@pytest.mark.parametrize("effort", ["medium", "minimal", -1, 0, 101, True, 1.5, []]) +def test_invalid_effort_is_a_request_error(effort): + with pytest.raises(ValueError, match="reasoning_effort"): + render([{"role": "user", "content": "question"}], reasoning_effort=effort) + + +@pytest.mark.parametrize( + "controls", + [ + {"thinking": False}, + {"enable_thinking": False}, + {"thinking": True, "reasoning_effort": "none"}, + {"enable_thinking": True, "reasoning_effort": "none"}, + ], +) +def test_chat_mode_has_closed_thinking_prefix(controls): + assert render([{"role": "user", "content": "question"}], **controls) == ( + "<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜>" + ) + + +def test_raw_text_parts_preserve_reference_separator(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second"}, + ], + } + ] + original = copy.deepcopy(messages) + assert ( + render( + messages, + conversation=[{"role": "user", "content": "first\nsecond"}], + thinking=False, + ) + == "<｜begin▁of▁sentence｜><｜User｜>first\n\nsecond<｜Assistant｜>" + ) + assert messages == original + + +def test_mid_system_gets_its_own_marker_and_generation_header(): + assert render( + [ + {"role": "user", "content": "question"}, + {"role": "system", "content": "answer briefly"}, + ], + thinking=False, + ) == ( + "<｜begin▁of▁sentence｜><｜User｜>question" + "<｜System｜>answer briefly<｜Assistant｜>" + ) + + +def test_top_level_effort_overrides_template_effort(): + request = ChatCompletionRequest( + model="deepseek-ai/DeepSeek-V4.1-Flash", + messages=[{"role": "user", "content": "question"}], + reasoning_effort="low", + chat_template_kwargs={"reasoning_effort": 100}, + ) + kwargs = request.build_chat_params(None, "auto").get_apply_chat_template_kwargs() + assert "Reasoning Effort: 25 " in render(request.messages, **kwargs) + + +def test_encode_uses_one_bos_and_forwards_truncation(): + tokenizer = get_deepseek_v41_tokenizer(FakeHfTokenizer()) + tokenizer.apply_chat_template( + [{"role": "user", "content": "question"}], + thinking=False, + truncation=True, + max_length=16, + ) + text, add_special_tokens, kwargs = tokenizer.last_encode + assert text.startswith("<｜begin▁of▁sentence｜>") + assert add_special_tokens is False + assert kwargs == {"truncation": True, "max_length": 16} + + +@pytest.mark.parametrize("image_type", ["image_url", "input_image", "image_pil"]) +def test_images_preserve_content_order_and_reference_separator(image_type): + from PIL import Image + + def image_part(color): + if image_type == "image_pil": + return {"type": image_type, "image_pil": Image.new("RGB", (2, 2), color)} + url = f"https://example.com/{color}.png" + return { + "type": image_type, + "image_url": {"url": url} if image_type == "image_url" else url, + } + + parts = [ + {"type": "text", "text": "first"}, + image_part("red"), + {"type": "text", "text": "second"}, + image_part("blue"), + {"type": "text", "text": "compare"}, + ] + assert render([{"role": "user", "content": parts}], thinking=False) == ( + "<｜begin▁of▁sentence｜><｜User｜>first\n\n<｜deepseek_image｜>\n\n" + "second\n\n<｜deepseek_image｜>\n\ncompare<｜Assistant｜>" + ) + assert parts[1]["type"] == parts[3]["type"] == image_type + + +def test_unsupported_media_is_a_request_error(): + with pytest.raises(ValueError, match="text and image content only"): + render([{"role": "user", "content": [{"type": "input_audio"}]}]) diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index c1e2e09437..b77b4efaff 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -80,6 +80,67 @@ def test_supported_structural_tag_models_include_vllm_builtins(): assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS +@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"]) +def test_deepseek_v41_format_constraints_ignore_parameter_schema( + choice, + sample_tools_strict, +): + tool_choice = ( + ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name=choice) + ) + if choice == "get_weather" + else choice + ) + tag = get_model_structural_tag( + "deepseek_v41", sample_tools_strict, tool_choice, reasoning=False + ) + grammar = Grammar.from_structural_tag(tag) + begin = '\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n' + end = "\n" + parameter = ( + '<｜DSML｜ parameter name="undeclared" string="false">' + '{"nested": [true, null, 1.5]}\n' + ) + # Required city is omitted; extra and repeated fields remain unconstrained. + assert _is_grammar_accept_string(grammar, begin + end) + assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end) + assert not _is_grammar_accept_string( + grammar, + begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end, + ) + assert not _is_grammar_accept_string( + grammar, (begin + end).replace('name="get_weather"', 'name="unknown"') + ) + assert not _is_grammar_accept_string( + grammar, begin + parameter.replace("｜ parameter", "｜parameter") + end + ) + assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto") + + +def test_deepseek_v41_named_choice_emits_one_call(sample_tools): + tag = get_model_structural_tag( + "deepseek_v41", + sample_tools, + ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_weather") + ), + reasoning=False, + ) + grammar = Grammar.from_structural_tag(tag) + call = '<｜DSML｜ invoke name="get_weather">\n\n' + assert _is_grammar_accept_string( + grammar, "\n\n<｜DSML｜ calls>\n" + call + "" + ) + assert not _is_grammar_accept_string( + grammar, "\n\n<｜DSML｜ calls>\n" + call * 2 + "" + ) + assert ( + get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False) + is None + ) + + @pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) def test_get_model_structural_tag_supports_all_xgrammar_builtins( model: str, diff --git a/vllm/config/model.py b/vllm/config/model.py index ad8bb06a08..e47f3eee96 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -152,6 +152,7 @@ class ModelConfig: - "mistral" will always use the tokenizer from `mistral_common`. - "deepseek_v32" will always use the tokenizer from `deepseek_v32`. - "deepseek_v4" will always use the tokenizer from `deepseek_v4`. + - "deepseek_v41" will use the DeepSeek V4.1 prompt encoder. - "kimi_k3" will always use the "hf" tokenizer but render chat prompts with Kimi K3's Python XTML encoding instead of a Jinja template. - "cohere" uses the standard HF tokenizer but renders the chat template @@ -705,6 +706,8 @@ class ModelConfig: "DeepseekV4ForConditionalGeneration", ): self.tokenizer_mode = "deepseek_v4" + elif arch == "DeepseekV41ForCausalLM": + self.tokenizer_mode = "deepseek_v41" elif arch in ("InklingForCausalLM", "InklingForConditionalGeneration"): self.tokenizer_mode = "inkling" diff --git a/vllm/models/deepseek_v4_1/common/mm_preprocess.py b/vllm/models/deepseek_v4_1/common/mm_preprocess.py new file mode 100644 index 0000000000..8bd8d70cae --- /dev/null +++ b/vllm/models/deepseek_v4_1/common/mm_preprocess.py @@ -0,0 +1,463 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Multimodal preprocessing for the DeepSeek-V4.1 vision variant. + +The image transform and image-span construction are ported from the +official repository's ``image_processor.py`` so that token counts bit-match +the reference. Each ``<｜deepseek_image｜>`` placeholder in the prompt expands +to ``[IMAGE_START] + ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h + +[IMAGE_END]``; every span position carries ``image_token_id`` (129264) in +``input_ids`` and the roles ride along in a per-image ``types`` tensor +(the reference's out-of-band ``token_types``). IMAGE slots receive aligner +rows in reading order; the delimiters take the learned ``image_start`` / +``image_newline`` / ``image_end`` vectors. + +One vLLM-side deviation from the reference token stream: a leading +compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO`` +positions, so the span always starts at the same compressor phase) is +prepended when the block is spliced into the final prompt. Pad positions +borrow the reserved in-vocab token ``<|place_holder_mm_span_0436|>`` so +they stay distinguishable from real span positions; they embed as the +plain image token (v4.1 has no ``image_pad`` vector) and are routed and +engram-deadened like image tokens. +""" + +import math +from collections.abc import Mapping, Sequence +from typing import Any, cast + +import numpy as np +import torch +from PIL import Image, ImageOps +from transformers import BatchFeature +from typing_extensions import assert_never + +from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.inputs import MultiModalDataDict +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems +from vllm.multimodal.parse import ImageSize, MultiModalDataItems +from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, + BaseMultiModalProcessor, + BaseProcessingInfo, + PromptReplacement, + PromptUpdate, + PromptUpdateDetails, +) +from vllm.multimodal.processing.processor import ( + MultiModalPromptUpdates, + MultiModalPromptUpdatesApplyResult, + PlaceholderFeaturesInfo, + UpdateMode, + _plan_prompt_updates, +) +from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config + +IMAGE_START, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(4) +# Image spans are padded so they always start at the same compressor group +# phase; v4.1 uses ratio-2 compressors (v4.0 used 4). +COMPRESS_PAD_TO = 2 + +IMAGE_PLACEHOLDER = "<｜deepseek_image｜>" + +# The checkpoint tokenizer's id for IMAGE_PLACEHOLDER (the config's +# ``image_token_id``). Every image-span position carries this id, exactly +# like the reference; the MoE router detects image tokens as the range +# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5), which also covers +# IMAGE_PAD_ID below. +IMAGE_SENTINEL_BASE_ID = 129264 +# Leading compressor-alignment pads borrow this reserved in-vocab token +# (``<|place_holder_mm_span_0436|>``) so they can be told apart from real +# span positions when the per-position role is decided from token ids. +IMAGE_PAD_ID = 129265 +IMAGE_PAD_TOKEN_NAME = "<|place_holder_mm_span_0436|>" + + +def image_sentinel_mask(token_ids: torch.Tensor) -> torch.Tensor: + """Boolean mask for image-span positions (span tokens and align pads).""" + return (token_ids == IMAGE_SENTINEL_BASE_ID) | (token_ids == IMAGE_PAD_ID) + + +def validate_image_sentinel_ids(tokenizer) -> None: + """Check the image/pad token ids against the tokenizer.""" + image_id = tokenizer.convert_tokens_to_ids(IMAGE_PLACEHOLDER) + if image_id != IMAGE_SENTINEL_BASE_ID: + raise ValueError( + f"Image placeholder {IMAGE_PLACEHOLDER!r} has id {image_id}, " + f"expected {IMAGE_SENTINEL_BASE_ID} (the config's " + "image_token_id); the DeepSeek-V4.1 vision path keys image " + "routing and engram masking off this id." + ) + pad_id = tokenizer.convert_tokens_to_ids(IMAGE_PAD_TOKEN_NAME) + if pad_id != IMAGE_PAD_ID: + raise ValueError( + f"Image pad token {IMAGE_PAD_TOKEN_NAME!r} has id {pad_id}, " + f"expected {IMAGE_PAD_ID}; the DeepSeek-V4.1 vision path " + "borrows this reserved id for compressor-alignment pads." + ) + + +def llm_grid(best_height, best_width, patch_size, downsample_ratio): + """Token grid the aligner produces from a patch grid of this pixel size.""" + return ( + math.ceil((best_height // patch_size) / downsample_ratio), + math.ceil((best_width // patch_size) / downsample_ratio), + ) + + +def num_image_tokens(n_llm_h: int, n_llm_w: int) -> int: + return n_llm_h * (n_llm_w + 1) + 2 + + +def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token): + """Largest aspect-preserving pixel size whose token grid still fits in + max_n_token. Returns (best_height, best_width).""" + r = height / width + max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5 + max_h_float = max_w_float * r + cell = patch_size * downsample_ratio + if max_w_float < 1.0: # very tall: collapse to a single column + return (max_n_token - 2) // 2 * cell, cell + if max_h_float < 1.0: # very wide: collapse to a single row + return cell, (max_n_token - 3) * cell + beta = min( + math.floor(max_w_float) * cell / width, + math.floor(max_h_float) * cell / height, + ) + return ( + math.floor(height * beta / patch_size) * patch_size, + math.floor(width * beta / patch_size) * patch_size, + ) + + +def safe_resize( + height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token +): + """Shrink the pixel size until the image costs at most max_n_token LLM + tokens (minus the reservation for the compressor-alignment pad).""" + max_n_token -= COMPRESS_PAD_TO - 1 + n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio) + if num_image_tokens(n_llm_h, n_llm_w) > max_n_token: + best_height, best_width = solve_resize_ratio( + height, width, patch_size, downsample_ratio, max_n_token + ) + n_llm_h, n_llm_w = llm_grid( + best_height, best_width, patch_size, downsample_ratio + ) + assert num_image_tokens(n_llm_h, n_llm_w) <= max_n_token + return n_llm_h, n_llm_w, best_height, best_width + + +def load_image( + image: Image.Image, + *, + patch_size: int, + downsample_ratio: int, + max_n_token: int, + min_pixels: int, + max_wh_ratio: float | None, +): + """Transform one PIL image into ViT patches. + + Same math as the reference ``load_image``, except the image is already + decoded (vLLM supplies PIL images instead of a record dict). + """ + p = patch_size + image = image.convert("RGB") + width, height = image.size + if max_wh_ratio is not None and width > height * max_wh_ratio: + width = height * max_wh_ratio + if 0 < width * height < min_pixels: + ratio = (min_pixels / (width * height)) ** 0.5 + width = int(width * ratio) + height = int(height * ratio) + best_width = math.ceil(width / p) * p + best_height = math.ceil(height / p) * p + n_llm_h, n_llm_w, best_height, best_width = safe_resize( + height, width, best_height, best_width, p, downsample_ratio, max_n_token + ) + n_vit_h, n_vit_w = best_height // p, best_width // p + if max_wh_ratio is not None and image.width >= max_wh_ratio * image.height: + image = image.resize((best_width, best_height)) + else: + image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127)) + x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255 + x = ((x - 0.5) / 0.5).to(torch.bfloat16) + patches = ( + x.reshape(3, n_vit_h, p, n_vit_w, p) + .permute(1, 3, 0, 2, 4) + .reshape(n_vit_h * n_vit_w, 3, p, p) + ) + return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w + + +def image_token_types(n_llm_h: int, n_llm_w: int) -> torch.Tensor: + """Reading-order span layout: one IMAGE_NEW_LINE per row.""" + types = [IMAGE_START] + types += ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h + types.append(IMAGE_END) + return torch.tensor(types, dtype=torch.int64) + + +class DeepseekV4VLImageProcessor: + """Per-image transform (the PIL-input equivalent of the reference + ``load_image``).""" + + def __init__(self, config: DeepseekV41Config) -> None: + super().__init__() + self.patch_size = config.vision_patch_size + self.downsample_ratio = config.vision_downsample_ratio + self.max_n_token = config.vision_max_n_token + self.min_pixels = config.vision_min_pixels + self.max_wh_ratio = config.vision_max_wh_ratio + + def __call__(self, image: Image.Image): + return load_image( + image, + patch_size=self.patch_size, + downsample_ratio=self.downsample_ratio, + max_n_token=self.max_n_token, + min_pixels=self.min_pixels, + max_wh_ratio=self.max_wh_ratio, + ) + + +class DeepseekV4VLProcessor: + """Minimal stand-in for the HF processor of DeepSeek-V4.1 vision models. + + The official repository ships image preprocessing as plain functions in + ``image_processor.py`` (no ``auto_map`` processor), so this class wraps + their ports directly and the model loads without ``--trust-remote-code``. + + ``__call__`` returns a ``BatchFeature`` with one entry per image + (flattened across images): + + - ``patches``: ``(sum(n_vit_h * n_vit_w), 3, p, p)`` bf16 ViT patches. + - ``vit_grid``: ``(num_images, 2)`` int64 ``[n_vit_h, n_vit_w]``. + - ``llm_grid``: ``(num_images, 2)`` int64 ``[n_llm_h, n_llm_w]``. + - ``types``: concatenated per-image pad-free span roles + (IMAGE_START/IMAGE/IMAGE_NEW_LINE/IMAGE_END); every span position + carries ``image_token_id`` in the prompt's token ids. + """ + + def __init__(self, config: DeepseekV41Config) -> None: + super().__init__() + self.config = config + self.image_processor = DeepseekV4VLImageProcessor(config) + + def __call__( + self, + text: str | None = None, + images: Sequence[Image.Image] | None = None, + return_tensors: str | None = None, + **kwargs: Any, + ) -> BatchFeature: + patches_list = [] + vit_grid = [] + llm_grid_list = [] + types_list = [] + for image in images or []: + patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = self.image_processor(image) + patches_list.append(patches) + vit_grid.append((n_vit_h, n_vit_w)) + llm_grid_list.append((n_llm_h, n_llm_w)) + types_list.append(image_token_types(n_llm_h, n_llm_w)) + + if not patches_list: + return BatchFeature({}) + + return BatchFeature( + { + "patches": torch.cat(patches_list), + "vit_grid": torch.tensor(vit_grid, dtype=torch.int64), + "llm_grid": torch.tensor(llm_grid_list, dtype=torch.int64), + "types": torch.cat(types_list), + } + ) + + +class DeepseekV4VLProcessingInfo(BaseProcessingInfo): + def get_hf_config(self) -> DeepseekV41Config: + return self.ctx.get_hf_config(DeepseekV41Config) + + def get_hf_processor(self, **kwargs: object) -> DeepseekV4VLProcessor: + if kwargs: + raise ValueError(f"Unexpected processor kwargs: {sorted(kwargs)}") + return DeepseekV4VLProcessor(self.get_hf_config()) + + def get_supported_mm_limits(self) -> Mapping[str, int | None]: + return {"image": None} + + def get_mm_max_tokens_per_item( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> Mapping[str, int]: + # ``safe_resize`` reserves COMPRESS_PAD_TO - 1 tokens of + # vision_max_n_token for the compressor-alignment pad, so the full + # image span is bounded by vision_max_n_token; the margin is kept + # in case that reservation changes. + return { + "image": self.get_hf_config().vision_max_n_token + COMPRESS_PAD_TO - 1, + } + + def get_image_placeholder_token_id(self) -> int: + token_id = self.get_tokenizer().convert_tokens_to_ids(IMAGE_PLACEHOLDER) + if token_id is None: + raise ValueError(f"Token not found in tokenizer: {IMAGE_PLACEHOLDER}") + return token_id + + def get_image_size_with_most_features(self) -> ImageSize: + hf_config = self.get_hf_config() + patch_size = hf_config.vision_patch_size + downsample_ratio = hf_config.vision_downsample_ratio + # A square maximizes the ViT patch count (area) within the token + # budget; solve the budget-derived size directly to keep the dummy + # image small. + budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1) + side = budget * patch_size * downsample_ratio + best_h, best_w = solve_resize_ratio( + side, side, patch_size, downsample_ratio, budget + ) + return ImageSize(width=best_w, height=best_h) + + +class DeepseekV4VLDummyInputsBuilder( + BaseDummyInputsBuilder[DeepseekV4VLProcessingInfo] +): + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: + return IMAGE_PLACEHOLDER * mm_counts.get("image", 0) + + def get_dummy_mm_data( + self, + seq_len: int, + mm_counts: Mapping[str, int], + mm_options: Mapping[str, BaseDummyOptions], + ) -> MultiModalDataDict: + size = self.info.get_image_size_with_most_features() + return { + "image": self._get_dummy_images( + width=size.width, + height=size.height, + num_images=mm_counts.get("image", 0), + overrides=cast(ImageDummyOptions | None, mm_options.get("image")), + ), + } + + +class DeepseekV4VLMultiModalProcessor( + BaseMultiModalProcessor[DeepseekV4VLProcessingInfo] +): + def _get_mm_fields_config( + self, + hf_inputs: BatchFeature, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + vit_grid = hf_inputs.get("vit_grid") + llm_grid = hf_inputs.get("llm_grid") + + if vit_grid is None or llm_grid is None: + empty = torch.empty(0, dtype=torch.long) + patch_sizes = types_sizes = empty + else: + patch_sizes = vit_grid.prod(-1) + n_llm_h, n_llm_w = llm_grid[:, 0], llm_grid[:, 1] + types_sizes = n_llm_h * (n_llm_w + 1) + 2 + + return { + "patches": MultiModalFieldConfig.flat_from_sizes("image", patch_sizes), + "vit_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True), + "llm_grid": MultiModalFieldConfig.batched("image", keep_on_cpu=True), + "types": MultiModalFieldConfig.flat_from_sizes( + "image", types_sizes, keep_on_cpu=True + ), + } + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + out_mm_kwargs: MultiModalKwargsItems, + ) -> Sequence[PromptUpdate]: + image_token_id = self.info.get_image_placeholder_token_id() + validate_image_sentinel_ids(self.info.get_tokenizer()) + + def get_image_replacement(item_idx: int) -> PromptUpdateDetails: + types: torch.Tensor = out_mm_kwargs["image"][item_idx]["types"].data + # Every span position carries image_token_id; the roles live in + # ``types``. All of them are embed positions (delimiters get the + # learned vectors from embed_multimodal, not the embed table). + full = [image_token_id] * types.numel() + return PromptUpdateDetails.select_token_id(full, image_token_id) + + return [ + PromptReplacement( + modality="image", + target=[image_token_id], + replacement=get_image_replacement, + ), + ] + + def _apply_token_matches_with_placeholders( + self, + token_ids: list[int], + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[ + list[int], + MultiModalPromptUpdatesApplyResult, + Mapping[str, list[PlaceholderFeaturesInfo]], + ]: + # Same as the base implementation, except that each image block gets + # its compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start % + # COMPRESS_PAD_TO`` IMAGE_PAD_ID tokens) prepended while splicing: + # the pad depends on the block's final position in the prompt, which + # is unknown when the (cacheable) prompt updates are built. Pad + # positions are not embed positions (their id differs from + # image_token_id), so ``is_embed`` stays aligned with the pad-free + # replacement content. + matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates) + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = { + modality: [] for modality in mm_prompt_updates + } + + new_token_ids = list[int]() + prev_end_idx = 0 + for matched_update in matched_updates: + update = matched_update.update + match = matched_update.match + + if update.mode == UpdateMode.INSERT: + end_idx_to_insert = match.end_idx + elif update.mode == UpdateMode.REPLACE: + end_idx_to_insert = match.start_idx + else: + assert_never(update.mode) + + new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert]) + start_idx = len(new_token_ids) + + tokens = list(update.content.full) + if tokens and update.modality == "image": + compress_pad = COMPRESS_PAD_TO - 1 - start_idx % COMPRESS_PAD_TO + tokens = [IMAGE_PAD_ID] * compress_pad + tokens + + if tokens: + content_is_embed = update.content.is_embed + is_embed = ( + content_is_embed(tokens) if content_is_embed is not None else None + ) + placeholders[update.modality].append( + PlaceholderFeaturesInfo( + modality=update.modality, + item_idx=update.item_idx, + start_idx=start_idx, + tokens=tokens, + is_embed=is_embed, + ) + ) + new_token_ids.extend(tokens) + + prev_end_idx = match.end_idx + + new_token_ids.extend(token_ids[prev_end_idx:]) + return new_token_ids, result, placeholders diff --git a/vllm/parser/deepseek_v4.py b/vllm/parser/deepseek_v4.py index 2508064b34..13eb83e6b8 100644 --- a/vllm/parser/deepseek_v4.py +++ b/vllm/parser/deepseek_v4.py @@ -70,11 +70,17 @@ _PARTIAL_PARAM_RE = re.compile( ) -def _dsml_arg_converter(raw_args: str, partial: bool) -> str: +def _dsml_arg_converter( + raw_args: str, + partial: bool, + *, + param_re: re.Pattern = _PARAM_RE, + partial_param_re: re.Pattern = _PARTIAL_PARAM_RE, +) -> str: params: dict[str, object] = {} last_end = 0 - for m in _PARAM_RE.finditer(raw_args): + for m in param_re.finditer(raw_args): name, is_str, value = m.group(1), m.group(2), m.group(3) if is_str == "true": params[name] = value @@ -86,7 +92,7 @@ def _dsml_arg_converter(raw_args: str, partial: bool) -> str: last_end = m.end() if partial: - pm = _PARTIAL_PARAM_RE.search(raw_args, last_end) + pm = partial_param_re.search(raw_args, last_end) if pm: name, is_str, value = pm.group(1), pm.group(2), pm.group(3) if is_str == "true": @@ -229,6 +235,8 @@ def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig: class DeepSeekV4Parser(ParserEngine): + parser_config = staticmethod(deepseek_v4_config) + def __init__( self, tokenizer: TokenizerLike, @@ -245,13 +253,15 @@ class DeepSeekV4Parser(ParserEngine): super().__init__( tokenizer, tools, - parser_engine_config=deepseek_v4_config(thinking=thinking), + parser_engine_config=self.parser_config(thinking=thinking), **kwargs, ) self._arg_converter = self._convert_args def _convert_args(self, raw_args: str, partial: bool) -> str: - result = _dsml_arg_converter(raw_args, partial) + converter = self.parser_engine_config.arg_converter + assert converter is not None + result = converter(raw_args, partial) if not self._tools: return result func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None) diff --git a/vllm/parser/deepseek_v41.py b/vllm/parser/deepseek_v41.py new file mode 100644 index 0000000000..668bdd4a6b --- /dev/null +++ b/vllm/parser/deepseek_v41.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek V4.1 reasoning and spaced DSML tool calls.""" + +import functools +from dataclasses import replace + +import regex as re + +from vllm.parser.deepseek_v4 import ( + DeepSeekV4Parser, + _dsml_arg_converter, + deepseek_v4_config, +) +from vllm.parser.engine.parser_engine_config import ParserEngineConfig + +DSML_TOOL_START = "<｜DSML｜ calls>" +DSML_TOOL_END = "" +DSML_INVOKE_PREFIX = '<｜DSML｜ invoke name="' +DSML_INVOKE_END = "" +DSML_PARAM_START = "<｜DSML｜ parameter" +DSML_PARAM_CLOSE = "" + +_PARAM_RE = re.compile( + r'<｜DSML｜ parameter\s+name="([^"]+)"\s+string="(true|false)">' + r"(.*?)" + r"(?:|(?=<｜DSML｜ parameter\s+name=))", + re.DOTALL, +) +_PARTIAL_PARAM_RE = re.compile( + r'<｜DSML｜ parameter\s+name="([^"]+)"\s+string="(true|false)">' + r"(.*)$", + re.DOTALL, +) + + +@functools.cache +def deepseek_v41_config(thinking: bool = False) -> ParserEngineConfig: + config = deepseek_v4_config(thinking=thinking) + terminal_overrides = { + "TOOL_START": DSML_TOOL_START, + "TOOL_END": DSML_TOOL_END, + "INVOKE_PREFIX": DSML_INVOKE_PREFIX, + "INVOKE_END": DSML_INVOKE_END, + "PARAM_START": DSML_PARAM_START, + "PARAM_CLOSE": DSML_PARAM_CLOSE, + } + return replace( + config, + name="deepseek_v41", + terminals={**config.terminals, **terminal_overrides}, + token_id_terminals={ + key: terminal_overrides.get(key, value) + for key, value in config.token_id_terminals.items() + }, + arg_converter=functools.partial( + _dsml_arg_converter, + param_re=_PARAM_RE, + partial_param_re=_PARTIAL_PARAM_RE, + ), + ) + + +class DeepSeekV41Parser(DeepSeekV4Parser): + parser_config = staticmethod(deepseek_v41_config) diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 57865e96b5..8e10a1b5ef 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -9,6 +9,7 @@ names so that :class:`ReasoningParserManager` and from vllm.parser.deepseek_v4 import DeepSeekV4Parser from vllm.parser.deepseek_v32 import DeepSeekV32Parser +from vllm.parser.deepseek_v41 import DeepSeekV41Parser from vllm.parser.engine.adapters import make_adapters from vllm.parser.gemma4 import Gemma4Parser from vllm.parser.glm47_moe import Glm47MoeParser @@ -30,6 +31,11 @@ from vllm.parser.seed_oss import SeedOssParser DeepSeekV4ParserToolAdapter, ) = make_adapters(DeepSeekV4Parser) +( + DeepSeekV41ParserReasoningAdapter, + DeepSeekV41ParserToolAdapter, +) = make_adapters(DeepSeekV41Parser) + ( MinimaxM2ParserReasoningAdapter, MinimaxM2ParserToolAdapter, diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 70469f6695..615820bb29 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -32,6 +32,10 @@ _REASONING_PARSERS_TO_REGISTER = { "deepseek_v4_engine_reasoning_parser", "DeepSeekV4ParserReasoningAdapter", ), + "deepseek_v41": ( + "deepseek_v41_engine_reasoning_parser", + "DeepSeekV41ParserReasoningAdapter", + ), "poolside_v1": ( "poolside_v1_reasoning_parser", "PoolsideV1ReasoningParser", diff --git a/vllm/reasoning/deepseek_v41_engine_reasoning_parser.py b/vllm/reasoning/deepseek_v41_engine_reasoning_parser.py new file mode 100644 index 0000000000..bace0eb4f8 --- /dev/null +++ b/vllm/reasoning/deepseek_v41_engine_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import DeepSeekV41ParserReasoningAdapter + +__all__ = ["DeepSeekV41ParserReasoningAdapter"] diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index c7e1fc11ad..740c2023b2 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -23,6 +23,7 @@ _VLLM_RENDERERS = { "cohere": ("cohere", "CohereRenderer"), "deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"), "deepseek_v4": ("deepseek_v4", "DeepseekV4Renderer"), + "deepseek_v41": ("deepseek_v4", "DeepseekV4Renderer"), "hf": ("hf", "HfRenderer"), "kimi_audio": ("hf", "HfRenderer"), "kimi_k3": ("kimi_k3", "KimiK3Renderer"), diff --git a/vllm/tokenizers/deepseek_v41.py b/vllm/tokenizers/deepseek_v41.py new file mode 100644 index 0000000000..3a57717270 --- /dev/null +++ b/vllm/tokenizers/deepseek_v41.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import copy +from typing import Any + +from transformers import TokenizersBackend + +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam + +from .deepseek_v41_encoding import ( + IMAGE_PLACEHOLDER, + REASONING_EFFORT_MAPPINGS, + encode_messages, +) +from .hf import HfTokenizer, get_cached_tokenizer +from .protocol import TokenizerLike + + +def _normalize_messages( + messages: list[ChatCompletionMessageParam], +) -> list[dict[str, Any]]: + result = [dict(message) for message in copy.deepcopy(messages)] + for message in result: + role = message.get("role") + if role not in ("system", "developer", "user", "assistant", "tool"): + raise ValueError(f"Invalid role: {role}") + if "reasoning" in message: + message["reasoning_content"] = message["reasoning"] + content = message.get("content") + if isinstance(content, list): + parts = [] + for block in content: + part_type = block.get("type") + if part_type == "text": + parts.append(block.get("text", "")) + elif part_type in ("image_url", "input_image", "image_pil"): + parts.append(IMAGE_PLACEHOLDER) + else: + raise ValueError( + "DeepSeek V4.1 supports text and image content only; " + f"got {part_type!r}" + ) + message["content"] = "\n\n".join(parts) + return result + + +def get_deepseek_v41_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: + """Wrap an HF tokenizer with the V4.1 prompt encoder.""" + wrapped = copy.copy(tokenizer) + added_vocab = tokenizer.get_added_vocab() + + class _DeepseekV41Tokenizer(tokenizer.__class__): # type: ignore + def apply_chat_template( + self, + messages: list[ChatCompletionMessageParam], + tools: list[dict[str, Any]] | None = None, + **kwargs, + ) -> str | list[int]: + # The generic renderer's conversation has already flattened text + # parts with '\n'. V4.1 encodes the original messages with '\n\n'. + conversation = _normalize_messages(messages) + if tools: + system = next((m for m in conversation if m["role"] == "system"), None) + if system is None: + system = {"role": "system", "content": ""} + conversation.insert(0, system) + system["tools"] = tools + + thinking = bool(kwargs.get("thinking") or kwargs.get("enable_thinking")) + if "thinking" not in kwargs and "enable_thinking" not in kwargs: + thinking = True + effort = kwargs.get("reasoning_effort") + if effort == "none": + thinking = False + effort = None + if effort is None: + effort = "high" + if not ( + (type(effort) is int and 1 <= effort <= 100) + or (isinstance(effort, str) and effort in REASONING_EFFORT_MAPPINGS) + ): + raise ValueError( + "DeepSeek V4.1 reasoning_effort must be low, high, xhigh, max, " + "or an integer within [1, 100] in chat_template_kwargs" + ) + + prompt = encode_messages( + conversation, + thinking_mode="thinking" if thinking else "chat", + drop_thinking=kwargs.get("drop_thinking", True), + reasoning_effort=effort, + ) + if kwargs.get("tokenize", True): + tokenizer_kwargs = { + key: kwargs[key] + for key in ("truncation", "max_length") + if key in kwargs + } + return self.encode(prompt, add_special_tokens=False, **tokenizer_kwargs) + return prompt + + def num_special_tokens_to_add(self) -> int: + return len(self.encode("")) + + def get_added_vocab(self) -> dict[str, int]: + return added_vocab.copy() + + def __reduce__(self): + return get_deepseek_v41_tokenizer, (tokenizer,) + + _DeepseekV41Tokenizer.__name__ = f"DSV41{tokenizer.__class__.__name__}" + wrapped.__class__ = _DeepseekV41Tokenizer + return wrapped + + +class DeepseekV41Tokenizer(TokenizerLike): + @classmethod + def from_pretrained(cls, *args, **kwargs) -> HfTokenizer: + tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs) + return get_cached_tokenizer(get_deepseek_v41_tokenizer(tokenizer)) diff --git a/vllm/tokenizers/deepseek_v41_encoding.py b/vllm/tokenizers/deepseek_v41_encoding.py new file mode 100644 index 0000000000..841de385cb --- /dev/null +++ b/vllm/tokenizers/deepseek_v41_encoding.py @@ -0,0 +1,577 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# ruff: noqa + +"""Text encoding adapted from the DeepSeek V4.1 reference encoder. + +Message normalization and serving controls live in deepseek_v41.py. +""" + +import copy +import json +from typing import Any, Dict, List, Optional, Union + +bos_token: str = "<｜begin▁of▁sentence｜>" +eos_token: str = "<｜end▁of▁sentence｜>" +thinking_start_token: str = "" +thinking_end_token: str = "" +dsml_token: str = "｜DSML｜" + +USER_SP_TOKEN = "<｜User｜>" +ASSISTANT_SP_TOKEN = "<｜Assistant｜>" +LATEST_REMINDER_SP_TOKEN = "<｜latest_reminder｜>" +IMAGE_PLACEHOLDER = "<｜deepseek_image｜>" + +DS_TASK_SP_TOKENS = { + "action": "<｜action｜>", + "query": "<｜query｜>", + "authority": "<｜authority｜>", + "domain": "<｜domain｜>", + "title": "<｜title｜>", + "read_url": "<｜read_url｜>", +} +VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) + +system_msg_template: str = "{content}" +user_msg_template: str = "{content}" +latest_reminder_msg_template: str = "{content}" +assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token +assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}" +thinking_template: str = "{reasoning_content}" + +response_format_template: str = ( + "## Response Format:\n\n" + "You MUST strictly adhere to the following schema to reply:\n{schema}" +) +tool_output_template: str = "{content}" + + +def to_json(value: Any) -> str: + """Serialize a value to JSON string.""" + try: + return json.dumps(value, ensure_ascii=False) + except: + return json.dumps(value, ensure_ascii=True) + + +def tools_from_openai_format(tools): + """Extract function definitions from OpenAI-format tool list.""" + return [tool["function"] for tool in tools] + + +def tool_calls_from_openai_format(tool_calls): + """Convert OpenAI-format tool calls to internal format.""" + return [ + { + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + } + for tool_call in tool_calls + ] + + +def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Merge tool messages into the preceding user message using content_blocks format. + + DeepSeek-V4.1 does not have a standalone "tool" role; instead, tool results + are encoded as blocks within user messages. + """ + merged: List[Dict[str, Any]] = [] + + for msg in messages: + msg = copy.deepcopy(msg) + role = msg.get("role") + + if role == "tool": + # Convert tool message to a user message with tool_result block + tool_block = { + "type": "tool_result", + "tool_use_id": msg.get("tool_call_id", ""), + "content": msg.get("content", ""), + } + # Merge into previous message if it's already a user (merged tool) + if ( + merged + and merged[-1].get("role") == "user" + and "content_blocks" in merged[-1] + ): + merged[-1]["content_blocks"].append(tool_block) + else: + merged.append( + { + "role": "user", + "content_blocks": [tool_block], + } + ) + elif role == "user": + content_blocks = msg.get("content_blocks") + if content_blocks is None: + content_blocks = [{"type": "text", "text": msg.get("content", "")}] + if ( + merged + and merged[-1].get("role") == "user" + and "content_blocks" in merged[-1] + and merged[-1].get("task") is None + ): + merged[-1]["content_blocks"].extend(content_blocks) + else: + # Preserve structured content and all message-level metadata. + new_msg = msg + new_msg["content_blocks"] = content_blocks + merged.append(new_msg) + else: + merged.append(msg) + + return merged + + +def sort_tool_results_by_call_order( + messages: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """ + Sort tool_result blocks within user messages by the order of tool_calls + in the preceding assistant message. + """ + last_tool_call_order: Dict[str, int] = {} + + for msg in messages: + role = msg.get("role") + if role == "assistant" and msg.get("tool_calls"): + last_tool_call_order = {} + for idx, tc in enumerate(msg["tool_calls"]): + tc_id = tc.get("id") or tc.get("function", {}).get("id", "") + if tc_id: + last_tool_call_order[tc_id] = idx + + elif role == "user" and msg.get("content_blocks"): + tool_blocks = [ + b for b in msg["content_blocks"] if b.get("type") == "tool_result" + ] + if len(tool_blocks) > 1 and last_tool_call_order: + sorted_blocks = sorted( + tool_blocks, + key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0), + ) + sorted_idx = 0 + new_blocks = [] + for block in msg["content_blocks"]: + if block.get("type") == "tool_result": + new_blocks.append(sorted_blocks[sorted_idx]) + sorted_idx += 1 + else: + new_blocks.append(block) + msg["content_blocks"] = new_blocks + + return messages + + +SYSTEM_SP_TOKEN = "<｜System｜>" + +tool_calls_block_name: str = " calls" +tool_call_tag_name: str = " invoke" +tool_parameter_tag_name: str = " parameter" +tool_call_template: str = '<{dsml_token}{tool_call_tag_name} name="{name}">\n{arguments}\n' +tool_calls_template = ( + "<{dsml_token}{tc_block_name}>\n{tool_calls}\n" +) + +REASONING_EFFORT_TEMPLATE = ( + "Reasoning Effort: {budget} " + "(range 1-100, the higher the value, the more thorough the reasoning)\n\n" +) +REASONING_EFFORT_MAPPINGS: Dict[str, int] = { + "low": 25, + "high": 50, + "xhigh": 75, + "max": 100, +} + +DEFAULT_REASONING_EFFORT = "high" + + +def render_reasoning_effort( + index: int, + thinking_mode: str, + effort: Union[str, int, None], +) -> str: + """Render the V4.1 numeric reasoning effort prefix (thinking mode, index 0 only).""" + if effort is None: + effort = DEFAULT_REASONING_EFFORT + assert ( + type(effort) is int and 1 <= effort <= 100 + ) or effort in REASONING_EFFORT_MAPPINGS, ( + "Invalid reasoning effort for deepseek_v41: " + f"{effort}, should be int within [1,100] or {list(REASONING_EFFORT_MAPPINGS)}" + ) + if type(effort) is str: + effort = REASONING_EFFORT_MAPPINGS[effort] + if index == 0 and thinking_mode == "thinking": + return REASONING_EFFORT_TEMPLATE.format(budget=effort) + return "" + + +TOOLS_TEMPLATE = """## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}{tc_block_name}>" block like the following: + +<{dsml_token}{tc_block_name}> +<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME"> +<{dsml_token}{tool_parameter_tag_name} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. + +Otherwise, output directly after {thinking_end_token} with tool calls or final response. + +### Available Tool Schemas + +{tool_schemas} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +""" + + +def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: + """Render tool schemas into the V4.1 system prompt format.""" + tools_json = [to_json(t) for t in tools] + + return TOOLS_TEMPLATE.format( + tool_schemas="\n".join(tools_json), + dsml_token=dsml_token, + tc_block_name=tool_calls_block_name, + tool_call_tag_name=tool_call_tag_name, + tool_parameter_tag_name=tool_parameter_tag_name, + thinking_start_token=thinking_start_token, + thinking_end_token=thinking_end_token, + ) + + +def encode_arguments_to_dsml(tool_call: Dict[str, Any]) -> str: + """Encode tool call arguments into V4.1 DSML parameter format.""" + p_dsml_template = ( + '<{dsml_token}{tool_parameter_tag_name} name="{key}" string="{is_str}">' + "{value}" + ) + P_dsml_strs = [] + + arguments = tool_call["arguments"] + if not isinstance(arguments, dict): + # Tolerate JSON strings, including double-encoded ones. + for _ in range(2): + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except Exception: + break + else: + break + if not isinstance(arguments, dict): + arguments = {"arguments": tool_call["arguments"]} + + for k, v in arguments.items(): + P_dsml_strs.append( + p_dsml_template.format( + dsml_token=dsml_token, + tool_parameter_tag_name=tool_parameter_tag_name, + key=k, + is_str="true" if isinstance(v, str) else "false", + value=v if isinstance(v, str) else to_json(v), + ) + ) + + return "\n".join(P_dsml_strs) + + +def find_last_user_index(messages: List[Dict[str, Any]]) -> int: + """ + Find the index of the last user/developer message. + + V4.1 supports mid-conversation system messages, which count as user + messages for the purposes of the assistant generation header. + """ + last_user_index = -1 + for idx in range(len(messages) - 1, -1, -1): + role = messages[idx].get("role") + if role in ["user", "developer"] or (role == "system" and idx > 0): + last_user_index = idx + break + return last_user_index + + +def render_message( + index: int, + messages: List[Dict[str, Any]], + thinking_mode: str, + drop_thinking: bool = True, + reasoning_effort: Union[str, int, None] = None, +) -> str: + """ + Render a single message at the given index into its V4.1 encoded string form. + """ + assert 0 <= index < len(messages) + assert thinking_mode in ["chat", "thinking"], ( + f"Invalid thinking_mode `{thinking_mode}`" + ) + + msg = messages[index] + last_user_idx = find_last_user_index(messages) + + role = msg.get("role") + content = msg.get("content") + tools = msg.get("tools") + response_format = msg.get("response_format") + tool_calls = msg.get("tool_calls") + reasoning_content = msg.get("reasoning_content") + wo_eos = msg.get("wo_eos", False) + + if tools: + tools = tools_from_openai_format(tools) + if tool_calls: + tool_calls = tool_calls_from_openai_format(tool_calls) + + # Reasoning effort prefix (thinking mode, index 0 only) + reasoning_effort_prompt = render_reasoning_effort( + index, thinking_mode, reasoning_effort + ) + # System token leads the conversation when there is a reasoning effort prompt + # or the first message is a system message. + prompt = ( + SYSTEM_SP_TOKEN + if index == 0 and (reasoning_effort_prompt or role == "system") + else "" + ) + prompt += reasoning_effort_prompt + + if role == "system": + if index > 0: + # Mid-conversation system message + prompt += SYSTEM_SP_TOKEN + prompt += system_msg_template.format(content=content or "") + if tools: + prompt += "\n\n" + render_tools(tools) + if response_format: + prompt += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + elif role == "developer": + assert content, f"Invalid message for role `{role}`: {msg}" + + content_developer = USER_SP_TOKEN + content_developer += content + + if tools: + content_developer += "\n\n" + render_tools(tools) + if response_format: + content_developer += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + prompt += user_msg_template.format(content=content_developer) + + elif role == "user": + prompt += USER_SP_TOKEN + + # Handle content blocks (tool results mixed with text) + content_blocks = msg.get("content_blocks") + if content_blocks: + parts = [] + for block in content_blocks: + block_type = block.get("type") + if block_type == "text": + parts.append(block.get("text", "")) + elif block_type == "tool_result": + tool_content = block.get("content", "") + if isinstance(tool_content, list): + text_parts = [] + for b in tool_content: + if b.get("type") == "text": + text_parts.append(b.get("text", "")) + else: + text_parts.append(f"[Unsupported {b.get('type')}]") + tool_content = "\n\n".join(text_parts) + parts.append(tool_output_template.format(content=tool_content)) + else: + parts.append(f"[Unsupported {block_type}]") + prompt += "\n\n".join(parts) + else: + prompt += content or "" + + elif role == "latest_reminder": + prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format( + content=content + ) + + elif role == "tool": + raise NotImplementedError( + "deepseek_v41 merges tool messages into user; please preprocess with merge_tool_messages()" + ) + + elif role == "assistant": + thinking_part = "" + tc_content = "" + + if tool_calls: + tc_list = [ + tool_call_template.format( + dsml_token=dsml_token, + tool_call_tag_name=tool_call_tag_name, + name=tc.get("name"), + arguments=encode_arguments_to_dsml(tc), + ) + for tc in tool_calls + ] + tc_content += "\n\n" + tool_calls_template.format( + dsml_token=dsml_token, + tool_calls="\n".join(tc_list), + tc_block_name=tool_calls_block_name, + ) + + summary_content = content or "" + rc = reasoning_content or "" + + # Check if previous message has a task - if so, this is a task output (no thinking) + prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None + + if thinking_mode == "thinking" and not prev_has_task: + if not drop_thinking or index > last_user_idx: + thinking_part = ( + thinking_template.format(reasoning_content=rc) + thinking_end_token + ) + else: + thinking_part = "" + + if wo_eos: + prompt += assistant_msg_wo_eos_template.format( + reasoning=thinking_part, + content=summary_content, + tool_calls=tc_content, + ) + else: + prompt += assistant_msg_template.format( + reasoning=thinking_part, + content=summary_content, + tool_calls=tc_content, + ) + else: + raise NotImplementedError(f"Unknown role: {role}") + + # Append transition tokens based on what follows + if index + 1 < len(messages) and messages[index + 1].get("role") not in [ + "assistant", + "latest_reminder", + ]: + return prompt + + task = messages[index].get("task") + if task is not None: + # Task special token for internal classification tasks + assert task in VALID_TASKS, ( + f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}" + ) + task_sp_token = DS_TASK_SP_TOKENS[task] + + if task != "action": + # Non-action tasks: append task sp token directly after the message + prompt += task_sp_token + else: + # Action task: append Assistant + thinking token + action sp token + prompt += ASSISTANT_SP_TOKEN + prompt += ( + thinking_end_token + if thinking_mode != "thinking" + else thinking_start_token + ) + prompt += task_sp_token + + elif messages[index].get("role") in ["user", "developer"] or ( + messages[index].get("role") == "system" and index > 0 + ): + # Normal generation: append Assistant + thinking token + # (mid-conversation system messages also trigger the assistant header) + prompt += ASSISTANT_SP_TOKEN + if not drop_thinking and thinking_mode == "thinking": + prompt += thinking_start_token + elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx: + prompt += thinking_start_token + else: + prompt += thinking_end_token + + return prompt + + +def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Drop reasoning_content and non-essential messages before the last user message. + Same as V4, but uses the V4.1 last-user definition (mid systems count). + """ + last_user_idx = find_last_user_index(messages) + result = [] + keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"} + + for idx, msg in enumerate(messages): + role = msg.get("role") + if role in keep_roles or idx >= last_user_idx: + result.append(msg) + elif role == "assistant": + msg = copy.copy(msg) + msg.pop("reasoning_content", None) + result.append(msg) + # developer and other roles before last_user_idx are dropped + + return result + + +def encode_messages( + messages: List[Dict[str, Any]], + thinking_mode: str, + context: Optional[List[Dict[str, Any]]] = None, + drop_thinking: bool = True, + add_default_bos_token: bool = True, + reasoning_effort: Union[str, int, None] = None, +) -> str: + """Encode preprocessed (text-only) messages into the V4.1 prompt format.""" + context = context if context else [] + + # Preprocess: merge tool messages and sort tool results + messages = merge_tool_messages(messages) + messages = sort_tool_results_by_call_order(context + messages)[len(context) :] + if context: + context = merge_tool_messages(context) + context = sort_tool_results_by_call_order(context) + + full_messages = context + messages + + prompt = bos_token if add_default_bos_token and len(context) == 0 else "" + + # Resolve drop_thinking: if any message has tools defined, don't drop thinking + effective_drop_thinking = drop_thinking + if any(m.get("tools") for m in full_messages): + effective_drop_thinking = False + + if thinking_mode == "thinking" and effective_drop_thinking: + full_messages = _drop_thinking_messages(full_messages) + num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) + context_len = len(full_messages) - num_to_render + else: + num_to_render = len(messages) + context_len = len(context) + + for idx in range(num_to_render): + prompt += render_message( + idx + context_len, + full_messages, + thinking_mode=thinking_mode, + drop_thinking=effective_drop_thinking, + reasoning_effort=reasoning_effort, + ) + + return prompt diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 713c6228f9..b9c9226675 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -45,6 +45,7 @@ _VLLM_TOKENIZERS = { "cohere": ("hf", "CachedHfTokenizer"), "deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"), "deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"), + "deepseek_v41": ("deepseek_v41", "DeepseekV41Tokenizer"), "hf": ("hf", "CachedHfTokenizer"), "kimi_audio": ("kimi_audio", "KimiAudioTokenizer"), "kimi_k3": ("hf", "CachedHfTokenizer"), diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index a536891bae..c43a34772f 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -42,6 +42,10 @@ _TOOL_PARSERS_TO_REGISTER = { "deepseekv4_engine_tool_parser", "DeepSeekV4EngineToolParser", ), + "deepseek_v41": ( + "deepseekv41_engine_tool_parser", + "DeepSeekV41EngineToolParser", + ), "cohere_command3": ( "cohere_command_tool_parser", "CohereCommand3ToolParser", diff --git a/vllm/tool_parsers/deepseekv41_engine_tool_parser.py b/vllm/tool_parsers/deepseekv41_engine_tool_parser.py new file mode 100644 index 0000000000..a4c699aeab --- /dev/null +++ b/vllm/tool_parsers/deepseekv41_engine_tool_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter + + +class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = "deepseek_v41" diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 413a97715f..34e57840be 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -71,7 +71,9 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( "deepseek_v4", } ) -VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"}) +VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( + {"deepseek_v41", "hermes", "hy_v4", "kimi_k3"} +) SUPPORTED_STRUCTURAL_TAG_MODELS = ( XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS ) @@ -225,6 +227,99 @@ def get_function_parameters(function) -> dict[str, Any] | bool: return function.parameters if function.parameters is not None else True +_V41_CALLS_START = "<｜DSML｜ calls>" +_V41_CALLS_END = "" +_V41_INVOKE_END = "" +_V41_PARAMETER_END = "" + + +@register_vllm_structural_tag("deepseek_v41") +def get_deepseek_v41_structural_tag( + tools: list[FunctionToolParam], + builtin_tools: list[BuiltinToolParam], + tool_choice: SimplifiedToolChoice, + reasoning: bool, + token_suffix: str = "", +) -> StructuralTag: + # Serving enables this visible-text grammar after the reasoning boundary. + del builtin_tools, reasoning, token_suffix + + # TODO: lower parameter schemas into DSML constraints. This builder constrains + # tool names and DSML/value syntax only. Parameter names, presence, uniqueness + # and value schemas remain unconstrained, including for strict=true. The request + # layer still uses strict to decide whether auto tool choice activates a grammar. + parameter = TagFormat( + begin='<｜DSML｜ parameter name="', + content=SequenceFormat( + elements=[ + RegexFormat(pattern=r'[^"]+'), + ConstStringFormat(value='" string="'), + OrFormat( + elements=[ + SequenceFormat( + elements=[ + ConstStringFormat(value='true">'), + AnyTextFormat( + excludes=[ + _V41_PARAMETER_END, + _V41_INVOKE_END, + _V41_CALLS_END, + ] + ), + ] + ), + SequenceFormat( + elements=[ + ConstStringFormat(value='false">'), + JSONSchemaFormat(json_schema=True), + ] + ), + ] + ), + ] + ), + end=f"{_V41_PARAMETER_END}\n", + ) + calls = TagsWithSeparatorFormat( + tags=[ + TagFormat( + begin=f'<｜DSML｜ invoke name="{tool.function.name}">\n', + content=StarFormat(content=parameter), + end=f"{_V41_INVOKE_END}\n", + ) + for tool in tools + ], + separator="", + at_least_one=True, + stop_after_first=tool_choice == "forced", + ) + if tool_choice == "auto": + return StructuralTag( + format=TriggeredTagsFormat( + triggers=[_V41_CALLS_START], + tags=[ + TagFormat( + begin=f"{_V41_CALLS_START}\n", + content=calls, + end=_V41_CALLS_END, + ) + ], + excludes=["", ""], + ) + if tools + else AnyTextFormat(excludes=["", ""]) + ) + return StructuralTag( + format=SequenceFormat( + elements=[ + ConstStringFormat(value=f"\n\n{_V41_CALLS_START}\n"), + calls, + ConstStringFormat(value=_V41_CALLS_END), + ] + ) + ) + + def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]: arguments_field_prefix = '", "arguments": ' formats = [ diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index d82e0d000a..b64dc5e090 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -152,6 +152,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( laguna="LagunaConfig", lfm2_moe="Lfm2MoeConfig", **{"unlimited-ocr": "UnlimitedOCRConfig"}, + **{"deepseek_v41": "DeepseekV41Config"}, inkling_mm_model="InklingMMConfig", inkling_model="InklingModelConfig", ) diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 9ddbb003fc..fda48fd20d 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -33,6 +33,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma", "DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2", "DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4", + "DeepseekV41Config": "vllm.transformers_utils.configs.deepseek_v41", "Dots3NoteConfig": "vllm.transformers_utils.configs.dots3_note", "K3DSparkConfig": "vllm.transformers_utils.configs.k3_dspark", "DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr", diff --git a/vllm/transformers_utils/configs/deepseek_v41.py b/vllm/transformers_utils/configs/deepseek_v41.py new file mode 100644 index 0000000000..3a406d24bd --- /dev/null +++ b/vllm/transformers_utils/configs/deepseek_v41.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +from transformers import PretrainedConfig + + +class DeepseekV41Config(PretrainedConfig): + """DeepSeek V4.1 config. + + The HF config nests the text model under ``text_config`` and the vision + tower under ``vision_config``. The vLLM model code (ported from + ``deepseek_v4``) reads flat attributes, so both sub-configs are flattened + onto the top level here: text fields are exposed as-is, vision fields + with the ``vision_*`` naming used by ``deepseek_v4_1.common.vision``. + """ + + model_type = "deepseek_v41" + + def __init__( + self, + text_config: dict[str, Any] | None = None, + vision_config: dict[str, Any] | None = None, + **kwargs, + ): + text_config = dict(text_config or {}) + vision_config = dict(vision_config or {}) + + # ``rope_scaling`` is a property in Transformers v5 (backed by + # ``rope_parameters``); capture it here and restore after + # super().__init__, which re-standardizes rope params. + rope_scaling = text_config.pop("rope_scaling", None) + + for key, value in text_config.items(): + if key == "model_type": + continue + # Don't clobber PretrainedConfig properties (e.g. is_encoder_decoder). + if isinstance(getattr(type(self), key, None), property): + continue + setattr(self, key, value) + + super().__init__(**kwargs) + if rope_scaling is not None: + self.rope_parameters = rope_scaling + + # The v4.1 quantization_config carries the expert dtype; the model + # code reads it from the top-level hf_config. + quant_cfg = getattr(self, "quantization_config", None) or {} + if not hasattr(self, "expert_dtype") and "expert_dtype" in quant_cfg: + self.expert_dtype = quant_cfg["expert_dtype"] + + vision_n_layers = vision_config.get("num_hidden_layers", 0) + self.vision_n_layers = vision_n_layers + self.vision_dim = vision_config.get("hidden_size", 1024) + self.vision_n_heads = vision_config.get("num_attention_heads", 16) + self.vision_inter_dim = vision_config.get("intermediate_size", 2816) + self.vision_patch_size = vision_config.get("patch_size", 14) + self.vision_rope_theta = vision_config.get("rope_theta", 10000.0) + self.vision_downsample_ratio = vision_config.get("downsample_ratio", 3) + self.vision_max_n_token = vision_config.get("max_image_tokens", 1024) + self.vision_min_pixels = vision_config.get("min_pixels", 295936) + self.vision_max_wh_ratio = vision_config.get("max_wh_ratio") + # The vision variant needs the mm-prefix plumbing (atomic image-span + # prefill + sparse-SWA window widening); the base arch-config + # convertor reads this config-level attribute. + self.is_mm_prefix_lm = vision_n_layers > 0 + # The sparse-SWA index kernels widen the window within image spans + # in-kernel, so mm-prefix ranges longer than sliding_window must be + # kept (they are only consumed by those kernels). + self.mm_prefix_clamp_sliding_window = vision_n_layers > 0 + # The visibility span covers the image block [IMAGE_START, + # IMAGE_END]; the mm placeholder additionally carries a leading + # compressor-alignment pad of ``COMPRESS_PAD_TO - 1 - offset % + # COMPRESS_PAD_TO`` tokens (see common/mm_preprocess.py), with + # COMPRESS_PAD_TO = 2 for v4.1's ratio-2 compressors. + self.mm_prefix_span_leading_pad_modulus = 2 if vision_n_layers > 0 else 0 diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index fa1af93f9d..72f2920480 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -317,6 +317,7 @@ class ModelArchConfigConvertorBase: "deepseek_v3", "deepseek_v32", "deepseek_v4", + "deepseek_v41", "dots3_note", "deepseek_mtp", "k3_dspark",