diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 5b5b8772c5..af9af8b51c 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -70,12 +70,26 @@ impl HfChatBackend { options, multimodal_render_info, )?), - RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()), - RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()), - RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()), - RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?), - RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?), - RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())), + RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new( + options.default_chat_template_kwargs, + )), + RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new( + options.default_chat_template_kwargs, + )), + RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new( + options.default_chat_template_kwargs, + )), + RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new( + options.default_chat_template_kwargs, + )?), + RendererSelection::Inkling => Arc::new(InklingChatRenderer::new( + tokenizer.clone(), + options.default_chat_template_kwargs, + )?), + RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new( + tokenizer.clone(), + options.default_chat_template_kwargs, + )), }; info!( @@ -280,6 +294,39 @@ mod tests { assert_eq!(prompt, "hello"); } + #[test] + fn native_renderer_inherits_deployment_reasoning_below_request_controls() { + let backend = HfChatBackend::from_resolved_model_files( + resolved_files(r#"{"model_type":"deepseek_v4"}"#, "{}"), + "test-model".to_string(), + LoadModelBackendsOptions { + default_chat_template_kwargs: [ + ("thinking".to_string(), serde_json::json!(false)), + ("reasoning_effort".to_string(), serde_json::json!("low")), + ] + .into(), + ..Default::default() + }, + test_tokenizer(), + ) + .unwrap(); + let mut request = request_with_user_text("hello"); + let rendered = backend.chat_renderer().render(&request).unwrap(); + assert!(rendered.prompt.into_text().unwrap().ends_with("")); + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], false); + + request.chat_options.reasoning_effort = Some(crate::EffortValue::from("max")); + let rendered = backend.chat_renderer().render(&request).unwrap(); + let prompt = rendered.prompt.into_text().unwrap(); + assert!(prompt.contains("Reasoning Effort: Beyond maximum")); + assert!(prompt.ends_with("")); + assert_eq!( + rendered.effective_template_kwargs["reasoning_effort"], + "max" + ); + assert_eq!(rendered.effective_template_kwargs["thinking"], true); + } + #[test] fn auto_uses_harmony_renderer_and_output_processor_for_gpt_oss_model_type() { let backend = backend_for_selection( diff --git a/rust/src/chat/src/backend/mod.rs b/rust/src/chat/src/backend/mod.rs index eed4e95cbe..af4579159e 100644 --- a/rust/src/chat/src/backend/mod.rs +++ b/rust/src/chat/src/backend/mod.rs @@ -78,8 +78,8 @@ pub struct LoadModelBackendsOptions { /// Optional server-default chat template override, provided either as an /// inline template or as a path to a template file. pub chat_template: Option, - /// Optional server-default keyword arguments merged into every - /// chat-template render before request-level `chat_template_kwargs`. + /// Server-default keyword arguments. HF merges these before request kwargs; + /// native renderers inherit supported reasoning controls below request controls. pub default_chat_template_kwargs: HashMap, /// Maximum number of input items allowed per prompt for each modality. /// Unspecified modalities are unlimited. diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index a8304f17aa..db985df58d 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -19,6 +19,8 @@ pub enum Error { ChatTemplate(String), #[error("{0}")] InvalidReasoningEffort(String), + #[error("{message}")] + InvalidReasoningControl { message: String }, #[error("multimodal input is not supported by this chat renderer")] UnsupportedMultimodalRenderer, #[error("unsupported multimodal content: {0}")] @@ -101,6 +103,7 @@ impl Error { match self { Self::PromptTooLong { .. } | Self::InvalidReasoningEffort(_) + | Self::InvalidReasoningControl { .. } | Self::DuplicateToolName { .. } | Self::ToolChoiceRequiresTools | Self::ToolChoiceFunctionNotFound { .. } => true, diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 9c59d14028..69d39ca78e 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -31,6 +31,7 @@ pub use parser::reasoning::{ }; pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory}; pub use parser::{ParserSelection, validate_parser_overrides}; +pub use reasoning::EffortValue; pub use renderer::hf::ChatTemplateContentFormatOption; pub use renderer::{ ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, @@ -39,7 +40,7 @@ pub use renderer::{ }; pub use request::{ ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool, - ChatToolChoice, GenerationPromptMode, ReasoningEffort, ResolvedToolContext, SamplingParams, + ChatToolChoice, GenerationPromptMode, ResolvedToolContext, SamplingParams, }; pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage}; pub use vllm_engine_core_client::protocol::multimodal::MmFeatures; @@ -52,6 +53,7 @@ mod event; pub mod multimodal; mod output; mod parser; +mod reasoning; mod renderer; mod request; mod stream; diff --git a/rust/src/chat/src/reasoning.rs b/rust/src/chat/src/reasoning.rs new file mode 100644 index 0000000000..b5a5b17684 --- /dev/null +++ b/rust/src/chat/src/reasoning.rs @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Shared reasoning-control resolution for native renderers. +//! +//! Each input source is parsed independently into [`ReasoningControl`]. Sources +//! then fall back in priority order: request, deployment defaults, model default. +//! Within a source, an explicit toggle decides the mode and effort supplies its +//! intensity. Keeping sources separate lets request effort override a deployment +//! toggle, and lets an enabled request inherit effort from enabled defaults. +//! +//! This module owns input types, precedence, and the standard `"none"` sentinel. +//! Model adapters own supported effort names, numeric ranges, and prompt encoding. +//! After model lowering, [`ReasoningControl::template_kwargs`] projects the prompt's +//! decision for downstream consumers. HF prepares raw template inputs separately. + +use std::collections::HashMap; +use std::fmt; + +use serde::{Deserialize, Serialize}; +use serde_json::{Number, Value, json}; + +use crate::error::{Result, invalid_reasoning_control}; +use crate::request::ChatRequest; + +/// A model-specific reasoning effort, before or after renderer lowering. +/// +/// Strings preserve model-specific names; JSON numbers preserve integer and +/// floating-point representations for model-local range validation. Missing/null +/// effort is represented by `Option::None` in [`crate::ChatOptions`]. +/// Supported names and ranges are validated by each renderer or HF template. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum EffortValue { + String(String), + Number(Number), +} + +impl TryFrom for EffortValue { + type Error = crate::error::Error; + + fn try_from(value: Value) -> Result { + match value { + Value::String(value) => Ok(Self::String(value)), + Value::Number(value) => Ok(Self::Number(value)), + _ => Err(invalid_reasoning_control!( + "reasoning effort must be a string or number, got {value}" + )), + } + } +} + +impl TryFrom for EffortValue { + type Error = crate::error::Error; + + /// Construct a numeric effort, rejecting NaN and infinities. + fn try_from(value: f64) -> Result { + Number::from_f64(value).map(Self::Number).ok_or_else(|| { + invalid_reasoning_control!("reasoning effort must be finite, got {value}") + }) + } +} + +impl From<&str> for EffortValue { + fn from(value: &str) -> Self { + Self::String(value.to_owned()) + } +} + +impl fmt::Display for EffortValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::String(value) => json!(value).fmt(f), + Self::Number(value) => value.fmt(f), + } + } +} + +impl EffortValue { + pub fn as_str(&self) -> Option<&str> { + match self { + Self::String(value) => Some(value), + Self::Number(_) => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + Self::Number(value) => value.as_f64(), + Self::String(_) => None, + } + } +} + +/// Reasoning intent for one source, or the result of falling back across sources. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum ReasoningControl { + /// This source leaves the mode and effort to a lower-priority source. + Default, + /// Reasoning is explicitly disabled; lower-priority defaults preserve this mode. + Disabled, + /// Reasoning is enabled. Missing effort inherits from an enabled default. + Enabled { effort: Option }, +} + +impl ReasoningControl { + /// Parse request controls; the typed effort field takes precedence over kwargs effort. + pub(crate) fn from_request(request: &ChatRequest) -> Result { + Self::from_kwargs( + &request.chat_options.template_kwargs, + request.chat_options.reasoning_effort.as_ref(), + ) + } + + /// Parse a deployment source, whose controls all come from template kwargs. + pub(crate) fn from_template_kwargs(kwargs: &HashMap) -> Result { + Self::from_kwargs(kwargs, None) + } + + /// Select controls within one source, then derive its mode and optional effort. + fn from_kwargs( + kwargs: &HashMap, + typed_effort: Option<&EffortValue>, + ) -> Result { + // Presence sets alias priority: `thinking` wins, including when its value + // is invalid. Validate the selected spelling as a boolean. + let toggle = ["thinking", "enable_thinking"] + .into_iter() + .find_map(|key| kwargs.get(key).map(|value| (key, value))) + .map(|(key, value)| { + value.as_bool().ok_or_else(|| { + invalid_reasoning_control!( + "template kwarg `{key}` must be a boolean, got {value}" + ) + }) + }) + .transpose()?; + + // The typed request field wins over kwargs. Parse kwargs only when used; + // disabled mode discards effort, and JSON null means omitted effort. + let effort = match (toggle, typed_effort) { + (Some(false), _) => None, + (_, Some(effort)) => Some(effort.clone()), + (_, None) => kwargs + .get("reasoning_effort") + .filter(|effort| !effort.is_null()) + .cloned() + .map(EffortValue::try_from) + .transpose()?, + }; + + match (toggle, effort) { + // 1. Explicit false disables reasoning and discards any effort, + // including values that would fail validation in an enabled mode. + (Some(false), _) => Ok(Self::Disabled), + // 2. `none` disables reasoning when the toggle is absent. Explicit true + // enables reasoning and leaves effort to enabled deployment/model defaults. + (toggle, Some(EffortValue::String(effort))) if effort == "none" => { + Ok(if toggle == Some(true) { + Self::Enabled { effort: None } + } else { + Self::Disabled + }) + } + // 3. Absent toggle and absent/null effort leave the entire decision + // to the next source in the fallback chain. + (None, None) => Ok(Self::Default), + // 4. Explicit true or a concrete effort enables reasoning; + // missing effort inherits later. + (_, effort) => Ok(Self::Enabled { effort }), + } + } + + /// Parse request and deployment independently, then apply request precedence. + /// + /// Each source validates its selected input types. Model adapters subsequently + /// apply their model default and validate/map the resulting effective effort. + pub(crate) fn resolve( + request: &ChatRequest, + defaults: &HashMap, + ) -> Result { + Ok(Self::from_request(request)?.fallback(Self::from_template_kwargs(defaults)?)) + } + + /// Fill this higher-priority source's missing decision or effort from `defaults`. + /// + /// `Default` inherits the whole lower-priority state. `Enabled { effort: None }` + /// inherits effort only from enabled defaults and retains its enabled mode. + /// Explicit disabling and concrete effort are final at this priority level. + pub(crate) fn fallback(self, defaults: Self) -> Self { + match (self, defaults) { + (Self::Default, defaults) => defaults, + (Self::Enabled { effort: None }, Self::Enabled { effort }) => Self::Enabled { effort }, + (request, _) => request, + } + } + + /// Construct an enabled model default or a model-lowered concrete effort. + pub(crate) fn enabled(effort: impl Into) -> Self { + Self::Enabled { + effort: Some(effort.into()), + } + } + + /// Report the resolved mode; callers apply model defaults before rendering. + pub(crate) fn is_enabled(&self) -> bool { + matches!(self, Self::Enabled { .. }) + } + + /// Return the active effort, which may still be missing before model fallback. + pub(crate) fn effort(&self) -> Option<&EffortValue> { + match self { + Self::Enabled { effort } => effort.as_ref(), + Self::Default | Self::Disabled => None, + } + } + + /// Project the same model-lowered decision used to render the native prompt. + /// + /// Preserve request extras and write both toggle aliases plus the effective + /// effort. Disabled mode emits `"none"`; enabled mode with no effective effort + /// clears the original effort key. `Default` preserves request kwargs. + pub(crate) fn template_kwargs(&self, request: &ChatRequest) -> HashMap { + let mut kwargs = request.chat_options.template_kwargs.clone(); + if matches!(self, Self::Default) { + return kwargs; + } + kwargs.insert("thinking".to_string(), json!(self.is_enabled())); + kwargs.insert("enable_thinking".to_string(), json!(self.is_enabled())); + match self { + Self::Disabled => { + kwargs.insert("reasoning_effort".to_string(), json!("none")); + } + Self::Enabled { + effort: Some(effort), + } => { + kwargs.insert("reasoning_effort".to_string(), json!(effort)); + } + Self::Enabled { effort: None } => { + kwargs.remove("reasoning_effort"); + } + Self::Default => unreachable!(), + } + kwargs + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kwargs(value: Value) -> HashMap { + serde_json::from_value(value).unwrap() + } + + #[test] + fn numeric_effort_rejects_non_finite_values() { + for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert!(EffortValue::try_from(value).unwrap_err().is_request_validation_error()); + } + } + + #[test] + fn explicit_toggle_decides_mode_and_none_inherits_enabled_effort() { + use ReasoningControl::{Default, Disabled, Enabled}; + + for (source, expected) in [ + (json!({}), Default), + (json!({"reasoning_effort": "none"}), Disabled), + ( + json!({"reasoning_effort": "high"}), + ReasoningControl::enabled("high"), + ), + (json!({"enable_thinking": true}), Enabled { effort: None }), + ( + json!({"enable_thinking": true, "reasoning_effort": "none"}), + Enabled { effort: None }, + ), + ( + json!({"enable_thinking": true, "reasoning_effort": "low"}), + ReasoningControl::enabled("low"), + ), + ( + json!({"enable_thinking": false, "reasoning_effort": "high"}), + Disabled, + ), + ( + json!({"thinking": true, "enable_thinking": false}), + Enabled { effort: None }, + ), + ( + json!({"thinking": false, "enable_thinking": true}), + Disabled, + ), + ( + json!({"reasoning_effort": 37}), + ReasoningControl::enabled(EffortValue::Number(37.into())), + ), + (json!({"reasoning_effort": null}), Default), + ] { + let source = kwargs(source); + assert_eq!( + ReasoningControl::from_template_kwargs(&source).unwrap(), + expected, + "{source:?}" + ); + } + } + + #[test] + fn request_and_deployment_states_fall_back_before_model_defaults() { + for (request, defaults, expected) in [ + ( + json!({}), + json!({"enable_thinking": false}), + ReasoningControl::Disabled, + ), + ( + json!({"reasoning_effort": "high"}), + json!({"thinking": false}), + ReasoningControl::enabled("high"), + ), + ( + json!({"enable_thinking": false}), + json!({"reasoning_effort": "high"}), + ReasoningControl::Disabled, + ), + ( + json!({"thinking": true, "reasoning_effort": "none"}), + json!({"reasoning_effort": "low"}), + ReasoningControl::enabled("low"), + ), + ( + json!({"thinking": true}), + json!({"thinking": false, "reasoning_effort": "low"}), + ReasoningControl::enabled("max"), + ), + (json!({}), json!({}), ReasoningControl::enabled("max")), + ] { + let request = kwargs(request); + let defaults = kwargs(defaults); + let actual = ReasoningControl::from_template_kwargs(&request) + .unwrap() + .fallback(ReasoningControl::from_template_kwargs(&defaults).unwrap()) + .fallback(ReasoningControl::enabled("max")); + assert_eq!( + actual, expected, + "request={request:?}, defaults={defaults:?}" + ); + } + } + + #[test] + fn selected_toggle_requires_a_boolean() { + for value in [ + json!("true"), + json!(1), + json!(null), + json!({"type": "enabled"}), + ] { + assert!( + ReasoningControl::from_template_kwargs(&kwargs(json!({ + "thinking": value, "enable_thinking": true + }))) + .unwrap_err() + .is_request_validation_error() + ); + } + } + + #[test] + fn invalid_effort_types_are_rejected_only_when_the_source_uses_them() { + for value in [json!(true), json!(false), json!([]), json!({})] { + for toggle in [None, Some(true), Some(false)] { + let mut source = kwargs(json!({"reasoning_effort": value})); + if let Some(toggle) = toggle { + source.insert("thinking".to_owned(), json!(toggle)); + } + let control = ReasoningControl::from_template_kwargs(&source); + if toggle == Some(false) { + assert_eq!(control.unwrap(), ReasoningControl::Disabled); + } else { + assert!(control.unwrap_err().is_request_validation_error()); + } + } + + // A typed request effort shadows malformed kwargs effort. + let mut request = ChatRequest::for_test(); + request.chat_options.reasoning_effort = Some(EffortValue::from("high")); + request.chat_options.template_kwargs = kwargs(json!({"reasoning_effort": value})); + assert_eq!( + ReasoningControl::from_request(&request).unwrap(), + ReasoningControl::enabled("high") + ); + } + } +} diff --git a/rust/src/chat/src/renderer/deepseek.rs b/rust/src/chat/src/renderer/deepseek.rs index 01576a7c43..7892fd2497 100644 --- a/rust/src/chat/src/renderer/deepseek.rs +++ b/rust/src/chat/src/renderer/deepseek.rs @@ -17,10 +17,10 @@ use serde_json_fmt::JsonFormat; use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER; use super::MediaPartSource; +use crate::EffortValue; use crate::error::{Error, Result}; -use crate::request::{ - ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort, -}; +use crate::reasoning::ReasoningControl; +use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool}; use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall}; const BOS_TOKEN: &str = "<|begin▁of▁sentence|>"; @@ -87,21 +87,26 @@ struct RenderedToolSchema<'a> { } /// Render one chat request into the final prompt string. -pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result { - render_request_with_media_order(request, dialect).map(|(prompt, _)| prompt) +pub(super) fn render_request( + request: &ChatRequest, + dialect: DsDialect, + reasoning: &ReasoningControl, +) -> Result { + render_request_with_media_order(request, dialect, reasoning).map(|(prompt, _)| prompt) } /// Render one chat request and record media in placeholder order. pub(super) fn render_request_with_media_order( request: &ChatRequest, dialect: DsDialect, + reasoning: &ReasoningControl, ) -> Result<(String, Vec)> { - let (thinking_mode, reasoning_effort_prompt) = match dialect { - DsDialect::V4 => { - resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))? - } - DsDialect::V41 => resolve_v41_thinking_options(request)?, + let thinking_mode = if reasoning.is_enabled() { + ThinkingMode::Thinking + } else { + ThinkingMode::Chat }; + let reasoning_effort_prompt = reasoning_prompt(reasoning, dialect); let request_tools = request_tools(request); let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools); let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true) @@ -254,76 +259,63 @@ pub(super) fn render_request_with_media_order( Ok((out, media_order)) } -/// Resolve DeepSeek V4's thinking controls. Unlike the Python tokenizer -/// wrapper, the Rust renderer only consumes the typed top-level -/// `reasoning_effort`; the generic template-kwargs map is left for HF -/// templates. -fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'static str)> { - let mut thinking_mode = match request.enable_thinking()?.unwrap_or(true) { - true => ThinkingMode::Thinking, - false => ThinkingMode::Chat, - }; - let mut reasoning_effort_prompt = REASONING_EFFORT_HIGH; - - match request.chat_options.reasoning_effort { - Some(ReasoningEffort::None) => thinking_mode = ThinkingMode::Chat, - Some(ReasoningEffort::Max) => { - reasoning_effort_prompt = REASONING_EFFORT_MAX; - } - Some(ReasoningEffort::XHigh | ReasoningEffort::High) => { - reasoning_effort_prompt = REASONING_EFFORT_HIGH; - } - Some(ReasoningEffort::Minimal | ReasoningEffort::Medium | ReasoningEffort::Low) => { - reasoning_effort_prompt = ""; - } - None => {} +/// Map the shared reasoning decision to the selected DeepSeek effort dialect. +pub(super) fn resolve_reasoning( + control: ReasoningControl, + dialect: DsDialect, +) -> Result { + let control = control.fallback(ReasoningControl::enabled("high")); + if !control.is_enabled() { + return Ok(control); } - - Ok((thinking_mode, reasoning_effort_prompt)) + let effort = control.effort().expect("enabled DeepSeek reasoning has a default effort"); + let effort = match dialect { + DsDialect::V4 => { + let effort = match effort.as_str() { + Some("minimal" | "medium" | "low") => "low", + Some("high" | "xhigh") => "high", + Some("max") => "max", + _ => { + return Err(Error::InvalidReasoningEffort(format!( + "DeepSeek V4 reasoning_effort must be minimal, low, medium, high, xhigh, or max, got {effort}" + ))); + } + }; + EffortValue::from(effort) + } + DsDialect::V41 => { + let budget = match effort { + EffortValue::String(effort) => match effort.as_str() { + "low" => 25, + "high" => 50, + "xhigh" => 75, + "max" => 100, + _ => return Err(invalid_v41_reasoning_effort()), + }, + EffortValue::Number(budget) => budget + .as_u64() + .filter(|budget| (1..=100).contains(budget)) + .ok_or_else(invalid_v41_reasoning_effort)?, + }; + EffortValue::Number(budget.into()) + } + }; + Ok(ReasoningControl::enabled(effort)) } -/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs. -fn resolve_v41_thinking_options( - request: &ChatRequest, -) -> Result<(ThinkingMode, Cow<'static, str>)> { - let mut thinking = request.enable_thinking()?.unwrap_or(true); - let budget = match request.chat_options.reasoning_effort { - Some(ReasoningEffort::None) => { - thinking = false; - 50 - } - Some(ReasoningEffort::Low) => 25, - Some(ReasoningEffort::High) => 50, - Some(ReasoningEffort::XHigh) => 75, - Some(ReasoningEffort::Max) => 100, - Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => { - return Err(invalid_v41_reasoning_effort()); - } - None => match request.chat_options.template_kwargs.get("reasoning_effort") { - None | Some(Value::Null) => 50, - Some(Value::String(effort)) => match effort.as_str() { - "low" => 25, - "high" => 50, - "xhigh" => 75, - "max" => 100, - _ => return Err(invalid_v41_reasoning_effort()), - }, - Some(Value::Number(budget)) => budget - .as_u64() - .filter(|budget| (1..=100).contains(budget)) - .ok_or_else(invalid_v41_reasoning_effort)?, - Some(_) => return Err(invalid_v41_reasoning_effort()), - }, +fn reasoning_prompt(control: &ReasoningControl, dialect: DsDialect) -> Cow<'static, str> { + let Some(effort) = control.effort() else { + return Cow::Borrowed(""); }; - if thinking { - Ok(( - ThinkingMode::Thinking, - Cow::Owned(format!( - "Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n" - )), - )) - } else { - Ok((ThinkingMode::Chat, Cow::Borrowed(""))) + match dialect { + DsDialect::V4 => Cow::Borrowed(match effort.as_str() { + Some("high") => REASONING_EFFORT_HIGH, + Some("max") => REASONING_EFFORT_MAX, + _ => "", + }), + DsDialect::V41 => Cow::Owned(format!( + "Reasoning Effort: {effort} (range 1-100, the higher the value, the more thorough the reasoning)\n\n" + )), } } diff --git a/rust/src/chat/src/renderer/deepseek_v32/encoding.rs b/rust/src/chat/src/renderer/deepseek_v32/encoding.rs index 4fc5f26530..74c7e4cbf2 100644 --- a/rust/src/chat/src/renderer/deepseek_v32/encoding.rs +++ b/rust/src/chat/src/renderer/deepseek_v32/encoding.rs @@ -14,6 +14,7 @@ use serde_json::Value; use serde_json_fmt::JsonFormat; use crate::error::{Error, Result}; +use crate::reasoning::ReasoningControl; use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatRole, ChatTool}; use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall}; @@ -42,8 +43,11 @@ struct RenderedToolSchema<'a> { } /// Render one chat request into the final prompt string. -pub(super) fn render_request(request: &ChatRequest) -> Result { - let thinking_mode = match request.enable_thinking()?.unwrap_or(false) { +pub(super) fn render_request( + request: &ChatRequest, + reasoning: &ReasoningControl, +) -> Result { + let thinking_mode = match reasoning.is_enabled() { true => ThinkingMode::Thinking, false => ThinkingMode::Chat, }; diff --git a/rust/src/chat/src/renderer/deepseek_v32/mod.rs b/rust/src/chat/src/renderer/deepseek_v32/mod.rs index 03c2765d19..b4ac1bae11 100644 --- a/rust/src/chat/src/renderer/deepseek_v32/mod.rs +++ b/rust/src/chat/src/renderer/deepseek_v32/mod.rs @@ -3,31 +3,41 @@ mod encoding; +use std::collections::HashMap; + +use serde_json::Value; use vllm_text::Prompt; -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use super::{ChatRenderer, RenderedPrompt}; use crate::Result; +use crate::reasoning::ReasoningControl; use crate::request::ChatRequest; /// Dedicated DeepSeek V3.2 renderer. -#[derive(Debug, Clone, Copy, Default)] -pub struct DeepSeekV32ChatRenderer; +#[derive(Debug, Clone, Default)] +pub struct DeepSeekV32ChatRenderer { + default_template_kwargs: HashMap, +} impl DeepSeekV32ChatRenderer { /// Create the dedicated DeepSeek V3.2 renderer. - pub fn new() -> Self { - Self + pub fn new(default_template_kwargs: HashMap) -> Self { + Self { + default_template_kwargs, + } } } impl ChatRenderer for DeepSeekV32ChatRenderer { fn render(&self, request: &ChatRequest) -> Result { request.validate()?; + let reasoning = ReasoningControl::resolve(request, &self.default_template_kwargs)? + .fallback(ReasoningControl::Disabled); Ok(RenderedPrompt { - prompt: Prompt::Text(encoding::render_request(request)?), + prompt: Prompt::Text(encoding::render_request(request, &reasoning)?), media_order: None, - effective_template_kwargs: request_template_kwargs(request), + effective_template_kwargs: reasoning.template_kwargs(request), }) } } diff --git a/rust/src/chat/src/renderer/deepseek_v32/tests.rs b/rust/src/chat/src/renderer/deepseek_v32/tests.rs index 56c03a92ca..dd9ec13ddc 100644 --- a/rust/src/chat/src/renderer/deepseek_v32/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v32/tests.rs @@ -8,6 +8,7 @@ use serde_json::{Value, json}; use thiserror_ext::AsReport; use super::DeepSeekV32ChatRenderer; +use crate::EffortValue; use crate::error::Error; use crate::event::{AssistantContentBlock, AssistantToolCall}; use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; @@ -18,7 +19,7 @@ use crate::request::{ use crate::{ChatRenderer, ChatRole}; fn render_request(request: &ChatRequest) -> String { - DeepSeekV32ChatRenderer::new() + DeepSeekV32ChatRenderer::default() .render(request) .unwrap() .prompt @@ -26,8 +27,30 @@ fn render_request(request: &ChatRequest) -> String { .expect("deepseek renderer should return text prompt") } +#[test] +fn effort_enables_binary_thinking_and_none_disables_it() { + let mut request = ChatRequest::for_test(); + for (effort, enabled) in [ + (EffortValue::from("high"), true), + (EffortValue::from("custom"), true), + (EffortValue::Number(42.into()), true), + (EffortValue::from("none"), false), + ] { + request.chat_options.reasoning_effort = Some(effort); + let rendered = DeepSeekV32ChatRenderer::default().render(&request).unwrap(); + assert_eq!( + rendered.prompt.into_text().unwrap().ends_with(""), + enabled + ); + assert_eq!( + rendered.effective_template_kwargs["enable_thinking"], + enabled + ); + } +} + fn render_result(request: &ChatRequest) -> Result { - DeepSeekV32ChatRenderer::new().render(request).map(|rendered| { + DeepSeekV32ChatRenderer::default().render(request).map(|rendered| { rendered .prompt .into_text() @@ -345,7 +368,7 @@ fn render_rejects_multimodal_input() { ..ChatRequest::for_test() }; - let error = DeepSeekV32ChatRenderer::new().render(&request).unwrap_err(); + let error = DeepSeekV32ChatRenderer::default().render(&request).unwrap_err(); assert!(matches!( error, diff --git a/rust/src/chat/src/renderer/deepseek_v4/mod.rs b/rust/src/chat/src/renderer/deepseek_v4/mod.rs index 54b111e505..136938fbc0 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/mod.rs +++ b/rust/src/chat/src/renderer/deepseek_v4/mod.rs @@ -1,31 +1,47 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::collections::HashMap; + +use serde_json::Value; use vllm_text::Prompt; use super::deepseek::{self, DsDialect}; -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use super::{ChatRenderer, RenderedPrompt}; use crate::Result; +use crate::reasoning::ReasoningControl; use crate::request::ChatRequest; /// Dedicated DeepSeek V4 renderer. -#[derive(Debug, Clone, Copy, Default)] -pub struct DeepSeekV4ChatRenderer; +#[derive(Debug, Clone, Default)] +pub struct DeepSeekV4ChatRenderer { + default_template_kwargs: HashMap, +} impl DeepSeekV4ChatRenderer { - pub fn new() -> Self { - Self + pub fn new(default_template_kwargs: HashMap) -> Self { + Self { + default_template_kwargs, + } } } impl ChatRenderer for DeepSeekV4ChatRenderer { fn render(&self, request: &ChatRequest) -> Result { request.validate()?; + let reasoning = deepseek::resolve_reasoning( + ReasoningControl::resolve(request, &self.default_template_kwargs)?, + DsDialect::V4, + )?; Ok(RenderedPrompt { - prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?), + prompt: Prompt::Text(deepseek::render_request( + request, + DsDialect::V4, + &reasoning, + )?), media_order: None, - effective_template_kwargs: request_template_kwargs(request), + effective_template_kwargs: reasoning.template_kwargs(request), }) } } diff --git a/rust/src/chat/src/renderer/deepseek_v4/tests.rs b/rust/src/chat/src/renderer/deepseek_v4/tests.rs index 9404b54296..21a33ec2d5 100644 --- a/rust/src/chat/src/renderer/deepseek_v4/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v4/tests.rs @@ -8,13 +8,14 @@ use serde_json::Value; use super::DeepSeekV4ChatRenderer; use crate::ChatRenderer; +use crate::EffortValue; use crate::event::{AssistantContentBlock, AssistantToolCall}; use crate::renderer::deepseek_v41::DeepSeekV41ChatRenderer; use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; -use crate::request::{ChatMessage, ChatRequest, ChatTool, GenerationPromptMode, ReasoningEffort}; +use crate::request::{ChatMessage, ChatRequest, ChatTool, GenerationPromptMode}; fn render_request(request: &ChatRequest) -> String { - DeepSeekV4ChatRenderer::new() + DeepSeekV4ChatRenderer::default() .render(request) .unwrap() .prompt @@ -31,13 +32,13 @@ fn thinking_request(messages: Vec) -> ChatRequest { .chat_options .template_kwargs .insert("thinking".to_string(), Value::Bool(true)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low); + request.chat_options.reasoning_effort = Some(EffortValue::from("low")); request } fn fixture_request(input_name: &str) -> ChatRequest { let mut request = fixture_chat_request(&fixture_path(input_name), deepseek_fixture_options()); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low); + request.chat_options.reasoning_effort = Some(EffortValue::from("low")); request } @@ -203,7 +204,7 @@ fn reasoning_effort_max_adds_prefix_when_thinking_is_enabled() { .chat_options .template_kwargs .insert("thinking".to_string(), Value::Bool(true)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Max); + request.chat_options.reasoning_effort = Some(EffortValue::from("max")); let rendered = render_request(&request); @@ -226,7 +227,7 @@ fn reasoning_effort_high_adds_0731_high_prefix() { .chat_options .template_kwargs .insert("thinking".to_string(), Value::Bool(true)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::High); + request.chat_options.reasoning_effort = Some(EffortValue::from("high")); let rendered = render_request(&request); @@ -258,7 +259,7 @@ fn reasoning_effort_xhigh_maps_to_high() { messages: vec![ChatMessage::user("solve it")], ..ChatRequest::for_test() }; - request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh); + request.chat_options.reasoning_effort = Some(EffortValue::from("xhigh")); let rendered = render_request(&request); @@ -268,7 +269,7 @@ fn reasoning_effort_xhigh_maps_to_high() { #[test] fn minimal_and_medium_reasoning_effort_map_to_low() { - for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] { + for effort in [EffortValue::from("minimal"), EffortValue::from("medium")] { let mut request = ChatRequest { messages: vec![ChatMessage::user("solve it")], ..ChatRequest::for_test() @@ -292,7 +293,7 @@ fn reasoning_effort_low_keeps_the_default_prompt() { .chat_options .template_kwargs .insert("thinking".to_string(), Value::Bool(true)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low); + request.chat_options.reasoning_effort = Some(EffortValue::from("low")); let rendered = render_request(&request); @@ -305,11 +306,7 @@ fn reasoning_effort_none_disables_thinking() { messages: vec![ChatMessage::user("answer directly")], ..ChatRequest::for_test() }; - request - .chat_options - .template_kwargs - .insert("thinking".to_string(), Value::Bool(true)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + request.chat_options.reasoning_effort = Some(EffortValue::from("none")); let rendered = render_request(&request); @@ -317,6 +314,49 @@ fn reasoning_effort_none_disables_thinking() { .assert_eq(&rendered); } +#[test] +fn normalized_reasoning_drives_both_prompt_and_compatibility_kwargs() { + for (kwargs, enabled, effort, prefix) in [ + ( + serde_json::json!({"enable_thinking": true, "reasoning_effort": "none"}), + true, + "high", + "Absolute maximum", + ), + ( + serde_json::json!({"thinking": false, "enable_thinking": true, "reasoning_effort": "max"}), + false, + "none", + "", + ), + ( + serde_json::json!({"reasoning_effort": "xhigh"}), + true, + "high", + "Absolute maximum", + ), + ] { + let mut request = ChatRequest::for_test(); + request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap(); + let rendered = DeepSeekV4ChatRenderer::default().render(&request).unwrap(); + let prompt = rendered.prompt.into_text().unwrap(); + assert_eq!(prompt.ends_with(""), enabled); + assert_eq!(prompt.contains("Reasoning Effort:"), enabled); + if enabled { + assert!(prompt.contains(prefix)); + } + assert_eq!(rendered.effective_template_kwargs["thinking"], enabled); + assert_eq!( + rendered.effective_template_kwargs["enable_thinking"], + enabled + ); + assert_eq!( + rendered.effective_template_kwargs["reasoning_effort"], + effort + ); + } +} + #[test] fn system_to_assistant_transitions_in_chat_and_thinking_modes() { let rendered = [false, true].map(|enable_thinking| { @@ -338,7 +378,7 @@ fn system_to_assistant_transitions_in_chat_and_thinking_modes() { .chat_options .template_kwargs .insert("enable_thinking".to_string(), Value::Bool(enable_thinking)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low); + request.chat_options.reasoning_effort = Some(EffortValue::from("low")); render_request(&request) }); @@ -366,7 +406,7 @@ fn trailing_system_transitions_in_chat_and_thinking_modes() { .chat_options .template_kwargs .insert("enable_thinking".to_string(), Value::Bool(enable_thinking)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low); + request.chat_options.reasoning_effort = Some(EffortValue::from("low")); render_request(&request) }); @@ -380,7 +420,7 @@ fn trailing_system_transitions_in_chat_and_thinking_modes() { } #[test] -fn reasoning_effort_template_kwarg_is_ignored() { +fn reasoning_effort_template_kwarg_selects_effort() { let mut request = ChatRequest { messages: vec![ChatMessage::user("solve it")], ..ChatRequest::for_test() @@ -396,15 +436,15 @@ fn reasoning_effort_template_kwarg_is_ignored() { let rendered = render_request(&request); - assert!(rendered.starts_with("<|begin▁of▁sentence|>Reasoning Effort: Absolute maximum")); + assert!(rendered.starts_with("<|begin▁of▁sentence|>Reasoning Effort: Beyond maximum")); assert!(rendered.ends_with("<|Assistant|>")); } #[test] fn tool_call_arguments_preserve_non_objects_like_recipe() { let renderers: [(&dyn ChatRenderer, &str); 2] = [ - (&DeepSeekV4ChatRenderer::new(), "parameter"), - (&DeepSeekV41ChatRenderer::new(), " parameter"), + (&DeepSeekV4ChatRenderer::default(), "parameter"), + (&DeepSeekV41ChatRenderer::default(), " parameter"), ]; for arguments in [ "not json", @@ -600,7 +640,7 @@ fn drop_thinking_false_keeps_prior_assistant_reasoning() { .chat_options .template_kwargs .insert("drop_thinking".to_string(), Value::Bool(false)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low); + request.chat_options.reasoning_effort = Some(EffortValue::from("low")); let rendered = render_request(&request); @@ -693,7 +733,7 @@ fn trailing_system_omits_generation_prompt_when_disabled() { .chat_options .template_kwargs .insert("enable_thinking".to_string(), Value::Bool(false)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low); + request.chat_options.reasoning_effort = Some(EffortValue::from("low")); let rendered = render_request(&request); diff --git a/rust/src/chat/src/renderer/deepseek_v41/mod.rs b/rust/src/chat/src/renderer/deepseek_v41/mod.rs index 7737f7ebda..54544ec383 100644 --- a/rust/src/chat/src/renderer/deepseek_v41/mod.rs +++ b/rust/src/chat/src/renderer/deepseek_v41/mod.rs @@ -1,33 +1,45 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::collections::HashMap; + +use serde_json::Value; use vllm_text::Prompt; use super::deepseek::{self, DsDialect}; -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use super::{ChatRenderer, RenderedPrompt}; use crate::Result; +use crate::reasoning::ReasoningControl; use crate::request::ChatRequest; /// Dedicated DeepSeek V4.1 renderer. -#[derive(Debug, Clone, Copy, Default)] -pub struct DeepSeekV41ChatRenderer; +#[derive(Debug, Clone, Default)] +pub struct DeepSeekV41ChatRenderer { + default_template_kwargs: HashMap, +} impl DeepSeekV41ChatRenderer { - pub fn new() -> Self { - Self + pub fn new(default_template_kwargs: HashMap) -> Self { + Self { + default_template_kwargs, + } } } impl ChatRenderer for DeepSeekV41ChatRenderer { fn render(&self, request: &ChatRequest) -> Result { request.validate()?; + let reasoning = deepseek::resolve_reasoning( + ReasoningControl::resolve(request, &self.default_template_kwargs)?, + DsDialect::V41, + )?; let (prompt, media_order) = - deepseek::render_request_with_media_order(request, DsDialect::V41)?; + deepseek::render_request_with_media_order(request, DsDialect::V41, &reasoning)?; Ok(RenderedPrompt { prompt: Prompt::Text(prompt), media_order: Some(media_order), - effective_template_kwargs: request_template_kwargs(request), + effective_template_kwargs: reasoning.template_kwargs(request), }) } } diff --git a/rust/src/chat/src/renderer/deepseek_v41/tests.rs b/rust/src/chat/src/renderer/deepseek_v41/tests.rs index 3d621c2246..62cf6d2024 100644 --- a/rust/src/chat/src/renderer/deepseek_v41/tests.rs +++ b/rust/src/chat/src/renderer/deepseek_v41/tests.rs @@ -8,14 +8,15 @@ use serde_json::{Value, json}; use super::DeepSeekV41ChatRenderer; use crate::ChatRenderer; +use crate::EffortValue; use crate::event::{AssistantContentBlock, AssistantToolCall}; use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; use crate::request::{ - ChatContent, ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort, + ChatContent, ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, }; fn render(request: &ChatRequest) -> String { - DeepSeekV41ChatRenderer::new() + DeepSeekV41ChatRenderer::default() .render(request) .unwrap() .prompt @@ -64,7 +65,7 @@ fn inlines_image_placeholder_at_image_part_positions() { // The placeholder is unconditional (matching the Python encoding's // `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later, // at media preparation. - let renderer = DeepSeekV41ChatRenderer::new(); + let renderer = DeepSeekV41ChatRenderer::default(); let request = ChatRequest { messages: vec![ChatMessage::User { content: ChatContent::Parts(vec![ @@ -87,7 +88,7 @@ fn inlines_image_placeholder_at_image_part_positions() { #[test] fn media_order_follows_reordered_tool_results() { - let renderer = DeepSeekV41ChatRenderer::new(); + let renderer = DeepSeekV41ChatRenderer::default(); let request = ChatRequest { messages: vec![ ChatMessage::assistant_blocks(vec![ @@ -136,7 +137,7 @@ fn media_order_follows_reordered_tool_results() { #[test] fn media_order_excludes_dropped_historical_developer_content() { - let renderer = DeepSeekV41ChatRenderer::new(); + let renderer = DeepSeekV41ChatRenderer::default(); let request = ChatRequest { messages: vec![ ChatMessage::developer(vec![image_part("historical-image")], None), @@ -165,10 +166,11 @@ fn image_part(uuid: &str) -> ChatContentPart { fn maps_reasoning_effort_to_reference_numeric_budget() { for (effort, budget) in [ (None, 50), - (Some(ReasoningEffort::Low), 25), - (Some(ReasoningEffort::High), 50), - (Some(ReasoningEffort::XHigh), 75), - (Some(ReasoningEffort::Max), 100), + (Some(EffortValue::from("low")), 25), + (Some(EffortValue::from("high")), 50), + (Some(EffortValue::from("xhigh")), 75), + (Some(EffortValue::from("max")), 100), + (Some(EffortValue::Number(37.into())), 37), ] { let mut request = request(); request.chat_options.reasoning_effort = effort; @@ -193,11 +195,43 @@ fn accepts_numeric_template_effort_with_top_level_precedence() { request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort); assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100"))); - request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh); + request.chat_options.reasoning_effort = Some(EffortValue::from("xhigh")); assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100")); } } +#[test] +fn enabling_with_none_inherits_deployment_effort_and_reports_numeric_control() { + let renderer = + DeepSeekV41ChatRenderer::new([("reasoning_effort".to_string(), json!(37))].into()); + let mut request = request(); + request.chat_options.reasoning_effort = Some(EffortValue::from("none")); + request + .chat_options + .template_kwargs + .insert("enable_thinking".into(), json!(true)); + let rendered = renderer.render(&request).unwrap(); + let prompt = rendered.prompt.into_text().unwrap(); + assert!(prompt.contains("Reasoning Effort: 37 (range 1-100")); + assert!(prompt.ends_with("")); + assert_eq!(rendered.effective_template_kwargs["reasoning_effort"], 37); + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], true); + + request.chat_options.template_kwargs.insert("thinking".into(), json!(false)); + request + .chat_options + .template_kwargs + .insert("reasoning_effort".into(), json!(101)); + request.chat_options.reasoning_effort = None; + let rendered = renderer.render(&request).unwrap(); + assert!(rendered.prompt.into_text().unwrap().ends_with("")); + assert_eq!( + rendered.effective_template_kwargs["reasoning_effort"], + "none" + ); + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], false); +} + #[test] fn rejects_undefined_effort_names_and_invalid_numeric_budgets() { for effort in [ @@ -211,13 +245,13 @@ fn rejects_undefined_effort_names_and_invalid_numeric_budgets() { ] { let mut request = request(); request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort); - let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err(); + let error = DeepSeekV41ChatRenderer::default().render(&request).unwrap_err(); assert!(error.is_request_validation_error()); } - for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] { + for effort in [EffortValue::from("minimal"), EffortValue::from("medium")] { let mut request = request(); request.chat_options.reasoning_effort = Some(effort); - let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err(); + let error = DeepSeekV41ChatRenderer::default().render(&request).unwrap_err(); assert!(error.is_request_validation_error()); } } @@ -229,8 +263,8 @@ fn chat_mode_and_none_effort_use_closed_thinking_prefix() { let expected = expect!["<|begin▁of▁sentence|><|User|>question<|Assistant|>"]; expected.assert_eq(&render(&request)); - request.chat_options.template_kwargs.insert("thinking".into(), json!(true)); - request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + request.chat_options.template_kwargs.remove("thinking"); + request.chat_options.reasoning_effort = Some(EffortValue::from("none")); expected.assert_eq(&render(&request)); } diff --git a/rust/src/chat/src/renderer/harmony/mod.rs b/rust/src/chat/src/renderer/harmony/mod.rs index 71431cc92b..83b59e9198 100644 --- a/rust/src/chat/src/renderer/harmony/mod.rs +++ b/rust/src/chat/src/renderer/harmony/mod.rs @@ -5,21 +5,26 @@ pub(crate) mod encoding; +use std::collections::HashMap; + use openai_harmony::HarmonyEncoding; use openai_harmony::chat::{ Author, Conversation, DeveloperContent, Message, ReasoningEffort as HarmonyReasoningEffort, Role, SystemContent, ToolDescription, }; +use serde_json::Value; use thiserror_ext::AsReport as _; use time::macros::format_description; use vllm_text::Prompt; use self::encoding::harmony_encoding; -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use super::{ChatRenderer, RenderedPrompt}; +use crate::AssistantMessageExt as _; +use crate::EffortValue; use crate::error::{Error, Result}; use crate::event::AssistantContentBlock; +use crate::reasoning::ReasoningControl; use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, GenerationPromptMode}; -use crate::{AssistantMessageExt as _, ReasoningEffort}; const SYSTEM_START_DATE_ENV: &str = "VLLM_SYSTEM_START_DATE"; const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS"; @@ -28,6 +33,7 @@ const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTR pub struct HarmonyChatRenderer { encoding: &'static HarmonyEncoding, options: Options, + default_template_kwargs: HashMap, } struct Options { @@ -44,8 +50,9 @@ impl HarmonyChatRenderer { /// unset, the renderer uses the current local date with a UTC fallback. /// - `VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS` moves leading instructions /// into the system model identity when set to a non-zero integer. - pub fn new() -> Result { + pub fn new(default_template_kwargs: HashMap) -> Result { Self::with_options( + default_template_kwargs, env_system_start_date(), env_use_harmony_system_instructions(), ) @@ -57,11 +64,13 @@ impl HarmonyChatRenderer { /// Production code should call [`Self::new`] so the renderer observes the /// same environment contract as the Python Harmony path. pub fn with_options( + default_template_kwargs: HashMap, system_start_date: impl Into, use_system_instructions: bool, ) -> Result { Ok(Self { encoding: harmony_encoding()?, + default_template_kwargs, options: Options { system_start_date: system_start_date.into(), use_system_instructions, @@ -73,7 +82,11 @@ impl HarmonyChatRenderer { /// /// Harmony owns both prompt formatting and tokenization, so the Rust /// frontend bypasses the generic HF tokenizer path for GPT-OSS input. - fn render_token_ids(&self, request: &ChatRequest) -> Result> { + fn render_token_ids( + &self, + request: &ChatRequest, + effort: HarmonyReasoningEffort, + ) -> Result> { if request.has_multimodal() { return Err(Error::UnsupportedMultimodalContent("image_url")); } @@ -86,7 +99,8 @@ impl HarmonyChatRenderer { )); } - let messages = auto_drop_analysis_messages(to_harmony_messages(request, &self.options)?); + let messages = + auto_drop_analysis_messages(to_harmony_messages(request, &self.options, effort)?); let conversation = Conversation::from_messages(messages); // Pass `None` so oss-harmony does not apply its narrower built-in // analysis-drop policy after the Rust-side Python-parity cleanup above. @@ -114,10 +128,13 @@ impl ChatRenderer for HarmonyChatRenderer { /// Render a chat request as [`Prompt::TokenIds`] with template kwargs echoed /// for downstream accounting/debugging. fn render(&self, request: &ChatRequest) -> Result { + let reasoning = ReasoningControl::resolve(request, &self.default_template_kwargs)? + .fallback(ReasoningControl::enabled("medium")); + let effort = to_harmony_reasoning_effort(reasoning.effort())?; Ok(RenderedPrompt { - prompt: Prompt::TokenIds(self.render_token_ids(request)?), + prompt: Prompt::TokenIds(self.render_token_ids(request, effort)?), media_order: None, - effective_template_kwargs: request_template_kwargs(request), + effective_template_kwargs: reasoning.template_kwargs(request), }) } } @@ -127,12 +144,21 @@ impl ChatRenderer for HarmonyChatRenderer { /// This adds the Harmony system/developer preamble, peels at most one leading /// system/developer instruction message, and then lowers the remaining chat /// history message-by-message. -fn to_harmony_messages(request: &ChatRequest, options: &Options) -> Result> { +fn to_harmony_messages( + request: &ChatRequest, + options: &Options, + effort: HarmonyReasoningEffort, +) -> Result> { let (instructions, leading_developer_tools, remaining_messages) = peel_leading_instructions(&request.messages)?; let tool_call_names = tool_call_names(&request.messages); - let mut messages = - build_harmony_preamble(request, instructions, leading_developer_tools, options)?; + let mut messages = build_harmony_preamble( + request, + instructions, + leading_developer_tools, + options, + effort, + )?; for message in remaining_messages { messages.extend(to_harmony_message(message, &tool_call_names, options)?); @@ -177,12 +203,13 @@ fn build_harmony_preamble( instructions: Option, leading_developer_tools: Option<&[ChatTool]>, options: &Options, + effort: HarmonyReasoningEffort, ) -> Result> { let mut messages = vec![Message::from_role_and_content( Role::System, system_content( instructions.as_deref().filter(|_| options.use_system_instructions), - request.chat_options.reasoning_effort, + Some(effort), &options.system_start_date, )?, )]; @@ -227,18 +254,18 @@ fn preamble_tool_descriptions( /// Construct the Harmony system content for the request preamble. /// -/// Harmony defaults the reasoning effort to `medium` when none is provided, so -/// this only sets an explicit effort after validating vLLM's request value. +/// The preamble receives normalized effort; later system messages use Harmony's +/// own default when their effort is omitted. fn system_content( instructions: Option<&str>, - reasoning_effort: Option, + reasoning_effort: Option, system_start_date: &str, ) -> Result { let mut content = SystemContent::new().with_conversation_start_date(system_start_date.to_string()); if let Some(reasoning_effort) = reasoning_effort { - content = content.with_reasoning_effort(to_harmony_reasoning_effort(reasoning_effort)?); + content = content.with_reasoning_effort(reasoning_effort); } if let Some(instructions) = instructions.filter(|text| !text.is_empty()) { @@ -445,20 +472,17 @@ fn to_tool_descriptions(tools: &[ChatTool]) -> Vec { .collect() } -/// Map supported OpenAI reasoning-effort values onto Harmony's enum. +/// Map supported reasoning-effort values onto Harmony's enum. fn to_harmony_reasoning_effort( - reasoning_effort: ReasoningEffort, + reasoning_effort: Option<&EffortValue>, ) -> Result { - match reasoning_effort { - ReasoningEffort::Low => Ok(HarmonyReasoningEffort::Low), - ReasoningEffort::Medium => Ok(HarmonyReasoningEffort::Medium), - ReasoningEffort::High => Ok(HarmonyReasoningEffort::High), - ReasoningEffort::None - | ReasoningEffort::Minimal - | ReasoningEffort::XHigh - | ReasoningEffort::Max => Err(Error::ChatTemplate(format!( - "reasoning_effort={:?} is not supported by Harmony. Supported values are: low, medium, high.", - reasoning_effort.as_str() + match reasoning_effort.and_then(EffortValue::as_str) { + Some("low") => Ok(HarmonyReasoningEffort::Low), + Some("medium") => Ok(HarmonyReasoningEffort::Medium), + Some("high") => Ok(HarmonyReasoningEffort::High), + _ => Err(Error::InvalidReasoningEffort(format!( + "reasoning_effort={} is not supported by Harmony. Supported values are: low, medium, high.", + reasoning_effort.unwrap_or(&EffortValue::from("none")) ))), } } diff --git a/rust/src/chat/src/renderer/harmony/tests.rs b/rust/src/chat/src/renderer/harmony/tests.rs index 4849c93223..609c1f1159 100644 --- a/rust/src/chat/src/renderer/harmony/tests.rs +++ b/rust/src/chat/src/renderer/harmony/tests.rs @@ -9,12 +9,11 @@ use thiserror_ext::AsReport as _; use super::HarmonyChatRenderer; use super::encoding::harmony_encoding; use crate::ChatRenderer; +use crate::EffortValue; use crate::error::Error; use crate::event::{AssistantContentBlock, AssistantToolCall}; use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; -use crate::request::{ - ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort, -}; +use crate::request::{ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode}; const PINNED_DATE: &str = "2025-06-28"; @@ -22,7 +21,7 @@ fn fixture_request(input_name: &str) -> ChatRequest { fixture_chat_request( &fixture_path(input_name), FixtureRequestOptions { - enable_thinking: Some(false), + enable_thinking: None, no_generation_prompt_when_last_assistant: false, }, ) @@ -36,7 +35,8 @@ fn fixture_path(name: &str) -> PathBuf { } fn test_renderer(use_system_instructions: bool) -> HarmonyChatRenderer { - HarmonyChatRenderer::with_options(PINNED_DATE, use_system_instructions).unwrap() + HarmonyChatRenderer::with_options(Default::default(), PINNED_DATE, use_system_instructions) + .unwrap() } fn render_token_ids(request: &ChatRequest) -> Vec { @@ -138,14 +138,48 @@ fn drops_stale_analysis_fixture() { #[test] fn rejects_invalid_reasoning_effort() { let mut request = ChatRequest::for_test(); - request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + request.chat_options.reasoning_effort = Some(EffortValue::from("none")); let error = test_renderer(false).render(&request).unwrap_err(); - expect![[r#"chat template error: reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]] + expect![[r#"reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]] .assert_eq(&error.to_report_string()); } +#[test] +fn normalized_reasoning_respects_harmony_capability_and_deployment_effort() { + let renderer = HarmonyChatRenderer::with_options( + [("reasoning_effort".to_string(), serde_json::json!("low"))].into(), + PINNED_DATE, + false, + ) + .unwrap(); + let mut request = ChatRequest::for_test(); + request.chat_options.reasoning_effort = Some(EffortValue::from("none")); + request + .chat_options + .template_kwargs + .insert("enable_thinking".into(), serde_json::json!(true)); + let rendered = renderer.render(&request).unwrap(); + let prompt = harmony_encoding() + .unwrap() + .tokenizer() + .decode_utf8(rendered.prompt.into_token_ids().unwrap()) + .unwrap(); + assert!(prompt.contains("Reasoning: low")); + assert_eq!( + rendered.effective_template_kwargs["reasoning_effort"], + "low" + ); + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], true); + + request + .chat_options + .template_kwargs + .insert("thinking".into(), serde_json::json!(false)); + assert!(renderer.render(&request).is_err()); +} + #[test] fn rejects_unknown_tool_response_id() { let request = ChatRequest { diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index ba2b76d100..911a73928a 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -16,7 +16,7 @@ use self::format::{ ChatTemplateContentFormat, ChatTemplateContentFormatOption as ContentFormatOption, }; use self::template::{CompiledChatTemplate, TemplateContext}; -use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs}; +use super::{ChatRenderer, RenderedPrompt}; use crate::error::Result; use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest}; use crate::{ @@ -33,6 +33,31 @@ pub use template::{load_chat_template, resolve_chat_template}; pub use self::format::ChatTemplateContentFormatOption; +/// Extract the effective chat-template kwargs visible to the renderer from the request, +/// using the provided defaults as the base. +fn effective_template_kwargs( + default_template_kwargs: &HashMap, + request: &ChatRequest, +) -> HashMap { + let mut kwargs = default_template_kwargs.clone(); + kwargs.extend(request.chat_options.template_kwargs.clone()); + + if let Some(reasoning_effort) = &request.chat_options.reasoning_effort { + kwargs.insert( + "reasoning_effort".to_string(), + serde_json::json!(reasoning_effort), + ); + if !request.chat_options.template_kwargs.contains_key("enable_thinking") { + kwargs.insert( + "enable_thinking".to_string(), + serde_json::json!(reasoning_effort.as_str() != Some("none")), + ); + } + } + + kwargs +} + /// Template-visible placeholder tokens per supported modality. /// /// A `None` token means the loaded model does not support that modality, and @@ -580,9 +605,10 @@ mod tests { use vllm_text::backend::hf::{HfSpecialTokens, NamedSpecialToken}; use super::{ChatTemplateContentFormatOption, HfChatRenderer, MultimodalRenderInfo}; + use crate::EffortValue; use crate::request::{ ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, - GenerationPromptMode, ReasoningEffort, ResolvedToolContext, + GenerationPromptMode, ResolvedToolContext, }; use crate::{AssistantContentBlock, ChatRenderer, Error, Result}; @@ -1149,7 +1175,7 @@ mod tests { #[test] fn chat_template_reasoning_effort_overrides_template_kwargs() { let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]); - request.chat_options.reasoning_effort = Some(ReasoningEffort::Max); + request.chat_options.reasoning_effort = Some(EffortValue::from("max")); request.chat_options.template_kwargs.insert( "reasoning_effort".to_string(), Value::String("low".to_string()), @@ -1181,7 +1207,7 @@ mod tests { #[test] fn chat_template_reasoning_effort_preserves_request_enable_thinking() { let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]); - request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + request.chat_options.reasoning_effort = Some(EffortValue::from("none")); request .chat_options .template_kwargs @@ -1207,6 +1233,70 @@ mod tests { ); } + #[test] + fn chat_template_preserves_typed_effort_values() { + for effort in [ + serde_json::json!(37), + serde_json::json!(0.37), + serde_json::json!("custom"), + ] { + let mut request = sample_request(vec![ChatMessage::user("hello")]); + request.chat_options.reasoning_effort = + Some(serde_json::from_value(effort.clone()).unwrap()); + request.chat_options.template_kwargs = [ + ("reasoning_effort".to_string(), serde_json::json!("high")), + ("enable_thinking".to_string(), serde_json::json!(true)), + ] + .into(); + let rendered = HfChatRenderer::new( + Some("{{ reasoning_effort|tojson }}|{{ enable_thinking }}".to_string()), + HashMap::new(), + ChatTemplateContentFormatOption::Auto, + ) + .unwrap() + .render(&request) + .unwrap(); + assert_eq!(rendered.prompt, Prompt::Text(format!("{effort}|True"))); + assert_eq!( + rendered.effective_template_kwargs["reasoning_effort"], + effort + ); + } + } + + #[test] + fn chat_template_keeps_native_control_conflicts_and_raw_effort_values() { + for effort in [ + serde_json::json!(37), + serde_json::json!(null), + serde_json::json!("none"), + ] { + let mut request = sample_request(vec![ChatMessage::user("hello")]); + request.chat_options.template_kwargs = [ + ("thinking".to_string(), serde_json::json!("custom-mode")), + ("enable_thinking".to_string(), serde_json::json!(true)), + ("reasoning_effort".to_string(), effort), + ] + .into(); + let rendered = HfChatRenderer::new( + Some( + "{{ thinking }}|{{ enable_thinking }}|{{ reasoning_effort is none }}" + .to_string(), + ), + HashMap::new(), + ChatTemplateContentFormatOption::Auto, + ) + .unwrap() + .render(&request) + .unwrap(); + assert!(rendered.prompt.into_text().unwrap().starts_with("custom-mode|True|")); + assert_eq!( + rendered.effective_template_kwargs, + request.chat_options.template_kwargs + ); + } + } + #[test] fn qwen3_template_omits_reasoning_for_historical_assistant_messages() { let request = sample_request(vec![ diff --git a/rust/src/chat/src/renderer/inkling/mod.rs b/rust/src/chat/src/renderer/inkling/mod.rs index e8afe66396..9feee371aa 100644 --- a/rust/src/chat/src/renderer/inkling/mod.rs +++ b/rust/src/chat/src/renderer/inkling/mod.rs @@ -8,8 +8,10 @@ use thiserror_ext::AsReport as _; use vllm_text::Prompt; use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use super::{ChatRenderer, RenderedPrompt}; +use crate::EffortValue; use crate::error::{Error, Result}; +use crate::reasoning::ReasoningControl; use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool}; use crate::{AssistantContentBlock, AssistantToolCall}; @@ -35,6 +37,7 @@ const MAX_REASONING_EFFORT: f64 = 0.99; pub struct InklingChatRenderer { tokenizer: DynTokenizer, special: InklingSpecialTokenIds, + default_template_kwargs: HashMap, } #[derive(Debug, Clone, Copy)] @@ -78,9 +81,16 @@ impl InklingSpecialTokenIds { } impl InklingChatRenderer { - pub fn new(tokenizer: DynTokenizer) -> Result { + pub fn new( + tokenizer: DynTokenizer, + default_template_kwargs: HashMap, + ) -> Result { let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?; - Ok(Self { tokenizer, special }) + Ok(Self { + tokenizer, + special, + default_template_kwargs, + }) } fn write_text_tokens(&self, out: &mut Vec, text: &str) -> Result<()> { @@ -150,11 +160,6 @@ impl InklingChatRenderer { } fn write_reasoning_effort(&self, out: &mut Vec, effort: f64) -> Result<()> { - if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) { - return Err(Error::ChatTemplate(format!( - "Inkling reasoning_effort must be in [0.0, 0.99], got {effort}" - ))); - } let formatted = format!("{effort:.2}"); let effort = formatted.trim_end_matches('0').trim_end_matches('.'); let effort = if matches!(effort, "0" | "-0") { @@ -294,11 +299,18 @@ impl ChatRenderer for InklingChatRenderer { let mut out = Vec::new(); let mut tool_call_id_to_name = HashMap::new(); - let effective_template_kwargs = request_template_kwargs(request); + let reasoning = resolve_reasoning(ReasoningControl::resolve( + request, + &self.default_template_kwargs, + )?)?; + let effective_template_kwargs = reasoning.template_kwargs(request); let tools = rendered_tools(request); self.write_tool_declarations(&mut out, &tools)?; - let mut reasoning_effort = - resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort")); + let mut reasoning_effort = if reasoning.is_enabled() { + reasoning.effort().and_then(EffortValue::as_f64) + } else { + Some(0.0) + }; for message in &request.messages { if !matches!( @@ -352,12 +364,33 @@ impl ChatRenderer for InklingChatRenderer { } } -fn resolve_reasoning_effort(value: Option<&Value>) -> Option { +fn resolve_reasoning(control: ReasoningControl) -> Result { + let control = control.fallback(ReasoningControl::enabled("high")); + if !control.is_enabled() { + return Ok(control); + } + let effort = resolve_reasoning_effort(control.effort()); + if let Some(effort) = effort { + if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) { + return Err(Error::InvalidReasoningEffort(format!( + "Inkling reasoning_effort must be in [0.0, 0.99], got {effort}" + ))); + } + if effort == 0.0 { + return Ok(ReasoningControl::Disabled); + } + } + Ok(ReasoningControl::Enabled { + effort: effort.map(EffortValue::try_from).transpose()?, + }) +} + +fn resolve_reasoning_effort(value: Option<&EffortValue>) -> Option { let Some(value) = value else { return Some(0.9); }; match value { - Value::String(name) => match name.as_str() { + EffortValue::String(name) => match name.as_str() { "none" => Some(0.0), "minimal" => Some(0.1), "low" => Some(0.2), @@ -366,8 +399,7 @@ fn resolve_reasoning_effort(value: Option<&Value>) -> Option { "xhigh" | "max" => Some(0.99), _ => None, }, - Value::Number(number) => number.as_f64(), - _ => None, + EffortValue::Number(number) => number.as_f64(), } } diff --git a/rust/src/chat/src/renderer/inkling/tests.rs b/rust/src/chat/src/renderer/inkling/tests.rs index 1c4cd7236d..d0dd3055e4 100644 --- a/rust/src/chat/src/renderer/inkling/tests.rs +++ b/rust/src/chat/src/renderer/inkling/tests.rs @@ -9,6 +9,7 @@ use serde_json::json; use thiserror_ext::AsReport; use vllm_text::tokenizer::Tokenizer; +use crate::EffortValue; use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; use super::{ @@ -17,7 +18,7 @@ use super::{ MESSAGE_TOOL, MESSAGE_USER, }; use crate::event::{AssistantContentBlock, AssistantToolCall}; -use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort}; +use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode}; use crate::{ChatRenderer, Error}; struct FixtureTokenizer; @@ -87,7 +88,7 @@ impl Tokenizer for FixtureTokenizer { } fn renderer() -> InklingChatRenderer { - InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap() + InklingChatRenderer::new(Arc::new(FixtureTokenizer), Default::default()).unwrap() } fn render_token_ids(request: &ChatRequest) -> Vec { @@ -105,7 +106,7 @@ fn fixture_request(name: &str) -> ChatRequest { fn inkling_fixture_options() -> FixtureRequestOptions { FixtureRequestOptions { - enable_thinking: Some(false), + enable_thinking: None, no_generation_prompt_when_last_assistant: false, } } @@ -206,13 +207,13 @@ fn renders_request_and_developer_tools_as_tool_declare() { #[test] fn renders_named_reasoning_effort_after_tool_declarations() { for (effort, expected) in [ - (ReasoningEffort::None, "0.0"), - (ReasoningEffort::Minimal, "0.1"), - (ReasoningEffort::Low, "0.2"), - (ReasoningEffort::Medium, "0.7"), - (ReasoningEffort::High, "0.9"), - (ReasoningEffort::XHigh, "0.99"), - (ReasoningEffort::Max, "0.99"), + (EffortValue::from("none"), "0.0"), + (EffortValue::from("minimal"), "0.1"), + (EffortValue::from("low"), "0.2"), + (EffortValue::from("medium"), "0.7"), + (EffortValue::from("high"), "0.9"), + (EffortValue::from("xhigh"), "0.99"), + (EffortValue::from("max"), "0.99"), ] { let mut request = fixture_request("tool_declare_input.json"); request.chat_options.reasoning_effort = Some(effort); @@ -284,6 +285,43 @@ fn defaults_reasoning_effort_to_high() { ); } +#[test] +fn normalized_mode_and_scalar_are_reflected_in_prompt_and_kwargs() { + for (kwargs, expected, enabled) in [ + ( + json!({"enable_thinking": true, "reasoning_effort": "none"}), + "0.9", + true, + ), + ( + json!({"thinking": false, "reasoning_effort": 10}), + "0.0", + false, + ), + (json!({"reasoning_effort": 0}), "0.0", false), + (json!({"reasoning_effort": null}), "0.9", true), + (json!({"reasoning_effort": 0.37}), "0.37", true), + ] { + let mut request = ChatRequest::for_test(); + request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap(); + let rendered = renderer().render(&request).unwrap(); + let prompt = render_symbolic_tokens(&rendered.prompt.into_token_ids().unwrap()); + assert!(prompt.contains(&format!("Thinking effort level: {expected}<|end_message|>"))); + assert_eq!( + rendered.effective_template_kwargs["enable_thinking"], + enabled + ); + assert_eq!( + rendered.effective_template_kwargs["reasoning_effort"], + if enabled { + json!(expected.parse::().unwrap()) + } else { + json!("none") + } + ); + } +} + #[test] fn renders_numeric_reasoning_effort_template_kwarg() { let mut request = ChatRequest::for_test(); @@ -301,19 +339,17 @@ fn renders_numeric_reasoning_effort_template_kwarg() { } #[test] -fn ignores_unsupported_reasoning_effort_values() { - for value in [json!(true), json!("invalid"), json!(null)] { - let mut request = ChatRequest::for_test(); - request - .chat_options - .template_kwargs - .insert("reasoning_effort".to_string(), value); +fn ignores_unsupported_reasoning_effort_names() { + let mut request = ChatRequest::for_test(); + request + .chat_options + .template_kwargs + .insert("reasoning_effort".to_string(), json!("invalid")); - assert_eq!( - render_symbolic_tokens(&render_token_ids(&request)), - "<|message_user|><|content_text|>test<|end_message|><|message_model|>" - ); - } + assert_eq!( + render_symbolic_tokens(&render_token_ids(&request)), + "<|message_user|><|content_text|>test<|end_message|><|message_model|>" + ); } #[test] diff --git a/rust/src/chat/src/renderer/kimi_k3/encoding.rs b/rust/src/chat/src/renderer/kimi_k3/encoding.rs index 88b4e3ae61..b434d72c53 100644 --- a/rust/src/chat/src/renderer/kimi_k3/encoding.rs +++ b/rust/src/chat/src/renderer/kimi_k3/encoding.rs @@ -11,7 +11,9 @@ use serde_json::{Map, Value, json}; use vllm_tokenizer::Tokenizer; use super::super::MediaPartSource; +use crate::EffortValue; use crate::error::{Error, Result}; +use crate::reasoning::ReasoningControl; use crate::request::{ ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice, }; @@ -76,9 +78,10 @@ impl<'a> K3TokenWriter<'a> { pub(super) fn render_request_with_media_order( request: &ChatRequest, tokenizer: &dyn Tokenizer, + reasoning: &ReasoningControl, ) -> Result<(Vec, Vec)> { - let thinking = thinking_enabled(request)?; - let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?; + let thinking = reasoning.is_enabled(); + let thinking_effort = reasoning.effort().and_then(EffortValue::as_str); let tools = request_tools(request); let mut out = K3TokenWriter::new(tokenizer); @@ -244,58 +247,26 @@ fn request_tools(request: &ChatRequest) -> &[ChatTool] { request.initial_tools() } -fn thinking_enabled(request: &ChatRequest) -> Result { - if let Some(thinking) = request.parse_template_bool("thinking")? { - return Ok(thinking); - } - if let Some(enable_thinking) = request.parse_template_bool("enable_thinking")? { - return Ok(enable_thinking); - } - if let Some(reasoning_effort) = request.chat_options.reasoning_effort { - return Ok(reasoning_effort != crate::request::ReasoningEffort::None); - } - Ok(request - .chat_options - .template_kwargs - .get("reasoning_effort") - .and_then(Value::as_str) - != Some("none")) -} - -fn thinking_effort(request: &ChatRequest) -> Result { - let effort = if let Some(value) = request.chat_options.template_kwargs.get("thinking_effort") { - value.as_str().ok_or_else(|| { - Error::ChatTemplate(format!( - "template kwarg `thinking_effort` must be a string, got {value}" - )) - })? - } else if let Some(effort) = request.chat_options.reasoning_effort { - if effort == crate::request::ReasoningEffort::None { - DEFAULT_THINKING_EFFORT - } else { - effort.as_str() - } - } else if let Some(value) = request.chat_options.template_kwargs.get("reasoning_effort") { +/// Resolve standard reasoning controls and validate K3's supported effort grades. +pub(super) fn resolve_reasoning( + request: &ChatRequest, + defaults: &HashMap, +) -> Result { + let reasoning = ReasoningControl::resolve(request, defaults)? + .fallback(ReasoningControl::enabled(DEFAULT_THINKING_EFFORT)); + if let Some(value) = reasoning.effort() { let effort = value.as_str().ok_or_else(|| { - Error::ChatTemplate(format!( - "template kwarg `reasoning_effort` must be a string, got {value}" + Error::InvalidReasoningEffort(format!( + "Kimi K3 reasoning_effort must be a string, got {value}" )) })?; - if effort == "none" { - DEFAULT_THINKING_EFFORT - } else { - effort + if !VALID_THINKING_EFFORTS.contains(&effort) { + return Err(Error::InvalidReasoningEffort(format!( + "unsupported reasoning_effort={effort:?}; supported values are `low`, `high`, and `max`" + ))); } - } else { - DEFAULT_THINKING_EFFORT - }; - - if !VALID_THINKING_EFFORTS.contains(&effort) { - return Err(Error::ChatTemplate(format!( - "unsupported thinking_effort={effort:?}; supported values are `low`, `high`, and `max`" - ))); } - Ok(effort.to_string()) + Ok(reasoning) } fn content_is_empty(content: &ChatContent) -> bool { diff --git a/rust/src/chat/src/renderer/kimi_k3/mod.rs b/rust/src/chat/src/renderer/kimi_k3/mod.rs index 90750eeac4..c39ea2985a 100644 --- a/rust/src/chat/src/renderer/kimi_k3/mod.rs +++ b/rust/src/chat/src/renderer/kimi_k3/mod.rs @@ -7,10 +7,13 @@ mod encoding; #[cfg(test)] mod tests; +use std::collections::HashMap; + +use serde_json::Value; use vllm_text::Prompt; use vllm_text::tokenizer::DynTokenizer; -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use super::{ChatRenderer, RenderedPrompt}; use crate::Result; use crate::request::ChatRequest; @@ -18,25 +21,42 @@ use crate::request::ChatRequest; #[derive(Clone)] pub struct KimiK3ChatRenderer { tokenizer: DynTokenizer, + default_template_kwargs: HashMap, } impl KimiK3ChatRenderer { /// Create a Kimi K3 renderer. - pub fn new(tokenizer: DynTokenizer) -> Self { - Self { tokenizer } + pub fn new(tokenizer: DynTokenizer, default_template_kwargs: HashMap) -> Self { + Self { + tokenizer, + default_template_kwargs, + } } } impl ChatRenderer for KimiK3ChatRenderer { fn render(&self, request: &ChatRequest) -> Result { request.validate()?; - let (token_ids, media_order) = - encoding::render_request_with_media_order(request, self.tokenizer.as_ref())?; + let reasoning = encoding::resolve_reasoning(request, &self.default_template_kwargs)?; + let mut effective_template_kwargs = reasoning.template_kwargs(request); + // Export K3's native effort key from the resolved standard controls. + // Input `thinking_effort` is ignored; clear it when reasoning is disabled. + if let Some(effort) = reasoning.effort() { + effective_template_kwargs + .insert("thinking_effort".to_string(), serde_json::json!(effort)); + } else { + effective_template_kwargs.remove("thinking_effort"); + } + let (token_ids, media_order) = encoding::render_request_with_media_order( + request, + self.tokenizer.as_ref(), + &reasoning, + )?; Ok(RenderedPrompt { prompt: Prompt::TokenIds(token_ids), media_order: Some(media_order), - effective_template_kwargs: request_template_kwargs(request), + effective_template_kwargs, }) } } diff --git a/rust/src/chat/src/renderer/kimi_k3/tests.rs b/rust/src/chat/src/renderer/kimi_k3/tests.rs index d0ecbe44d3..5aadb3d403 100644 --- a/rust/src/chat/src/renderer/kimi_k3/tests.rs +++ b/rust/src/chat/src/renderer/kimi_k3/tests.rs @@ -15,11 +15,10 @@ use vllm_tokenizer::test_utils::TestTokenizer; use super::KimiK3ChatRenderer; use crate::ChatRenderer; +use crate::EffortValue; use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP}; use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; -use crate::request::{ - ChatContentPart, ChatMessage, ChatTool, GenerationPromptMode, ReasoningEffort, -}; +use crate::request::{ChatContentPart, ChatMessage, ChatTool, GenerationPromptMode}; use crate::{AssistantContentBlock, AssistantToolCall}; const OPEN_ID: u32 = 256; @@ -37,8 +36,70 @@ fn test_tokenizer() -> TestTokenizer { .with_special_token(IMAGE_PLACEHOLDER, MEDIA_ID) } +#[test] +fn standard_effort_controls_prompt_and_exported_native_effort() { + let tokenizer = Arc::new(test_tokenizer()); + for (kwargs, typed, defaults, effort) in [ + (json!({}), None, json!({}), "max"), + ( + json!({}), + Some(EffortValue::from("low")), + json!({"reasoning_effort": "high"}), + "low", + ), + ( + json!({"enable_thinking": true, "reasoning_effort": "none"}), + None, + json!({"reasoning_effort": "low"}), + "low", + ), + ( + json!({}), + Some(EffortValue::from("none")), + json!({"reasoning_effort": "low"}), + "none", + ), + ( + json!({"thinking": true}), + Some(EffortValue::from("none")), + json!({}), + "max", + ), + ] { + let renderer = + KimiK3ChatRenderer::new(tokenizer.clone(), serde_json::from_value(defaults).unwrap()); + let mut request = crate::ChatRequest::for_test(); + request.chat_options.reasoning_effort = typed; + request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap(); + let rendered = renderer.render(&request).unwrap(); + let prompt = tokenizer.decode(&rendered.prompt.into_token_ids().unwrap(), false).unwrap(); + let enabled = effort != "none"; + assert_eq!(prompt.ends_with("<|open|>think<|sep|>"), enabled); + assert_eq!( + rendered.effective_template_kwargs["reasoning_effort"], + effort + ); + assert_eq!( + rendered.effective_template_kwargs["enable_thinking"], + enabled + ); + if enabled { + assert!(prompt.contains(&format!("thinking_effort={effort}"))); + assert_eq!( + rendered.effective_template_kwargs["thinking_effort"], + effort + ); + } else { + assert!(!rendered.effective_template_kwargs.contains_key("thinking_effort")); + } + } +} + fn render_token_ids(request: &crate::request::ChatRequest, tokenizer: DynTokenizer) -> Vec { - let prompt = KimiK3ChatRenderer::new(tokenizer).render(request).unwrap().prompt; + let prompt = KimiK3ChatRenderer::new(tokenizer, Default::default()) + .render(request) + .unwrap() + .prompt; let Prompt::TokenIds(token_ids) = prompt else { panic!("kimi k3 renderer should return token IDs") }; @@ -67,7 +128,11 @@ fn kimi_k3_fixture_options() -> FixtureRequestOptions { fn assert_golden(name: &str) { let input_name = format!("{name}_input.json"); - let request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options()); + let mut request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options()); + // Reference fixtures use encoder-native keys; Rust accepts standard effort input. + if let Some(effort) = request.chat_options.template_kwargs.remove("thinking_effort") { + request.chat_options.reasoning_effort = Some(EffortValue::try_from(effort).unwrap()); + } let rendered = render_request(&request); expect_file![format!("fixtures/{name}_output.txt")].assert_eq(&rendered); } @@ -310,7 +375,7 @@ fn media_order_follows_reordered_tool_results() { ), ]; request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt; - let renderer = KimiK3ChatRenderer::new(Arc::new(test_tokenizer())); + let renderer = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()), Default::default()); let rendered = renderer.render(&request).unwrap(); let media_order = rendered.media_order.unwrap(); @@ -371,19 +436,18 @@ fn translates_standard_thinking_kwargs() { } #[test] -fn native_k3_kwargs_take_precedence() { +fn thinking_alias_takes_precedence() { let mut request = crate::request::ChatRequest::for_test(); request.chat_options.template_kwargs.extend([ ("thinking".to_string(), json!(true)), ("enable_thinking".to_string(), json!(false)), - ("thinking_effort".to_string(), json!("low")), ("reasoning_effort".to_string(), json!("high")), ]); let rendered = render_request(&request); - assert!(rendered.contains("thinking_effort=low")); - assert!(!rendered.contains("thinking_effort=high")); + assert!(rendered.contains("thinking_effort=high")); + assert!(rendered.ends_with("<|open|>think<|sep|>")); } #[test] @@ -431,7 +495,7 @@ fn enable_thinking_true_overrides_standard_none() { #[test] fn typed_none_disables_thinking() { let mut request = crate::request::ChatRequest::for_test(); - request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + request.chat_options.reasoning_effort = Some(EffortValue::from("none")); let rendered = render_request(&request); @@ -440,20 +504,17 @@ fn typed_none_disables_thinking() { } #[test] -fn rejects_removed_medium_thinking_effort() { +fn rejects_unsupported_standard_reasoning_effort() { let mut request = crate::request::ChatRequest::for_test(); - request - .chat_options - .template_kwargs - .insert("thinking_effort".to_string(), json!("medium")); + request.chat_options.reasoning_effort = Some(EffortValue::from("medium")); - let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer())) + let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()), Default::default()) .render(&request) .unwrap_err(); expect![[r#" - ChatTemplate( - "unsupported thinking_effort=\"medium\"; supported values are `low`, `high`, and `max`", + InvalidReasoningEffort( + "unsupported reasoning_effort=\"medium\"; supported values are `low`, `high`, and `max`", ) "#]] .assert_debug_eq(&error); diff --git a/rust/src/chat/src/renderer/mod.rs b/rust/src/chat/src/renderer/mod.rs index bc1831c375..347ec43d03 100644 --- a/rust/src/chat/src/renderer/mod.rs +++ b/rust/src/chat/src/renderer/mod.rs @@ -4,11 +4,11 @@ use std::collections::HashMap; use std::sync::Arc; -use serde_json::{Value, json}; +use serde_json::Value; use vllm_text::Prompt; use crate::error::Result; -use crate::request::{ChatRequest, ReasoningEffort}; +use crate::request::ChatRequest; mod deepseek; pub mod deepseek_v32; @@ -62,33 +62,3 @@ pub trait ChatRenderer: Send + Sync { /// Shared trait-object form of [`ChatRenderer`]. pub type DynChatRenderer = Arc; - -/// Extract the effective chat-template kwargs visible to the renderer from the request, -/// using the provided defaults as the base. -pub(crate) fn effective_template_kwargs( - default_template_kwargs: &HashMap, - request: &ChatRequest, -) -> HashMap { - let mut kwargs = default_template_kwargs.clone(); - kwargs.extend(request.chat_options.template_kwargs.clone()); - - if let Some(reasoning_effort) = request.chat_options.reasoning_effort { - kwargs.insert( - "reasoning_effort".to_string(), - Value::String(reasoning_effort.as_str().to_string()), - ); - if !request.chat_options.template_kwargs.contains_key("enable_thinking") { - kwargs.insert( - "enable_thinking".to_string(), - json!(reasoning_effort != ReasoningEffort::None), - ); - } - } - - kwargs -} - -/// Extract the effective chat-template kwargs visible to the renderer from the request. -pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap { - effective_template_kwargs(&HashMap::new(), request) -} diff --git a/rust/src/chat/src/renderer/test_utils.rs b/rust/src/chat/src/renderer/test_utils.rs index 1090a4039f..2c59ed7a22 100644 --- a/rust/src/chat/src/renderer/test_utils.rs +++ b/rust/src/chat/src/renderer/test_utils.rs @@ -8,10 +8,11 @@ use std::path::Path; use serde::Deserialize; use serde_json::Value; +use crate::EffortValue; use crate::event::{AssistantContentBlock, AssistantToolCall}; use crate::request::{ ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice, - GenerationPromptMode, ReasoningEffort, ResolvedToolContext, + GenerationPromptMode, ResolvedToolContext, }; /// Options for constructing a [`ChatRequest`] from a fixture file. @@ -47,7 +48,7 @@ pub(crate) struct FixtureRequest { tools: Vec, messages: Vec, add_generation_prompt: Option, - reasoning_effort: Option, + reasoning_effort: Option, /// Standard response format passed to model-specific renderers. #[serde(default)] response_format: Option, diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index 50e00bbc43..e3bbcaa2aa 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -11,9 +11,9 @@ pub use vllm_parser::tool::Tool as ChatTool; use vllm_text::TextDecodeOptions; pub use vllm_text::{PromptTruncation, SamplingParams}; -use crate::AssistantMessageExt; use crate::error::{Error, Result}; use crate::event::{AssistantContentBlock, AssistantMessage}; +use crate::{AssistantMessageExt, EffortValue}; /// Role label for one text-only chat message. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -362,33 +362,6 @@ pub enum GenerationPromptMode { NoGenerationPrompt, } -/// Effort level for reasoning models. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ReasoningEffort { - None, - Minimal, - Low, - Medium, - High, - XHigh, - Max, -} - -impl ReasoningEffort { - pub fn as_str(self) -> &'static str { - match self { - Self::None => "none", - Self::Minimal => "minimal", - Self::Low => "low", - Self::Medium => "medium", - Self::High => "high", - Self::XHigh => "xhigh", - Self::Max => "max", - } - } -} - /// Chat-template-related request options. /// /// These are the small subset of chat controls that currently affect prompt @@ -404,8 +377,10 @@ pub struct ChatOptions { /// used instead of the model's default chat template. pub chat_template: Option, - /// Effort level exposed to chat templates for reasoning models. - pub reasoning_effort: Option, + /// Model-specific reasoning effort, typically `none`, `minimal`, `low`, + /// `medium`, `high`, `xhigh`, or `max`. Supported names and numeric ranges + /// are validated by the selected renderer or HF template. + pub reasoning_effort: Option, /// Standard response format available to model-specific renderers. #[serde(default)] @@ -673,29 +648,6 @@ impl ChatRequest { self.tool_context.parsing_enabled() } - /// Return the request-level thinking toggle when explicitly requested. - /// - /// We currently accept the two request kwargs `thinking` and - /// `enable_thinking`. Both must be booleans when present. If both are - /// present, they must have the same value. If neither key is provided, - /// return `None`. - pub(crate) fn enable_thinking(&self) -> Result> { - let thinking = self.parse_template_bool("thinking")?; - let enable_thinking = self.parse_template_bool("enable_thinking")?; - - match (thinking, enable_thinking) { - (None, None) => Ok(None), - (Some(thinking), Some(enable_thinking)) if thinking != enable_thinking => { - Err(Error::ChatTemplate( - "template kwargs `thinking` and `enable_thinking` must match when both are set" - .to_string(), - )) - } - (Some(thinking), _) => Ok(Some(thinking)), - (None, Some(enable_thinking)) => Ok(Some(enable_thinking)), - } - } - pub(crate) fn parse_template_bool(&self, key: &str) -> Result> { match self.chat_options.template_kwargs.get(key) { None => Ok(None), @@ -726,7 +678,7 @@ mod tests { use serde_json::{json, to_value}; use super::{ - ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, + ChatContent, ChatContentPart, ChatMessage, ChatRole, ChatTool, ChatToolChoice, ResolvedToolContext, }; use crate::Error; @@ -919,56 +871,4 @@ mod tests { Error::ToolChoiceFunctionNotFound { name } if name == "missing" )); } - - #[test] - fn enable_thinking_is_none_when_no_kwargs_are_present() { - let request = ChatRequest::for_test(); - assert_eq!(request.enable_thinking().unwrap(), None); - } - - #[test] - fn enable_thinking_accepts_matching_duplicate_kwargs() { - let mut request = ChatRequest::for_test(); - request.chat_options.template_kwargs.insert("thinking".to_string(), json!(true)); - request - .chat_options - .template_kwargs - .insert("enable_thinking".to_string(), json!(true)); - - assert_eq!(request.enable_thinking().unwrap(), Some(true)); - } - - #[test] - fn enable_thinking_rejects_non_boolean_kwargs() { - let mut request = ChatRequest::for_test(); - request - .chat_options - .template_kwargs - .insert("thinking".to_string(), json!("yes")); - - assert!(matches!( - request.enable_thinking(), - Err(Error::ChatTemplate(message)) - if message.contains("`thinking` must be a boolean") - )); - } - - #[test] - fn enable_thinking_rejects_conflicting_duplicate_kwargs() { - let mut request = ChatRequest::for_test(); - request - .chat_options - .template_kwargs - .insert("thinking".to_string(), json!(false)); - request - .chat_options - .template_kwargs - .insert("enable_thinking".to_string(), json!(true)); - - assert!(matches!( - request.enable_thinking(), - Err(Error::ChatTemplate(message)) - if message.contains("`thinking` and `enable_thinking` must match") - )); - } } diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index 9f083a1a61..3280cd1dd5 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -175,16 +175,22 @@ mod tests { } #[test] - fn invalid_reasoning_effort_maps_to_invalid_request() { - let error = vllm_chat::Error::InvalidReasoningEffort( - "DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(), - ); - let api_error = chat_submit_error("failed to submit chat request", error); - assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); - assert_eq!( - api_error.to_error_response().error.error_type, - "invalid_request_error" - ); + fn invalid_reasoning_parameters_map_to_invalid_request() { + for error in [ + vllm_chat::Error::InvalidReasoningEffort( + "DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(), + ), + vllm_chat::Error::InvalidReasoningControl { + message: "template kwarg `thinking` must be a boolean".to_string(), + }, + ] { + let api_error = chat_submit_error("failed to submit chat request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + assert_eq!( + api_error.to_error_response().error.error_type, + "invalid_request_error" + ); + } } #[test] diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 51b0fb68ad..1815856636 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -180,7 +180,7 @@ pub(super) fn prepare_chat_request( chat_options: ChatOptions { generation_prompt_mode, chat_template: request.chat_template, - reasoning_effort: request.reasoning_effort, + reasoning_effort: request.reasoning_effort.map(|effort| effort.as_str().into()), response_format, template_kwargs, }, @@ -488,6 +488,50 @@ mod tests { } } + #[test] + fn chat_http_reasoning_effort_preserves_omission_none_and_kwargs() { + for effort in [None, Some(json!(null))].into_iter().chain( + ["none", "minimal", "low", "medium", "high", "xhigh", "max"] + .map(|name| Some(json!(name))), + ) { + let kwargs = json!({"reasoning_effort": 37, "thinking": true}); + let mut value = json!({ + "messages": [{"role": "user", "content": "hello"}], + "chat_template_kwargs": kwargs, + }); + if let Some(effort) = &effort { + value["reasoning_effort"] = effort.clone(); + } + let request: ChatCompletionRequest = serde_json::from_value(value).unwrap(); + let prepared = prepare_chat_request( + request, + &served(&["test-model"]), + request_context(&HeaderMap::new(), None), + ) + .unwrap(); + let options = prepared.chat_request.chat_options; + assert_eq!( + serde_json::to_value(options.reasoning_effort).unwrap(), + effort.unwrap_or(serde_json::Value::Null) + ); + assert_eq!( + serde_json::to_value(options.template_kwargs).unwrap(), + kwargs + ); + } + } + + #[test] + fn chat_http_reasoning_effort_rejects_model_extensions_at_top_level() { + for effort in [json!(37), json!("custom")] { + let request = json!({ + "messages": [{"role": "user", "content": "hello"}], + "reasoning_effort": effort, + }); + assert!(serde_json::from_value::(request).is_err()); + } + } + #[test] fn chat_http_request_defaults_missing_or_null_model() { for model in [None, Some(serde_json::Value::Null)] { @@ -747,7 +791,7 @@ mod tests { ); let tokenizer = Arc::new(TestTokenizer::new()); - let prompt = KimiK3ChatRenderer::new(tokenizer.clone()) + let prompt = KimiK3ChatRenderer::new(tokenizer.clone(), Default::default()) .render(&prepared.chat_request) .expect("Kimi K3 rendering succeeds") .prompt; diff --git a/rust/src/server/src/routes/openai/chat_completions/types.rs b/rust/src/server/src/routes/openai/chat_completions/types.rs index a371462694..13c7c7233f 100644 --- a/rust/src/server/src/routes/openai/chat_completions/types.rs +++ b/rust/src/server/src/routes/openai/chat_completions/types.rs @@ -9,15 +9,15 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use serde_with::SerializeDisplay; use validator::Validate; -use vllm_chat::ReasoningEffort; use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams; use vllm_text::TruncationSide; use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::types::{ - ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope, - StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, - deserialize_request_top_k, validate_messages, validate_stop, validate_top_p_value, + ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, ReasoningEffort, StreamOptions, + StreamResponseEnvelope, StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage, + default_true, deserialize_request_top_k, validate_messages, validate_stop, + validate_top_p_value, }; /// vLLM-compatible request type for the Chat Completions API. diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index 7260155799..1bfc1d9a09 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -125,8 +125,8 @@ fn validate_function_tools(tools: &[Tool], param: &'static str) -> Result<(), Ap mod tests { use std::collections::HashMap; + use crate::routes::openai::utils::types::ReasoningEffort; use serde_json::json; - use vllm_chat::ReasoningEffort; use super::validate_request_compat; use crate::routes::openai::chat_completions::types::ChatCompletionRequest; diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index e9aab2d2b2..5554b5d366 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -39,6 +39,36 @@ where } } +/// Effort level for reasoning models. +/// +/// Fixed OpenAI HTTP request grades. Request conversion maps these names to +/// the model-independent `vllm_chat::EffortValue` before renderer validation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningEffort { + None, + Minimal, + Low, + Medium, + High, + XHigh, + Max, +} + +impl ReasoningEffort { + pub fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Minimal => "minimal", + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + Self::XHigh => "xhigh", + Self::Max => "max", + } + } +} + // ============================================================================ // String/Array Utilities // ============================================================================