iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Normalize native renderer reasoning controls (#56998)

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-16 18:15:05 -07:00
committed by GitHub
parent 3e267bae70
commit e52da038d1
29 changed files with 1253 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/chat/src/backend/hf.rs
+53 -6
View File
@@ -70,12 +70,26 @@ impl HfChatBackend {
options,
multimodal_render_info,
)?),
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()),
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new(
options.default_chat_template_kwargs,
)),
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new(
options.default_chat_template_kwargs,
)),
RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new(
options.default_chat_template_kwargs,
)),
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new(
options.default_chat_template_kwargs,
)?),
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(
tokenizer.clone(),
options.default_chat_template_kwargs,
)?),
RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(
tokenizer.clone(),
options.default_chat_template_kwargs,
)),
};
info!(
@@ -280,6 +294,39 @@ mod tests {
assert_eq!(prompt, "hello");
}
#[test]
fn native_renderer_inherits_deployment_reasoning_below_request_controls() {
let backend = HfChatBackend::from_resolved_model_files(
resolved_files(r#"{"model_type":"deepseek_v4"}"#, "{}"),
"test-model".to_string(),
LoadModelBackendsOptions {
default_chat_template_kwargs: [
("thinking".to_string(), serde_json::json!(false)),
("reasoning_effort".to_string(), serde_json::json!("low")),
]
.into(),
..Default::default()
},
test_tokenizer(),
)
.unwrap();
let mut request = request_with_user_text("hello");
let rendered = backend.chat_renderer().render(&request).unwrap();
assert!(rendered.prompt.into_text().unwrap().ends_with("</think>"));
assert_eq!(rendered.effective_template_kwargs["enable_thinking"], false);
request.chat_options.reasoning_effort = Some(crate::EffortValue::from("max"));
let rendered = backend.chat_renderer().render(&request).unwrap();
let prompt = rendered.prompt.into_text().unwrap();
assert!(prompt.contains("Reasoning Effort: Beyond maximum"));
assert!(prompt.ends_with("<think>"));
assert_eq!(
rendered.effective_template_kwargs["reasoning_effort"],
"max"
);
assert_eq!(rendered.effective_template_kwargs["thinking"], true);
}
#[test]
fn auto_uses_harmony_renderer_and_output_processor_for_gpt_oss_model_type() {
let backend = backend_for_selection(
rust/src/chat/src/backend/mod.rs
+2 -2
View File
@@ -78,8 +78,8 @@ pub struct LoadModelBackendsOptions {
/// Optional server-default chat template override, provided either as an
/// inline template or as a path to a template file.
pub chat_template: Option<String>,
/// Optional server-default keyword arguments merged into every
/// chat-template render before request-level `chat_template_kwargs`.
/// Server-default keyword arguments. HF merges these before request kwargs;
/// native renderers inherit supported reasoning controls below request controls.
pub default_chat_template_kwargs: HashMap<String, Value>,
/// Maximum number of input items allowed per prompt for each modality.
/// Unspecified modalities are unlimited.
rust/src/chat/src/error.rs
+3
View File
@@ -19,6 +19,8 @@ pub enum Error {
ChatTemplate(String),
#[error("{0}")]
InvalidReasoningEffort(String),
#[error("{message}")]
InvalidReasoningControl { message: String },
#[error("multimodal input is not supported by this chat renderer")]
UnsupportedMultimodalRenderer,
#[error("unsupported multimodal content: {0}")]
@@ -101,6 +103,7 @@ impl Error {
match self {
Self::PromptTooLong { .. }
| Self::InvalidReasoningEffort(_)
| Self::InvalidReasoningControl { .. }
| Self::DuplicateToolName { .. }
| Self::ToolChoiceRequiresTools
| Self::ToolChoiceFunctionNotFound { .. } => true,
rust/src/chat/src/lib.rs
+3 -1
View File
@@ -31,6 +31,7 @@ pub use parser::reasoning::{
};
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use parser::{ParserSelection, validate_parser_overrides};
pub use reasoning::EffortValue;
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
@@ -39,7 +40,7 @@ pub use renderer::{
};
pub use request::{
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
ChatToolChoice, GenerationPromptMode, ReasoningEffort, ResolvedToolContext, SamplingParams,
ChatToolChoice, GenerationPromptMode, ResolvedToolContext, SamplingParams,
};
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
@@ -52,6 +53,7 @@ mod event;
pub mod multimodal;
mod output;
mod parser;
mod reasoning;
mod renderer;
mod request;
mod stream;
rust/src/chat/src/reasoning.rs
+397
View File
@@ -0,0 +1,397 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Shared reasoning-control resolution for native renderers.
//!
//! Each input source is parsed independently into [`ReasoningControl`]. Sources
//! then fall back in priority order: request, deployment defaults, model default.
//! Within a source, an explicit toggle decides the mode and effort supplies its
//! intensity. Keeping sources separate lets request effort override a deployment
//! toggle, and lets an enabled request inherit effort from enabled defaults.
//!
//! This module owns input types, precedence, and the standard `"none"` sentinel.
//! Model adapters own supported effort names, numeric ranges, and prompt encoding.
//! After model lowering, [`ReasoningControl::template_kwargs`] projects the prompt's
//! decision for downstream consumers. HF prepares raw template inputs separately.
use std::collections::HashMap;
use std::fmt;
use serde::{Deserialize, Serialize};
use serde_json::{Number, Value, json};
use crate::error::{Result, invalid_reasoning_control};
use crate::request::ChatRequest;
/// A model-specific reasoning effort, before or after renderer lowering.
///
/// Strings preserve model-specific names; JSON numbers preserve integer and
/// floating-point representations for model-local range validation. Missing/null
/// effort is represented by `Option::None` in [`crate::ChatOptions`].
/// Supported names and ranges are validated by each renderer or HF template.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum EffortValue {
String(String),
Number(Number),
}
impl TryFrom<Value> for EffortValue {
type Error = crate::error::Error;
fn try_from(value: Value) -> Result<Self> {
match value {
Value::String(value) => Ok(Self::String(value)),
Value::Number(value) => Ok(Self::Number(value)),
_ => Err(invalid_reasoning_control!(
"reasoning effort must be a string or number, got {value}"
)),
}
}
}
impl TryFrom<f64> for EffortValue {
type Error = crate::error::Error;
/// Construct a numeric effort, rejecting NaN and infinities.
fn try_from(value: f64) -> Result<Self> {
Number::from_f64(value).map(Self::Number).ok_or_else(|| {
invalid_reasoning_control!("reasoning effort must be finite, got {value}")
})
}
}
impl From<&str> for EffortValue {
fn from(value: &str) -> Self {
Self::String(value.to_owned())
}
}
impl fmt::Display for EffortValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::String(value) => json!(value).fmt(f),
Self::Number(value) => value.fmt(f),
}
}
}
impl EffortValue {
pub fn as_str(&self) -> Option<&str> {
match self {
Self::String(value) => Some(value),
Self::Number(_) => None,
}
}
pub fn as_f64(&self) -> Option<f64> {
match self {
Self::Number(value) => value.as_f64(),
Self::String(_) => None,
}
}
}
/// Reasoning intent for one source, or the result of falling back across sources.
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum ReasoningControl {
/// This source leaves the mode and effort to a lower-priority source.
Default,
/// Reasoning is explicitly disabled; lower-priority defaults preserve this mode.
Disabled,
/// Reasoning is enabled. Missing effort inherits from an enabled default.
Enabled { effort: Option<EffortValue> },
}
impl ReasoningControl {
/// Parse request controls; the typed effort field takes precedence over kwargs effort.
pub(crate) fn from_request(request: &ChatRequest) -> Result<Self> {
Self::from_kwargs(
&request.chat_options.template_kwargs,
request.chat_options.reasoning_effort.as_ref(),
)
}
/// Parse a deployment source, whose controls all come from template kwargs.
pub(crate) fn from_template_kwargs(kwargs: &HashMap<String, Value>) -> Result<Self> {
Self::from_kwargs(kwargs, None)
}
/// Select controls within one source, then derive its mode and optional effort.
fn from_kwargs(
kwargs: &HashMap<String, Value>,
typed_effort: Option<&EffortValue>,
) -> Result<Self> {
// Presence sets alias priority: `thinking` wins, including when its value
// is invalid. Validate the selected spelling as a boolean.
let toggle = ["thinking", "enable_thinking"]
.into_iter()
.find_map(|key| kwargs.get(key).map(|value| (key, value)))
.map(|(key, value)| {
value.as_bool().ok_or_else(|| {
invalid_reasoning_control!(
"template kwarg `{key}` must be a boolean, got {value}"
)
})
})
.transpose()?;
// The typed request field wins over kwargs. Parse kwargs only when used;
// disabled mode discards effort, and JSON null means omitted effort.
let effort = match (toggle, typed_effort) {
(Some(false), _) => None,
(_, Some(effort)) => Some(effort.clone()),
(_, None) => kwargs
.get("reasoning_effort")
.filter(|effort| !effort.is_null())
.cloned()
.map(EffortValue::try_from)
.transpose()?,
};
match (toggle, effort) {
// 1. Explicit false disables reasoning and discards any effort,
// including values that would fail validation in an enabled mode.
(Some(false), _) => Ok(Self::Disabled),
// 2. `none` disables reasoning when the toggle is absent. Explicit true
// enables reasoning and leaves effort to enabled deployment/model defaults.
(toggle, Some(EffortValue::String(effort))) if effort == "none" => {
Ok(if toggle == Some(true) {
Self::Enabled { effort: None }
} else {
Self::Disabled
})
}
// 3. Absent toggle and absent/null effort leave the entire decision
// to the next source in the fallback chain.
(None, None) => Ok(Self::Default),
// 4. Explicit true or a concrete effort enables reasoning;
// missing effort inherits later.
(_, effort) => Ok(Self::Enabled { effort }),
}
}
/// Parse request and deployment independently, then apply request precedence.
///
/// Each source validates its selected input types. Model adapters subsequently
/// apply their model default and validate/map the resulting effective effort.
pub(crate) fn resolve(
request: &ChatRequest,
defaults: &HashMap<String, Value>,
) -> Result<Self> {
Ok(Self::from_request(request)?.fallback(Self::from_template_kwargs(defaults)?))
}
/// Fill this higher-priority source's missing decision or effort from `defaults`.
///
/// `Default` inherits the whole lower-priority state. `Enabled { effort: None }`
/// inherits effort only from enabled defaults and retains its enabled mode.
/// Explicit disabling and concrete effort are final at this priority level.
pub(crate) fn fallback(self, defaults: Self) -> Self {
match (self, defaults) {
(Self::Default, defaults) => defaults,
(Self::Enabled { effort: None }, Self::Enabled { effort }) => Self::Enabled { effort },
(request, _) => request,
}
}
/// Construct an enabled model default or a model-lowered concrete effort.
pub(crate) fn enabled(effort: impl Into<EffortValue>) -> Self {
Self::Enabled {
effort: Some(effort.into()),
}
}
/// Report the resolved mode; callers apply model defaults before rendering.
pub(crate) fn is_enabled(&self) -> bool {
matches!(self, Self::Enabled { .. })
}
/// Return the active effort, which may still be missing before model fallback.
pub(crate) fn effort(&self) -> Option<&EffortValue> {
match self {
Self::Enabled { effort } => effort.as_ref(),
Self::Default | Self::Disabled => None,
}
}
/// Project the same model-lowered decision used to render the native prompt.
///
/// Preserve request extras and write both toggle aliases plus the effective
/// effort. Disabled mode emits `"none"`; enabled mode with no effective effort
/// clears the original effort key. `Default` preserves request kwargs.
pub(crate) fn template_kwargs(&self, request: &ChatRequest) -> HashMap<String, Value> {
let mut kwargs = request.chat_options.template_kwargs.clone();
if matches!(self, Self::Default) {
return kwargs;
}
kwargs.insert("thinking".to_string(), json!(self.is_enabled()));
kwargs.insert("enable_thinking".to_string(), json!(self.is_enabled()));
match self {
Self::Disabled => {
kwargs.insert("reasoning_effort".to_string(), json!("none"));
}
Self::Enabled {
effort: Some(effort),
} => {
kwargs.insert("reasoning_effort".to_string(), json!(effort));
}
Self::Enabled { effort: None } => {
kwargs.remove("reasoning_effort");
}
Self::Default => unreachable!(),
}
kwargs
}
}
#[cfg(test)]
mod tests {
use super::*;
fn kwargs(value: Value) -> HashMap<String, Value> {
serde_json::from_value(value).unwrap()
}
#[test]
fn numeric_effort_rejects_non_finite_values() {
for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
assert!(EffortValue::try_from(value).unwrap_err().is_request_validation_error());
}
}
#[test]
fn explicit_toggle_decides_mode_and_none_inherits_enabled_effort() {
use ReasoningControl::{Default, Disabled, Enabled};
for (source, expected) in [
(json!({}), Default),
(json!({"reasoning_effort": "none"}), Disabled),
(
json!({"reasoning_effort": "high"}),
ReasoningControl::enabled("high"),
),
(json!({"enable_thinking": true}), Enabled { effort: None }),
(
json!({"enable_thinking": true, "reasoning_effort": "none"}),
Enabled { effort: None },
),
(
json!({"enable_thinking": true, "reasoning_effort": "low"}),
ReasoningControl::enabled("low"),
),
(
json!({"enable_thinking": false, "reasoning_effort": "high"}),
Disabled,
),
(
json!({"thinking": true, "enable_thinking": false}),
Enabled { effort: None },
),
(
json!({"thinking": false, "enable_thinking": true}),
Disabled,
),
(
json!({"reasoning_effort": 37}),
ReasoningControl::enabled(EffortValue::Number(37.into())),
),
(json!({"reasoning_effort": null}), Default),
] {
let source = kwargs(source);
assert_eq!(
ReasoningControl::from_template_kwargs(&source).unwrap(),
expected,
"{source:?}"
);
}
}
#[test]
fn request_and_deployment_states_fall_back_before_model_defaults() {
for (request, defaults, expected) in [
(
json!({}),
json!({"enable_thinking": false}),
ReasoningControl::Disabled,
),
(
json!({"reasoning_effort": "high"}),
json!({"thinking": false}),
ReasoningControl::enabled("high"),
),
(
json!({"enable_thinking": false}),
json!({"reasoning_effort": "high"}),
ReasoningControl::Disabled,
),
(
json!({"thinking": true, "reasoning_effort": "none"}),
json!({"reasoning_effort": "low"}),
ReasoningControl::enabled("low"),
),
(
json!({"thinking": true}),
json!({"thinking": false, "reasoning_effort": "low"}),
ReasoningControl::enabled("max"),
),
(json!({}), json!({}), ReasoningControl::enabled("max")),
] {
let request = kwargs(request);
let defaults = kwargs(defaults);
let actual = ReasoningControl::from_template_kwargs(&request)
.unwrap()
.fallback(ReasoningControl::from_template_kwargs(&defaults).unwrap())
.fallback(ReasoningControl::enabled("max"));
assert_eq!(
actual, expected,
"request={request:?}, defaults={defaults:?}"
);
}
}
#[test]
fn selected_toggle_requires_a_boolean() {
for value in [
json!("true"),
json!(1),
json!(null),
json!({"type": "enabled"}),
] {
assert!(
ReasoningControl::from_template_kwargs(&kwargs(json!({
"thinking": value, "enable_thinking": true
})))
.unwrap_err()
.is_request_validation_error()
);
}
}
#[test]
fn invalid_effort_types_are_rejected_only_when_the_source_uses_them() {
for value in [json!(true), json!(false), json!([]), json!({})] {
for toggle in [None, Some(true), Some(false)] {
let mut source = kwargs(json!({"reasoning_effort": value}));
if let Some(toggle) = toggle {
source.insert("thinking".to_owned(), json!(toggle));
}
let control = ReasoningControl::from_template_kwargs(&source);
if toggle == Some(false) {
assert_eq!(control.unwrap(), ReasoningControl::Disabled);
} else {
assert!(control.unwrap_err().is_request_validation_error());
}
}
// A typed request effort shadows malformed kwargs effort.
let mut request = ChatRequest::for_test();
request.chat_options.reasoning_effort = Some(EffortValue::from("high"));
request.chat_options.template_kwargs = kwargs(json!({"reasoning_effort": value}));
assert_eq!(
ReasoningControl::from_request(&request).unwrap(),
ReasoningControl::enabled("high")
);
}
}
}
rust/src/chat/src/renderer/deepseek.rs
+68 -76
View File
@@ -17,10 +17,10 @@ use serde_json_fmt::JsonFormat;
use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER;
use super::MediaPartSource;
use crate::EffortValue;
use crate::error::{Error, Result};
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort,
};
use crate::reasoning::ReasoningControl;
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
const BOS_TOKEN: &str = "<｜begin▁of▁sentence｜>";
@@ -87,21 +87,26 @@ struct RenderedToolSchema<'a> {
}
/// Render one chat request into the final prompt string.
pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result<String> {
render_request_with_media_order(request, dialect).map(|(prompt, _)| prompt)
pub(super) fn render_request(
request: &ChatRequest,
dialect: DsDialect,
reasoning: &ReasoningControl,
) -> Result<String> {
render_request_with_media_order(request, dialect, reasoning).map(|(prompt, _)| prompt)
}
/// Render one chat request and record media in placeholder order.
pub(super) fn render_request_with_media_order(
request: &ChatRequest,
dialect: DsDialect,
reasoning: &ReasoningControl,
) -> Result<(String, Vec<MediaPartSource>)> {
let (thinking_mode, reasoning_effort_prompt) = match dialect {
DsDialect::V4 => {
resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))?
}
DsDialect::V41 => resolve_v41_thinking_options(request)?,
let thinking_mode = if reasoning.is_enabled() {
ThinkingMode::Thinking
} else {
ThinkingMode::Chat
};
let reasoning_effort_prompt = reasoning_prompt(reasoning, dialect);
let request_tools = request_tools(request);
let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools);
let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true)
@@ -254,76 +259,63 @@ pub(super) fn render_request_with_media_order(
Ok((out, media_order))
}
/// Resolve DeepSeek V4's thinking controls. Unlike the Python tokenizer
/// wrapper, the Rust renderer only consumes the typed top-level
/// `reasoning_effort`; the generic template-kwargs map is left for HF
/// templates.
fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'static str)> {
let mut thinking_mode = match request.enable_thinking()?.unwrap_or(true) {
true => ThinkingMode::Thinking,
false => ThinkingMode::Chat,
};
let mut reasoning_effort_prompt = REASONING_EFFORT_HIGH;
match request.chat_options.reasoning_effort {
Some(ReasoningEffort::None) => thinking_mode = ThinkingMode::Chat,
Some(ReasoningEffort::Max) => {
reasoning_effort_prompt = REASONING_EFFORT_MAX;
}
Some(ReasoningEffort::XHigh | ReasoningEffort::High) => {
reasoning_effort_prompt = REASONING_EFFORT_HIGH;
}
Some(ReasoningEffort::Minimal | ReasoningEffort::Medium | ReasoningEffort::Low) => {
reasoning_effort_prompt = "";
}
None => {}
/// Map the shared reasoning decision to the selected DeepSeek effort dialect.
pub(super) fn resolve_reasoning(
control: ReasoningControl,
dialect: DsDialect,
) -> Result<ReasoningControl> {
let control = control.fallback(ReasoningControl::enabled("high"));
if !control.is_enabled() {
return Ok(control);
}
Ok((thinking_mode, reasoning_effort_prompt))
let effort = control.effort().expect("enabled DeepSeek reasoning has a default effort");
let effort = match dialect {
DsDialect::V4 => {
let effort = match effort.as_str() {
Some("minimal" | "medium" | "low") => "low",
Some("high" | "xhigh") => "high",
Some("max") => "max",
_ => {
return Err(Error::InvalidReasoningEffort(format!(
"DeepSeek V4 reasoning_effort must be minimal, low, medium, high, xhigh, or max, got {effort}"
)));
}
};
EffortValue::from(effort)
}
DsDialect::V41 => {
let budget = match effort {
EffortValue::String(effort) => match effort.as_str() {
"low" => 25,
"high" => 50,
"xhigh" => 75,
"max" => 100,
_ => return Err(invalid_v41_reasoning_effort()),
},
EffortValue::Number(budget) => budget
.as_u64()
.filter(|budget| (1..=100).contains(budget))
.ok_or_else(invalid_v41_reasoning_effort)?,
};
EffortValue::Number(budget.into())
}
};
Ok(ReasoningControl::enabled(effort))
}
/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs.
fn resolve_v41_thinking_options(
request: &ChatRequest,
) -> Result<(ThinkingMode, Cow<'static, str>)> {
let mut thinking = request.enable_thinking()?.unwrap_or(true);
let budget = match request.chat_options.reasoning_effort {
Some(ReasoningEffort::None) => {
thinking = false;
50
}
Some(ReasoningEffort::Low) => 25,
Some(ReasoningEffort::High) => 50,
Some(ReasoningEffort::XHigh) => 75,
Some(ReasoningEffort::Max) => 100,
Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => {
return Err(invalid_v41_reasoning_effort());
}
None => match request.chat_options.template_kwargs.get("reasoning_effort") {
None | Some(Value::Null) => 50,
Some(Value::String(effort)) => match effort.as_str() {
"low" => 25,
"high" => 50,
"xhigh" => 75,
"max" => 100,
_ => return Err(invalid_v41_reasoning_effort()),
},
Some(Value::Number(budget)) => budget
.as_u64()
.filter(|budget| (1..=100).contains(budget))
.ok_or_else(invalid_v41_reasoning_effort)?,
Some(_) => return Err(invalid_v41_reasoning_effort()),
},
fn reasoning_prompt(control: &ReasoningControl, dialect: DsDialect) -> Cow<'static, str> {
let Some(effort) = control.effort() else {
return Cow::Borrowed("");
};
if thinking {
Ok((
ThinkingMode::Thinking,
Cow::Owned(format!(
"Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
)),
))
} else {
Ok((ThinkingMode::Chat, Cow::Borrowed("")))
match dialect {
DsDialect::V4 => Cow::Borrowed(match effort.as_str() {
Some("high") => REASONING_EFFORT_HIGH,
Some("max") => REASONING_EFFORT_MAX,
_ => "",
}),
DsDialect::V41 => Cow::Owned(format!(
"Reasoning Effort: {effort} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
)),
}
}
rust/src/chat/src/renderer/deepseek_v32/encoding.rs
+6 -2
View File
@@ -14,6 +14,7 @@ use serde_json::Value;
use serde_json_fmt::JsonFormat;
use crate::error::{Error, Result};
use crate::reasoning::ReasoningControl;
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatRole, ChatTool};
use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
@@ -42,8 +43,11 @@ struct RenderedToolSchema<'a> {
}
/// Render one chat request into the final prompt string.
pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
let thinking_mode = match request.enable_thinking()?.unwrap_or(false) {
pub(super) fn render_request(
request: &ChatRequest,
reasoning: &ReasoningControl,
) -> Result<String> {
let thinking_mode = match reasoning.is_enabled() {
true => ThinkingMode::Thinking,
false => ThinkingMode::Chat,
};
rust/src/chat/src/renderer/deepseek_v32/mod.rs
+17 -7
View File
@@ -3,31 +3,41 @@
mod encoding;
use std::collections::HashMap;
use serde_json::Value;
use vllm_text::Prompt;
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use super::{ChatRenderer, RenderedPrompt};
use crate::Result;
use crate::reasoning::ReasoningControl;
use crate::request::ChatRequest;
/// Dedicated DeepSeek V3.2 renderer.
#[derive(Debug, Clone, Copy, Default)]
pub struct DeepSeekV32ChatRenderer;
#[derive(Debug, Clone, Default)]
pub struct DeepSeekV32ChatRenderer {
default_template_kwargs: HashMap<String, Value>,
}
impl DeepSeekV32ChatRenderer {
/// Create the dedicated DeepSeek V3.2 renderer.
pub fn new() -> Self {
Self
pub fn new(default_template_kwargs: HashMap<String, Value>) -> Self {
Self {
default_template_kwargs,
}
}
}
impl ChatRenderer for DeepSeekV32ChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
let reasoning = ReasoningControl::resolve(request, &self.default_template_kwargs)?
.fallback(ReasoningControl::Disabled);
Ok(RenderedPrompt {
prompt: Prompt::Text(encoding::render_request(request)?),
prompt: Prompt::Text(encoding::render_request(request, &reasoning)?),
media_order: None,
effective_template_kwargs: request_template_kwargs(request),
effective_template_kwargs: reasoning.template_kwargs(request),
})
}
}
rust/src/chat/src/renderer/deepseek_v32/tests.rs
+26 -3
View File
@@ -8,6 +8,7 @@ use serde_json::{Value, json};
use thiserror_ext::AsReport;
use super::DeepSeekV32ChatRenderer;
use crate::EffortValue;
use crate::error::Error;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
@@ -18,7 +19,7 @@ use crate::request::{
use crate::{ChatRenderer, ChatRole};
fn render_request(request: &ChatRequest) -> String {
DeepSeekV32ChatRenderer::new()
DeepSeekV32ChatRenderer::default()
.render(request)
.unwrap()
.prompt
@@ -26,8 +27,30 @@ fn render_request(request: &ChatRequest) -> String {
.expect("deepseek renderer should return text prompt")
}
#[test]
fn effort_enables_binary_thinking_and_none_disables_it() {
let mut request = ChatRequest::for_test();
for (effort, enabled) in [
(EffortValue::from("high"), true),
(EffortValue::from("custom"), true),
(EffortValue::Number(42.into()), true),
(EffortValue::from("none"), false),
] {
request.chat_options.reasoning_effort = Some(effort);
let rendered = DeepSeekV32ChatRenderer::default().render(&request).unwrap();
assert_eq!(
rendered.prompt.into_text().unwrap().ends_with("<think>"),
enabled
);
assert_eq!(
rendered.effective_template_kwargs["enable_thinking"],
enabled
);
}
}
fn render_result(request: &ChatRequest) -> Result<String, Error> {
DeepSeekV32ChatRenderer::new().render(request).map(|rendered| {
DeepSeekV32ChatRenderer::default().render(request).map(|rendered| {
rendered
.prompt
.into_text()
@@ -345,7 +368,7 @@ fn render_rejects_multimodal_input() {
..ChatRequest::for_test()
};
let error = DeepSeekV32ChatRenderer::new().render(&request).unwrap_err();
let error = DeepSeekV32ChatRenderer::default().render(&request).unwrap_err();
assert!(matches!(
error,
rust/src/chat/src/renderer/deepseek_v4/mod.rs
+23 -7
View File
@@ -1,31 +1,47 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use serde_json::Value;
use vllm_text::Prompt;
use super::deepseek::{self, DsDialect};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use super::{ChatRenderer, RenderedPrompt};
use crate::Result;
use crate::reasoning::ReasoningControl;
use crate::request::ChatRequest;
/// Dedicated DeepSeek V4 renderer.
#[derive(Debug, Clone, Copy, Default)]
pub struct DeepSeekV4ChatRenderer;
#[derive(Debug, Clone, Default)]
pub struct DeepSeekV4ChatRenderer {
default_template_kwargs: HashMap<String, Value>,
}
impl DeepSeekV4ChatRenderer {
pub fn new() -> Self {
Self
pub fn new(default_template_kwargs: HashMap<String, Value>) -> Self {
Self {
default_template_kwargs,
}
}
}
impl ChatRenderer for DeepSeekV4ChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
let reasoning = deepseek::resolve_reasoning(
ReasoningControl::resolve(request, &self.default_template_kwargs)?,
DsDialect::V4,
)?;
Ok(RenderedPrompt {
prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?),
prompt: Prompt::Text(deepseek::render_request(
request,
DsDialect::V4,
&reasoning,
)?),
media_order: None,
effective_template_kwargs: request_template_kwargs(request),
effective_template_kwargs: reasoning.template_kwargs(request),
})
}
}
rust/src/chat/src/renderer/deepseek_v4/tests.rs
+62 -22
View File
@@ -8,13 +8,14 @@ use serde_json::Value;
use super::DeepSeekV4ChatRenderer;
use crate::ChatRenderer;
use crate::EffortValue;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::renderer::deepseek_v41::DeepSeekV41ChatRenderer;
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{ChatMessage, ChatRequest, ChatTool, GenerationPromptMode, ReasoningEffort};
use crate::request::{ChatMessage, ChatRequest, ChatTool, GenerationPromptMode};
fn render_request(request: &ChatRequest) -> String {
DeepSeekV4ChatRenderer::new()
DeepSeekV4ChatRenderer::default()
.render(request)
.unwrap()
.prompt
@@ -31,13 +32,13 @@ fn thinking_request(messages: Vec<ChatMessage>) -> ChatRequest {
.chat_options
.template_kwargs
.insert("thinking".to_string(), Value::Bool(true));
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
request
}
fn fixture_request(input_name: &str) -> ChatRequest {
let mut request = fixture_chat_request(&fixture_path(input_name), deepseek_fixture_options());
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
request
}
@@ -203,7 +204,7 @@ fn reasoning_effort_max_adds_prefix_when_thinking_is_enabled() {
.chat_options
.template_kwargs
.insert("thinking".to_string(), Value::Bool(true));
request.chat_options.reasoning_effort = Some(ReasoningEffort::Max);
request.chat_options.reasoning_effort = Some(EffortValue::from("max"));
let rendered = render_request(&request);
@@ -226,7 +227,7 @@ fn reasoning_effort_high_adds_0731_high_prefix() {
.chat_options
.template_kwargs
.insert("thinking".to_string(), Value::Bool(true));
request.chat_options.reasoning_effort = Some(ReasoningEffort::High);
request.chat_options.reasoning_effort = Some(EffortValue::from("high"));
let rendered = render_request(&request);
@@ -258,7 +259,7 @@ fn reasoning_effort_xhigh_maps_to_high() {
messages: vec![ChatMessage::user("solve it")],
..ChatRequest::for_test()
};
request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
request.chat_options.reasoning_effort = Some(EffortValue::from("xhigh"));
let rendered = render_request(&request);
@@ -268,7 +269,7 @@ fn reasoning_effort_xhigh_maps_to_high() {
#[test]
fn minimal_and_medium_reasoning_effort_map_to_low() {
for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
for effort in [EffortValue::from("minimal"), EffortValue::from("medium")] {
let mut request = ChatRequest {
messages: vec![ChatMessage::user("solve it")],
..ChatRequest::for_test()
@@ -292,7 +293,7 @@ fn reasoning_effort_low_keeps_the_default_prompt() {
.chat_options
.template_kwargs
.insert("thinking".to_string(), Value::Bool(true));
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
let rendered = render_request(&request);
@@ -305,11 +306,7 @@ fn reasoning_effort_none_disables_thinking() {
messages: vec![ChatMessage::user("answer directly")],
..ChatRequest::for_test()
};
request
.chat_options
.template_kwargs
.insert("thinking".to_string(), Value::Bool(true));
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
let rendered = render_request(&request);
@@ -317,6 +314,49 @@ fn reasoning_effort_none_disables_thinking() {
.assert_eq(&rendered);
}
#[test]
fn normalized_reasoning_drives_both_prompt_and_compatibility_kwargs() {
for (kwargs, enabled, effort, prefix) in [
(
serde_json::json!({"enable_thinking": true, "reasoning_effort": "none"}),
true,
"high",
"Absolute maximum",
),
(
serde_json::json!({"thinking": false, "enable_thinking": true, "reasoning_effort": "max"}),
false,
"none",
"",
),
(
serde_json::json!({"reasoning_effort": "xhigh"}),
true,
"high",
"Absolute maximum",
),
] {
let mut request = ChatRequest::for_test();
request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap();
let rendered = DeepSeekV4ChatRenderer::default().render(&request).unwrap();
let prompt = rendered.prompt.into_text().unwrap();
assert_eq!(prompt.ends_with("<think>"), enabled);
assert_eq!(prompt.contains("Reasoning Effort:"), enabled);
if enabled {
assert!(prompt.contains(prefix));
}
assert_eq!(rendered.effective_template_kwargs["thinking"], enabled);
assert_eq!(
rendered.effective_template_kwargs["enable_thinking"],
enabled
);
assert_eq!(
rendered.effective_template_kwargs["reasoning_effort"],
effort
);
}
}
#[test]
fn system_to_assistant_transitions_in_chat_and_thinking_modes() {
let rendered = [false, true].map(|enable_thinking| {
@@ -338,7 +378,7 @@ fn system_to_assistant_transitions_in_chat_and_thinking_modes() {
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
render_request(&request)
});
@@ -366,7 +406,7 @@ fn trailing_system_transitions_in_chat_and_thinking_modes() {
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
render_request(&request)
});
@@ -380,7 +420,7 @@ fn trailing_system_transitions_in_chat_and_thinking_modes() {
}
#[test]
fn reasoning_effort_template_kwarg_is_ignored() {
fn reasoning_effort_template_kwarg_selects_effort() {
let mut request = ChatRequest {
messages: vec![ChatMessage::user("solve it")],
..ChatRequest::for_test()
@@ -396,15 +436,15 @@ fn reasoning_effort_template_kwarg_is_ignored() {
let rendered = render_request(&request);
assert!(rendered.starts_with("<｜begin▁of▁sentence｜>Reasoning Effort: Absolute maximum"));
assert!(rendered.starts_with("<｜begin▁of▁sentence｜>Reasoning Effort: Beyond maximum"));
assert!(rendered.ends_with("<｜Assistant｜><think>"));
}
#[test]
fn tool_call_arguments_preserve_non_objects_like_recipe() {
let renderers: [(&dyn ChatRenderer, &str); 2] = [
(&DeepSeekV4ChatRenderer::new(), "parameter"),
(&DeepSeekV41ChatRenderer::new(), " parameter"),
(&DeepSeekV4ChatRenderer::default(), "parameter"),
(&DeepSeekV41ChatRenderer::default(), " parameter"),
];
for arguments in [
"not json",
@@ -600,7 +640,7 @@ fn drop_thinking_false_keeps_prior_assistant_reasoning() {
.chat_options
.template_kwargs
.insert("drop_thinking".to_string(), Value::Bool(false));
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
let rendered = render_request(&request);
@@ -693,7 +733,7 @@ fn trailing_system_omits_generation_prompt_when_disabled() {
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), Value::Bool(false));
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
let rendered = render_request(&request);
rust/src/chat/src/renderer/deepseek_v41/mod.rs
+19 -7
View File
@@ -1,33 +1,45 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use serde_json::Value;
use vllm_text::Prompt;
use super::deepseek::{self, DsDialect};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use super::{ChatRenderer, RenderedPrompt};
use crate::Result;
use crate::reasoning::ReasoningControl;
use crate::request::ChatRequest;
/// Dedicated DeepSeek V4.1 renderer.
#[derive(Debug, Clone, Copy, Default)]
pub struct DeepSeekV41ChatRenderer;
#[derive(Debug, Clone, Default)]
pub struct DeepSeekV41ChatRenderer {
default_template_kwargs: HashMap<String, Value>,
}
impl DeepSeekV41ChatRenderer {
pub fn new() -> Self {
Self
pub fn new(default_template_kwargs: HashMap<String, Value>) -> Self {
Self {
default_template_kwargs,
}
}
}
impl ChatRenderer for DeepSeekV41ChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
let reasoning = deepseek::resolve_reasoning(
ReasoningControl::resolve(request, &self.default_template_kwargs)?,
DsDialect::V41,
)?;
let (prompt, media_order) =
deepseek::render_request_with_media_order(request, DsDialect::V41)?;
deepseek::render_request_with_media_order(request, DsDialect::V41, &reasoning)?;
Ok(RenderedPrompt {
prompt: Prompt::Text(prompt),
media_order: Some(media_order),
effective_template_kwargs: request_template_kwargs(request),
effective_template_kwargs: reasoning.template_kwargs(request),
})
}
}
rust/src/chat/src/renderer/deepseek_v41/tests.rs
+49 -15
View File
@@ -8,14 +8,15 @@ use serde_json::{Value, json};
use super::DeepSeekV41ChatRenderer;
use crate::ChatRenderer;
use crate::EffortValue;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort,
ChatContent, ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode,
};
fn render(request: &ChatRequest) -> String {
DeepSeekV41ChatRenderer::new()
DeepSeekV41ChatRenderer::default()
.render(request)
.unwrap()
.prompt
@@ -64,7 +65,7 @@ fn inlines_image_placeholder_at_image_part_positions() {
// The placeholder is unconditional (matching the Python encoding's
// `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later,
// at media preparation.
let renderer = DeepSeekV41ChatRenderer::new();
let renderer = DeepSeekV41ChatRenderer::default();
let request = ChatRequest {
messages: vec![ChatMessage::User {
content: ChatContent::Parts(vec![
@@ -87,7 +88,7 @@ fn inlines_image_placeholder_at_image_part_positions() {
#[test]
fn media_order_follows_reordered_tool_results() {
let renderer = DeepSeekV41ChatRenderer::new();
let renderer = DeepSeekV41ChatRenderer::default();
let request = ChatRequest {
messages: vec![
ChatMessage::assistant_blocks(vec![
@@ -136,7 +137,7 @@ fn media_order_follows_reordered_tool_results() {
#[test]
fn media_order_excludes_dropped_historical_developer_content() {
let renderer = DeepSeekV41ChatRenderer::new();
let renderer = DeepSeekV41ChatRenderer::default();
let request = ChatRequest {
messages: vec![
ChatMessage::developer(vec![image_part("historical-image")], None),
@@ -165,10 +166,11 @@ fn image_part(uuid: &str) -> ChatContentPart {
fn maps_reasoning_effort_to_reference_numeric_budget() {
for (effort, budget) in [
(None, 50),
(Some(ReasoningEffort::Low), 25),
(Some(ReasoningEffort::High), 50),
(Some(ReasoningEffort::XHigh), 75),
(Some(ReasoningEffort::Max), 100),
(Some(EffortValue::from("low")), 25),
(Some(EffortValue::from("high")), 50),
(Some(EffortValue::from("xhigh")), 75),
(Some(EffortValue::from("max")), 100),
(Some(EffortValue::Number(37.into())), 37),
] {
let mut request = request();
request.chat_options.reasoning_effort = effort;
@@ -193,11 +195,43 @@ fn accepts_numeric_template_effort_with_top_level_precedence() {
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100")));
request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
request.chat_options.reasoning_effort = Some(EffortValue::from("xhigh"));
assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100"));
}
}
#[test]
fn enabling_with_none_inherits_deployment_effort_and_reports_numeric_control() {
let renderer =
DeepSeekV41ChatRenderer::new([("reasoning_effort".to_string(), json!(37))].into());
let mut request = request();
request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
request
.chat_options
.template_kwargs
.insert("enable_thinking".into(), json!(true));
let rendered = renderer.render(&request).unwrap();
let prompt = rendered.prompt.into_text().unwrap();
assert!(prompt.contains("Reasoning Effort: 37 (range 1-100"));
assert!(prompt.ends_with("<think>"));
assert_eq!(rendered.effective_template_kwargs["reasoning_effort"], 37);
assert_eq!(rendered.effective_template_kwargs["enable_thinking"], true);
request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
request
.chat_options
.template_kwargs
.insert("reasoning_effort".into(), json!(101));
request.chat_options.reasoning_effort = None;
let rendered = renderer.render(&request).unwrap();
assert!(rendered.prompt.into_text().unwrap().ends_with("</think>"));
assert_eq!(
rendered.effective_template_kwargs["reasoning_effort"],
"none"
);
assert_eq!(rendered.effective_template_kwargs["enable_thinking"], false);
}
#[test]
fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
for effort in [
@@ -211,13 +245,13 @@ fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
] {
let mut request = request();
request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
let error = DeepSeekV41ChatRenderer::default().render(&request).unwrap_err();
assert!(error.is_request_validation_error());
}
for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
for effort in [EffortValue::from("minimal"), EffortValue::from("medium")] {
let mut request = request();
request.chat_options.reasoning_effort = Some(effort);
let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
let error = DeepSeekV41ChatRenderer::default().render(&request).unwrap_err();
assert!(error.is_request_validation_error());
}
}
@@ -229,8 +263,8 @@ fn chat_mode_and_none_effort_use_closed_thinking_prefix() {
let expected = expect!["<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜></think>"];
expected.assert_eq(&render(&request));
request.chat_options.template_kwargs.insert("thinking".into(), json!(true));
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
request.chat_options.template_kwargs.remove("thinking");
request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
expected.assert_eq(&render(&request));
}
rust/src/chat/src/renderer/harmony/mod.rs
+51 -27
View File
@@ -5,21 +5,26 @@
pub(crate) mod encoding;
use std::collections::HashMap;
use openai_harmony::HarmonyEncoding;
use openai_harmony::chat::{
Author, Conversation, DeveloperContent, Message, ReasoningEffort as HarmonyReasoningEffort,
Role, SystemContent, ToolDescription,
};
use serde_json::Value;
use thiserror_ext::AsReport as _;
use time::macros::format_description;
use vllm_text::Prompt;
use self::encoding::harmony_encoding;
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use super::{ChatRenderer, RenderedPrompt};
use crate::AssistantMessageExt as _;
use crate::EffortValue;
use crate::error::{Error, Result};
use crate::event::AssistantContentBlock;
use crate::reasoning::ReasoningControl;
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, GenerationPromptMode};
use crate::{AssistantMessageExt as _, ReasoningEffort};
const SYSTEM_START_DATE_ENV: &str = "VLLM_SYSTEM_START_DATE";
const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS";
@@ -28,6 +33,7 @@ const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTR
pub struct HarmonyChatRenderer {
encoding: &'static HarmonyEncoding,
options: Options,
default_template_kwargs: HashMap<String, Value>,
}
struct Options {
@@ -44,8 +50,9 @@ impl HarmonyChatRenderer {
/// unset, the renderer uses the current local date with a UTC fallback.
/// - `VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS` moves leading instructions
/// into the system model identity when set to a non-zero integer.
pub fn new() -> Result<Self> {
pub fn new(default_template_kwargs: HashMap<String, Value>) -> Result<Self> {
Self::with_options(
default_template_kwargs,
env_system_start_date(),
env_use_harmony_system_instructions(),
)
@@ -57,11 +64,13 @@ impl HarmonyChatRenderer {
/// Production code should call [`Self::new`] so the renderer observes the
/// same environment contract as the Python Harmony path.
pub fn with_options(
default_template_kwargs: HashMap<String, Value>,
system_start_date: impl Into<String>,
use_system_instructions: bool,
) -> Result<Self> {
Ok(Self {
encoding: harmony_encoding()?,
default_template_kwargs,
options: Options {
system_start_date: system_start_date.into(),
use_system_instructions,
@@ -73,7 +82,11 @@ impl HarmonyChatRenderer {
///
/// Harmony owns both prompt formatting and tokenization, so the Rust
/// frontend bypasses the generic HF tokenizer path for GPT-OSS input.
fn render_token_ids(&self, request: &ChatRequest) -> Result<Vec<u32>> {
fn render_token_ids(
&self,
request: &ChatRequest,
effort: HarmonyReasoningEffort,
) -> Result<Vec<u32>> {
if request.has_multimodal() {
return Err(Error::UnsupportedMultimodalContent("image_url"));
}
@@ -86,7 +99,8 @@ impl HarmonyChatRenderer {
));
}
let messages = auto_drop_analysis_messages(to_harmony_messages(request, &self.options)?);
let messages =
auto_drop_analysis_messages(to_harmony_messages(request, &self.options, effort)?);
let conversation = Conversation::from_messages(messages);
// Pass `None` so oss-harmony does not apply its narrower built-in
// analysis-drop policy after the Rust-side Python-parity cleanup above.
@@ -114,10 +128,13 @@ impl ChatRenderer for HarmonyChatRenderer {
/// Render a chat request as [`Prompt::TokenIds`] with template kwargs echoed
/// for downstream accounting/debugging.
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
let reasoning = ReasoningControl::resolve(request, &self.default_template_kwargs)?
.fallback(ReasoningControl::enabled("medium"));
let effort = to_harmony_reasoning_effort(reasoning.effort())?;
Ok(RenderedPrompt {
prompt: Prompt::TokenIds(self.render_token_ids(request)?),
prompt: Prompt::TokenIds(self.render_token_ids(request, effort)?),
media_order: None,
effective_template_kwargs: request_template_kwargs(request),
effective_template_kwargs: reasoning.template_kwargs(request),
})
}
}
@@ -127,12 +144,21 @@ impl ChatRenderer for HarmonyChatRenderer {
/// This adds the Harmony system/developer preamble, peels at most one leading
/// system/developer instruction message, and then lowers the remaining chat
/// history message-by-message.
fn to_harmony_messages(request: &ChatRequest, options: &Options) -> Result<Vec<Message>> {
fn to_harmony_messages(
request: &ChatRequest,
options: &Options,
effort: HarmonyReasoningEffort,
) -> Result<Vec<Message>> {
let (instructions, leading_developer_tools, remaining_messages) =
peel_leading_instructions(&request.messages)?;
let tool_call_names = tool_call_names(&request.messages);
let mut messages =
build_harmony_preamble(request, instructions, leading_developer_tools, options)?;
let mut messages = build_harmony_preamble(
request,
instructions,
leading_developer_tools,
options,
effort,
)?;
for message in remaining_messages {
messages.extend(to_harmony_message(message, &tool_call_names, options)?);
@@ -177,12 +203,13 @@ fn build_harmony_preamble(
instructions: Option<String>,
leading_developer_tools: Option<&[ChatTool]>,
options: &Options,
effort: HarmonyReasoningEffort,
) -> Result<Vec<Message>> {
let mut messages = vec![Message::from_role_and_content(
Role::System,
system_content(
instructions.as_deref().filter(|_| options.use_system_instructions),
request.chat_options.reasoning_effort,
Some(effort),
&options.system_start_date,
)?,
)];
@@ -227,18 +254,18 @@ fn preamble_tool_descriptions(
/// Construct the Harmony system content for the request preamble.
///
/// Harmony defaults the reasoning effort to `medium` when none is provided, so
/// this only sets an explicit effort after validating vLLM's request value.
/// The preamble receives normalized effort; later system messages use Harmony's
/// own default when their effort is omitted.
fn system_content(
instructions: Option<&str>,
reasoning_effort: Option<ReasoningEffort>,
reasoning_effort: Option<HarmonyReasoningEffort>,
system_start_date: &str,
) -> Result<SystemContent> {
let mut content =
SystemContent::new().with_conversation_start_date(system_start_date.to_string());
if let Some(reasoning_effort) = reasoning_effort {
content = content.with_reasoning_effort(to_harmony_reasoning_effort(reasoning_effort)?);
content = content.with_reasoning_effort(reasoning_effort);
}
if let Some(instructions) = instructions.filter(|text| !text.is_empty()) {
@@ -445,20 +472,17 @@ fn to_tool_descriptions(tools: &[ChatTool]) -> Vec<ToolDescription> {
.collect()
}
/// Map supported OpenAI reasoning-effort values onto Harmony's enum.
/// Map supported reasoning-effort values onto Harmony's enum.
fn to_harmony_reasoning_effort(
reasoning_effort: ReasoningEffort,
reasoning_effort: Option<&EffortValue>,
) -> Result<HarmonyReasoningEffort> {
match reasoning_effort {
ReasoningEffort::Low => Ok(HarmonyReasoningEffort::Low),
ReasoningEffort::Medium => Ok(HarmonyReasoningEffort::Medium),
ReasoningEffort::High => Ok(HarmonyReasoningEffort::High),
ReasoningEffort::None
| ReasoningEffort::Minimal
| ReasoningEffort::XHigh
| ReasoningEffort::Max => Err(Error::ChatTemplate(format!(
"reasoning_effort={:?} is not supported by Harmony. Supported values are: low, medium, high.",
reasoning_effort.as_str()
match reasoning_effort.and_then(EffortValue::as_str) {
Some("low") => Ok(HarmonyReasoningEffort::Low),
Some("medium") => Ok(HarmonyReasoningEffort::Medium),
Some("high") => Ok(HarmonyReasoningEffort::High),
_ => Err(Error::InvalidReasoningEffort(format!(
"reasoning_effort={} is not supported by Harmony. Supported values are: low, medium, high.",
reasoning_effort.unwrap_or(&EffortValue::from("none"))
))),
}
}
rust/src/chat/src/renderer/harmony/tests.rs
+41 -7
View File
@@ -9,12 +9,11 @@ use thiserror_ext::AsReport as _;
use super::HarmonyChatRenderer;
use super::encoding::harmony_encoding;
use crate::ChatRenderer;
use crate::EffortValue;
use crate::error::Error;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{
ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort,
};
use crate::request::{ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode};
const PINNED_DATE: &str = "2025-06-28";
@@ -22,7 +21,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
fixture_chat_request(
&fixture_path(input_name),
FixtureRequestOptions {
enable_thinking: Some(false),
enable_thinking: None,
no_generation_prompt_when_last_assistant: false,
},
)
@@ -36,7 +35,8 @@ fn fixture_path(name: &str) -> PathBuf {
}
fn test_renderer(use_system_instructions: bool) -> HarmonyChatRenderer {
HarmonyChatRenderer::with_options(PINNED_DATE, use_system_instructions).unwrap()
HarmonyChatRenderer::with_options(Default::default(), PINNED_DATE, use_system_instructions)
.unwrap()
}
fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
@@ -138,14 +138,48 @@ fn drops_stale_analysis_fixture() {
#[test]
fn rejects_invalid_reasoning_effort() {
let mut request = ChatRequest::for_test();
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
let error = test_renderer(false).render(&request).unwrap_err();
expect![[r#"chat template error: reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]]
expect![[r#"reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]]
.assert_eq(&error.to_report_string());
}
#[test]
fn normalized_reasoning_respects_harmony_capability_and_deployment_effort() {
let renderer = HarmonyChatRenderer::with_options(
[("reasoning_effort".to_string(), serde_json::json!("low"))].into(),
PINNED_DATE,
false,
)
.unwrap();
let mut request = ChatRequest::for_test();
request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
request
.chat_options
.template_kwargs
.insert("enable_thinking".into(), serde_json::json!(true));
let rendered = renderer.render(&request).unwrap();
let prompt = harmony_encoding()
.unwrap()
.tokenizer()
.decode_utf8(rendered.prompt.into_token_ids().unwrap())
.unwrap();
assert!(prompt.contains("Reasoning: low"));
assert_eq!(
rendered.effective_template_kwargs["reasoning_effort"],
"low"
);
assert_eq!(rendered.effective_template_kwargs["enable_thinking"], true);
request
.chat_options
.template_kwargs
.insert("thinking".into(), serde_json::json!(false));
assert!(renderer.render(&request).is_err());
}
#[test]
fn rejects_unknown_tool_response_id() {
let request = ChatRequest {
rust/src/chat/src/renderer/hf/mod.rs
+94 -4
View File
@@ -16,7 +16,7 @@ use self::format::{
ChatTemplateContentFormat, ChatTemplateContentFormatOption as ContentFormatOption,
};
use self::template::{CompiledChatTemplate, TemplateContext};
use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs};
use super::{ChatRenderer, RenderedPrompt};
use crate::error::Result;
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
use crate::{
@@ -33,6 +33,31 @@ pub use template::{load_chat_template, resolve_chat_template};
pub use self::format::ChatTemplateContentFormatOption;
/// Extract the effective chat-template kwargs visible to the renderer from the request,
/// using the provided defaults as the base.
fn effective_template_kwargs(
default_template_kwargs: &HashMap<String, JsonValue>,
request: &ChatRequest,
) -> HashMap<String, JsonValue> {
let mut kwargs = default_template_kwargs.clone();
kwargs.extend(request.chat_options.template_kwargs.clone());
if let Some(reasoning_effort) = &request.chat_options.reasoning_effort {
kwargs.insert(
"reasoning_effort".to_string(),
serde_json::json!(reasoning_effort),
);
if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
kwargs.insert(
"enable_thinking".to_string(),
serde_json::json!(reasoning_effort.as_str() != Some("none")),
);
}
}
kwargs
}
/// Template-visible placeholder tokens per supported modality.
///
/// A `None` token means the loaded model does not support that modality, and
@@ -580,9 +605,10 @@ mod tests {
use vllm_text::backend::hf::{HfSpecialTokens, NamedSpecialToken};
use super::{ChatTemplateContentFormatOption, HfChatRenderer, MultimodalRenderInfo};
use crate::EffortValue;
use crate::request::{
ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
GenerationPromptMode, ResolvedToolContext,
};
use crate::{AssistantContentBlock, ChatRenderer, Error, Result};
@@ -1149,7 +1175,7 @@ mod tests {
#[test]
fn chat_template_reasoning_effort_overrides_template_kwargs() {
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
request.chat_options.reasoning_effort = Some(ReasoningEffort::Max);
request.chat_options.reasoning_effort = Some(EffortValue::from("max"));
request.chat_options.template_kwargs.insert(
"reasoning_effort".to_string(),
Value::String("low".to_string()),
@@ -1181,7 +1207,7 @@ mod tests {
#[test]
fn chat_template_reasoning_effort_preserves_request_enable_thinking() {
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
request
.chat_options
.template_kwargs
@@ -1207,6 +1233,70 @@ mod tests {
);
}
#[test]
fn chat_template_preserves_typed_effort_values() {
for effort in [
serde_json::json!(37),
serde_json::json!(0.37),
serde_json::json!("custom"),
] {
let mut request = sample_request(vec![ChatMessage::user("hello")]);
request.chat_options.reasoning_effort =
Some(serde_json::from_value(effort.clone()).unwrap());
request.chat_options.template_kwargs = [
("reasoning_effort".to_string(), serde_json::json!("high")),
("enable_thinking".to_string(), serde_json::json!(true)),
]
.into();
let rendered = HfChatRenderer::new(
Some("{{ reasoning_effort|tojson }}|{{ enable_thinking }}".to_string()),
HashMap::new(),
ChatTemplateContentFormatOption::Auto,
)
.unwrap()
.render(&request)
.unwrap();
assert_eq!(rendered.prompt, Prompt::Text(format!("{effort}|True")));
assert_eq!(
rendered.effective_template_kwargs["reasoning_effort"],
effort
);
}
}
#[test]
fn chat_template_keeps_native_control_conflicts_and_raw_effort_values() {
for effort in [
serde_json::json!(37),
serde_json::json!(null),
serde_json::json!("none"),
] {
let mut request = sample_request(vec![ChatMessage::user("hello")]);
request.chat_options.template_kwargs = [
("thinking".to_string(), serde_json::json!("custom-mode")),
("enable_thinking".to_string(), serde_json::json!(true)),
("reasoning_effort".to_string(), effort),
]
.into();
let rendered = HfChatRenderer::new(
Some(
"{{ thinking }}|{{ enable_thinking }}|{{ reasoning_effort is none }}"
.to_string(),
),
HashMap::new(),
ChatTemplateContentFormatOption::Auto,
)
.unwrap()
.render(&request)
.unwrap();
assert!(rendered.prompt.into_text().unwrap().starts_with("custom-mode|True|"));
assert_eq!(
rendered.effective_template_kwargs,
request.chat_options.template_kwargs
);
}
}
#[test]
fn qwen3_template_omits_reasoning_for_historical_assistant_messages() {
let request = sample_request(vec![
rust/src/chat/src/renderer/inkling/mod.rs
+47 -15
View File
@@ -8,8 +8,10 @@ use thiserror_ext::AsReport as _;
use vllm_text::Prompt;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use super::{ChatRenderer, RenderedPrompt};
use crate::EffortValue;
use crate::error::{Error, Result};
use crate::reasoning::ReasoningControl;
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
use crate::{AssistantContentBlock, AssistantToolCall};
@@ -35,6 +37,7 @@ const MAX_REASONING_EFFORT: f64 = 0.99;
pub struct InklingChatRenderer {
tokenizer: DynTokenizer,
special: InklingSpecialTokenIds,
default_template_kwargs: HashMap<String, Value>,
}
#[derive(Debug, Clone, Copy)]
@@ -78,9 +81,16 @@ impl InklingSpecialTokenIds {
}
impl InklingChatRenderer {
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
pub fn new(
tokenizer: DynTokenizer,
default_template_kwargs: HashMap<String, Value>,
) -> Result<Self> {
let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?;
Ok(Self { tokenizer, special })
Ok(Self {
tokenizer,
special,
default_template_kwargs,
})
}
fn write_text_tokens(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
@@ -150,11 +160,6 @@ impl InklingChatRenderer {
}
fn write_reasoning_effort(&self, out: &mut Vec<u32>, effort: f64) -> Result<()> {
if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
return Err(Error::ChatTemplate(format!(
"Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
)));
}
let formatted = format!("{effort:.2}");
let effort = formatted.trim_end_matches('0').trim_end_matches('.');
let effort = if matches!(effort, "0" | "-0") {
@@ -294,11 +299,18 @@ impl ChatRenderer for InklingChatRenderer {
let mut out = Vec::new();
let mut tool_call_id_to_name = HashMap::new();
let effective_template_kwargs = request_template_kwargs(request);
let reasoning = resolve_reasoning(ReasoningControl::resolve(
request,
&self.default_template_kwargs,
)?)?;
let effective_template_kwargs = reasoning.template_kwargs(request);
let tools = rendered_tools(request);
self.write_tool_declarations(&mut out, &tools)?;
let mut reasoning_effort =
resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort"));
let mut reasoning_effort = if reasoning.is_enabled() {
reasoning.effort().and_then(EffortValue::as_f64)
} else {
Some(0.0)
};
for message in &request.messages {
if !matches!(
@@ -352,12 +364,33 @@ impl ChatRenderer for InklingChatRenderer {
}
}
fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
fn resolve_reasoning(control: ReasoningControl) -> Result<ReasoningControl> {
let control = control.fallback(ReasoningControl::enabled("high"));
if !control.is_enabled() {
return Ok(control);
}
let effort = resolve_reasoning_effort(control.effort());
if let Some(effort) = effort {
if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
return Err(Error::InvalidReasoningEffort(format!(
"Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
)));
}
if effort == 0.0 {
return Ok(ReasoningControl::Disabled);
}
}
Ok(ReasoningControl::Enabled {
effort: effort.map(EffortValue::try_from).transpose()?,
})
}
fn resolve_reasoning_effort(value: Option<&EffortValue>) -> Option<f64> {
let Some(value) = value else {
return Some(0.9);
};
match value {
Value::String(name) => match name.as_str() {
EffortValue::String(name) => match name.as_str() {
"none" => Some(0.0),
"minimal" => Some(0.1),
"low" => Some(0.2),
@@ -366,8 +399,7 @@ fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
"xhigh" | "max" => Some(0.99),
_ => None,
},
Value::Number(number) => number.as_f64(),
_ => None,
EffortValue::Number(number) => number.as_f64(),
}
}
rust/src/chat/src/renderer/inkling/tests.rs
+58 -22
View File
@@ -9,6 +9,7 @@ use serde_json::json;
use thiserror_ext::AsReport;
use vllm_text::tokenizer::Tokenizer;
use crate::EffortValue;
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use super::{
@@ -17,7 +18,7 @@ use super::{
MESSAGE_TOOL, MESSAGE_USER,
};
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort};
use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode};
use crate::{ChatRenderer, Error};
struct FixtureTokenizer;
@@ -87,7 +88,7 @@ impl Tokenizer for FixtureTokenizer {
}
fn renderer() -> InklingChatRenderer {
InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap()
InklingChatRenderer::new(Arc::new(FixtureTokenizer), Default::default()).unwrap()
}
fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
@@ -105,7 +106,7 @@ fn fixture_request(name: &str) -> ChatRequest {
fn inkling_fixture_options() -> FixtureRequestOptions {
FixtureRequestOptions {
enable_thinking: Some(false),
enable_thinking: None,
no_generation_prompt_when_last_assistant: false,
}
}
@@ -206,13 +207,13 @@ fn renders_request_and_developer_tools_as_tool_declare() {
#[test]
fn renders_named_reasoning_effort_after_tool_declarations() {
for (effort, expected) in [
(ReasoningEffort::None, "0.0"),
(ReasoningEffort::Minimal, "0.1"),
(ReasoningEffort::Low, "0.2"),
(ReasoningEffort::Medium, "0.7"),
(ReasoningEffort::High, "0.9"),
(ReasoningEffort::XHigh, "0.99"),
(ReasoningEffort::Max, "0.99"),
(EffortValue::from("none"), "0.0"),
(EffortValue::from("minimal"), "0.1"),
(EffortValue::from("low"), "0.2"),
(EffortValue::from("medium"), "0.7"),
(EffortValue::from("high"), "0.9"),
(EffortValue::from("xhigh"), "0.99"),
(EffortValue::from("max"), "0.99"),
] {
let mut request = fixture_request("tool_declare_input.json");
request.chat_options.reasoning_effort = Some(effort);
@@ -284,6 +285,43 @@ fn defaults_reasoning_effort_to_high() {
);
}
#[test]
fn normalized_mode_and_scalar_are_reflected_in_prompt_and_kwargs() {
for (kwargs, expected, enabled) in [
(
json!({"enable_thinking": true, "reasoning_effort": "none"}),
"0.9",
true,
),
(
json!({"thinking": false, "reasoning_effort": 10}),
"0.0",
false,
),
(json!({"reasoning_effort": 0}), "0.0", false),
(json!({"reasoning_effort": null}), "0.9", true),
(json!({"reasoning_effort": 0.37}), "0.37", true),
] {
let mut request = ChatRequest::for_test();
request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap();
let rendered = renderer().render(&request).unwrap();
let prompt = render_symbolic_tokens(&rendered.prompt.into_token_ids().unwrap());
assert!(prompt.contains(&format!("Thinking effort level: {expected}<|end_message|>")));
assert_eq!(
rendered.effective_template_kwargs["enable_thinking"],
enabled
);
assert_eq!(
rendered.effective_template_kwargs["reasoning_effort"],
if enabled {
json!(expected.parse::<f64>().unwrap())
} else {
json!("none")
}
);
}
}
#[test]
fn renders_numeric_reasoning_effort_template_kwarg() {
let mut request = ChatRequest::for_test();
@@ -301,19 +339,17 @@ fn renders_numeric_reasoning_effort_template_kwarg() {
}
#[test]
fn ignores_unsupported_reasoning_effort_values() {
for value in [json!(true), json!("invalid"), json!(null)] {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), value);
fn ignores_unsupported_reasoning_effort_names() {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!("invalid"));
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_user|><|content_text|>test<|end_message|><|message_model|>"
);
}
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_user|><|content_text|>test<|end_message|><|message_model|>"
);
}
#[test]
rust/src/chat/src/renderer/kimi_k3/encoding.rs
+20 -49
View File
@@ -11,7 +11,9 @@ use serde_json::{Map, Value, json};
use vllm_tokenizer::Tokenizer;
use super::super::MediaPartSource;
use crate::EffortValue;
use crate::error::{Error, Result};
use crate::reasoning::ReasoningControl;
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
};
@@ -76,9 +78,10 @@ impl<'a> K3TokenWriter<'a> {
pub(super) fn render_request_with_media_order(
request: &ChatRequest,
tokenizer: &dyn Tokenizer,
reasoning: &ReasoningControl,
) -> Result<(Vec<u32>, Vec<MediaPartSource>)> {
let thinking = thinking_enabled(request)?;
let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?;
let thinking = reasoning.is_enabled();
let thinking_effort = reasoning.effort().and_then(EffortValue::as_str);
let tools = request_tools(request);
let mut out = K3TokenWriter::new(tokenizer);
@@ -244,58 +247,26 @@ fn request_tools(request: &ChatRequest) -> &[ChatTool] {
request.initial_tools()
}
fn thinking_enabled(request: &ChatRequest) -> Result<bool> {
if let Some(thinking) = request.parse_template_bool("thinking")? {
return Ok(thinking);
}
if let Some(enable_thinking) = request.parse_template_bool("enable_thinking")? {
return Ok(enable_thinking);
}
if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
return Ok(reasoning_effort != crate::request::ReasoningEffort::None);
}
Ok(request
.chat_options
.template_kwargs
.get("reasoning_effort")
.and_then(Value::as_str)
!= Some("none"))
}
fn thinking_effort(request: &ChatRequest) -> Result<String> {
let effort = if let Some(value) = request.chat_options.template_kwargs.get("thinking_effort") {
value.as_str().ok_or_else(|| {
Error::ChatTemplate(format!(
"template kwarg `thinking_effort` must be a string, got {value}"
))
})?
} else if let Some(effort) = request.chat_options.reasoning_effort {
if effort == crate::request::ReasoningEffort::None {
DEFAULT_THINKING_EFFORT
} else {
effort.as_str()
}
} else if let Some(value) = request.chat_options.template_kwargs.get("reasoning_effort") {
/// Resolve standard reasoning controls and validate K3's supported effort grades.
pub(super) fn resolve_reasoning(
request: &ChatRequest,
defaults: &HashMap<String, Value>,
) -> Result<ReasoningControl> {
let reasoning = ReasoningControl::resolve(request, defaults)?
.fallback(ReasoningControl::enabled(DEFAULT_THINKING_EFFORT));
if let Some(value) = reasoning.effort() {
let effort = value.as_str().ok_or_else(|| {
Error::ChatTemplate(format!(
"template kwarg `reasoning_effort` must be a string, got {value}"
Error::InvalidReasoningEffort(format!(
"Kimi K3 reasoning_effort must be a string, got {value}"
))
})?;
if effort == "none" {
DEFAULT_THINKING_EFFORT
} else {
effort
if !VALID_THINKING_EFFORTS.contains(&effort) {
return Err(Error::InvalidReasoningEffort(format!(
"unsupported reasoning_effort={effort:?}; supported values are `low`, `high`, and `max`"
)));
}
} else {
DEFAULT_THINKING_EFFORT
};
if !VALID_THINKING_EFFORTS.contains(&effort) {
return Err(Error::ChatTemplate(format!(
"unsupported thinking_effort={effort:?}; supported values are `low`, `high`, and `max`"
)));
}
Ok(effort.to_string())
Ok(reasoning)
}
fn content_is_empty(content: &ChatContent) -> bool {
rust/src/chat/src/renderer/kimi_k3/mod.rs
+26 -6
View File
@@ -7,10 +7,13 @@ mod encoding;
#[cfg(test)]
mod tests;
use std::collections::HashMap;
use serde_json::Value;
use vllm_text::Prompt;
use vllm_text::tokenizer::DynTokenizer;
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use super::{ChatRenderer, RenderedPrompt};
use crate::Result;
use crate::request::ChatRequest;
@@ -18,25 +21,42 @@ use crate::request::ChatRequest;
#[derive(Clone)]
pub struct KimiK3ChatRenderer {
tokenizer: DynTokenizer,
default_template_kwargs: HashMap<String, Value>,
}
impl KimiK3ChatRenderer {
/// Create a Kimi K3 renderer.
pub fn new(tokenizer: DynTokenizer) -> Self {
Self { tokenizer }
pub fn new(tokenizer: DynTokenizer, default_template_kwargs: HashMap<String, Value>) -> Self {
Self {
tokenizer,
default_template_kwargs,
}
}
}
impl ChatRenderer for KimiK3ChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
let (token_ids, media_order) =
encoding::render_request_with_media_order(request, self.tokenizer.as_ref())?;
let reasoning = encoding::resolve_reasoning(request, &self.default_template_kwargs)?;
let mut effective_template_kwargs = reasoning.template_kwargs(request);
// Export K3's native effort key from the resolved standard controls.
// Input `thinking_effort` is ignored; clear it when reasoning is disabled.
if let Some(effort) = reasoning.effort() {
effective_template_kwargs
.insert("thinking_effort".to_string(), serde_json::json!(effort));
} else {
effective_template_kwargs.remove("thinking_effort");
}
let (token_ids, media_order) = encoding::render_request_with_media_order(
request,
self.tokenizer.as_ref(),
&reasoning,
)?;
Ok(RenderedPrompt {
prompt: Prompt::TokenIds(token_ids),
media_order: Some(media_order),
effective_template_kwargs: request_template_kwargs(request),
effective_template_kwargs,
})
}
}
rust/src/chat/src/renderer/kimi_k3/tests.rs
+80 -19
View File
@@ -15,11 +15,10 @@ use vllm_tokenizer::test_utils::TestTokenizer;
use super::KimiK3ChatRenderer;
use crate::ChatRenderer;
use crate::EffortValue;
use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{
ChatContentPart, ChatMessage, ChatTool, GenerationPromptMode, ReasoningEffort,
};
use crate::request::{ChatContentPart, ChatMessage, ChatTool, GenerationPromptMode};
use crate::{AssistantContentBlock, AssistantToolCall};
const OPEN_ID: u32 = 256;
@@ -37,8 +36,70 @@ fn test_tokenizer() -> TestTokenizer {
.with_special_token(IMAGE_PLACEHOLDER, MEDIA_ID)
}
#[test]
fn standard_effort_controls_prompt_and_exported_native_effort() {
let tokenizer = Arc::new(test_tokenizer());
for (kwargs, typed, defaults, effort) in [
(json!({}), None, json!({}), "max"),
(
json!({}),
Some(EffortValue::from("low")),
json!({"reasoning_effort": "high"}),
"low",
),
(
json!({"enable_thinking": true, "reasoning_effort": "none"}),
None,
json!({"reasoning_effort": "low"}),
"low",
),
(
json!({}),
Some(EffortValue::from("none")),
json!({"reasoning_effort": "low"}),
"none",
),
(
json!({"thinking": true}),
Some(EffortValue::from("none")),
json!({}),
"max",
),
] {
let renderer =
KimiK3ChatRenderer::new(tokenizer.clone(), serde_json::from_value(defaults).unwrap());
let mut request = crate::ChatRequest::for_test();
request.chat_options.reasoning_effort = typed;
request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap();
let rendered = renderer.render(&request).unwrap();
let prompt = tokenizer.decode(&rendered.prompt.into_token_ids().unwrap(), false).unwrap();
let enabled = effort != "none";
assert_eq!(prompt.ends_with("<|open|>think<|sep|>"), enabled);
assert_eq!(
rendered.effective_template_kwargs["reasoning_effort"],
effort
);
assert_eq!(
rendered.effective_template_kwargs["enable_thinking"],
enabled
);
if enabled {
assert!(prompt.contains(&format!("thinking_effort={effort}")));
assert_eq!(
rendered.effective_template_kwargs["thinking_effort"],
effort
);
} else {
assert!(!rendered.effective_template_kwargs.contains_key("thinking_effort"));
}
}
}
fn render_token_ids(request: &crate::request::ChatRequest, tokenizer: DynTokenizer) -> Vec<u32> {
let prompt = KimiK3ChatRenderer::new(tokenizer).render(request).unwrap().prompt;
let prompt = KimiK3ChatRenderer::new(tokenizer, Default::default())
.render(request)
.unwrap()
.prompt;
let Prompt::TokenIds(token_ids) = prompt else {
panic!("kimi k3 renderer should return token IDs")
};
@@ -67,7 +128,11 @@ fn kimi_k3_fixture_options() -> FixtureRequestOptions {
fn assert_golden(name: &str) {
let input_name = format!("{name}_input.json");
let request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options());
let mut request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options());
// Reference fixtures use encoder-native keys; Rust accepts standard effort input.
if let Some(effort) = request.chat_options.template_kwargs.remove("thinking_effort") {
request.chat_options.reasoning_effort = Some(EffortValue::try_from(effort).unwrap());
}
let rendered = render_request(&request);
expect_file![format!("fixtures/{name}_output.txt")].assert_eq(&rendered);
}
@@ -310,7 +375,7 @@ fn media_order_follows_reordered_tool_results() {
),
];
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
let renderer = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()));
let renderer = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()), Default::default());
let rendered = renderer.render(&request).unwrap();
let media_order = rendered.media_order.unwrap();
@@ -371,19 +436,18 @@ fn translates_standard_thinking_kwargs() {
}
#[test]
fn native_k3_kwargs_take_precedence() {
fn thinking_alias_takes_precedence() {
let mut request = crate::request::ChatRequest::for_test();
request.chat_options.template_kwargs.extend([
("thinking".to_string(), json!(true)),
("enable_thinking".to_string(), json!(false)),
("thinking_effort".to_string(), json!("low")),
("reasoning_effort".to_string(), json!("high")),
]);
let rendered = render_request(&request);
assert!(rendered.contains("thinking_effort=low"));
assert!(!rendered.contains("thinking_effort=high"));
assert!(rendered.contains("thinking_effort=high"));
assert!(rendered.ends_with("<|open|>think<|sep|>"));
}
#[test]
@@ -431,7 +495,7 @@ fn enable_thinking_true_overrides_standard_none() {
#[test]
fn typed_none_disables_thinking() {
let mut request = crate::request::ChatRequest::for_test();
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
let rendered = render_request(&request);
@@ -440,20 +504,17 @@ fn typed_none_disables_thinking() {
}
#[test]
fn rejects_removed_medium_thinking_effort() {
fn rejects_unsupported_standard_reasoning_effort() {
let mut request = crate::request::ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("thinking_effort".to_string(), json!("medium"));
request.chat_options.reasoning_effort = Some(EffortValue::from("medium"));
let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()))
let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()), Default::default())
.render(&request)
.unwrap_err();
expect![[r#"
ChatTemplate(
"unsupported thinking_effort=\"medium\"; supported values are `low`, `high`, and `max`",
InvalidReasoningEffort(
"unsupported reasoning_effort=\"medium\"; supported values are `low`, `high`, and `max`",
)
"#]]
.assert_debug_eq(&error);
rust/src/chat/src/renderer/mod.rs
+2 -32
View File
@@ -4,11 +4,11 @@
use std::collections::HashMap;
use std::sync::Arc;
use serde_json::{Value, json};
use serde_json::Value;
use vllm_text::Prompt;
use crate::error::Result;
use crate::request::{ChatRequest, ReasoningEffort};
use crate::request::ChatRequest;
mod deepseek;
pub mod deepseek_v32;
@@ -62,33 +62,3 @@ pub trait ChatRenderer: Send + Sync {
/// Shared trait-object form of [`ChatRenderer`].
pub type DynChatRenderer = Arc<dyn ChatRenderer>;
/// Extract the effective chat-template kwargs visible to the renderer from the request,
/// using the provided defaults as the base.
pub(crate) fn effective_template_kwargs(
default_template_kwargs: &HashMap<String, Value>,
request: &ChatRequest,
) -> HashMap<String, Value> {
let mut kwargs = default_template_kwargs.clone();
kwargs.extend(request.chat_options.template_kwargs.clone());
if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
kwargs.insert(
"reasoning_effort".to_string(),
Value::String(reasoning_effort.as_str().to_string()),
);
if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
kwargs.insert(
"enable_thinking".to_string(),
json!(reasoning_effort != ReasoningEffort::None),
);
}
}
kwargs
}
/// Extract the effective chat-template kwargs visible to the renderer from the request.
pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap<String, Value> {
effective_template_kwargs(&HashMap::new(), request)
}
rust/src/chat/src/renderer/test_utils.rs
+3 -2
View File
@@ -8,10 +8,11 @@ use std::path::Path;
use serde::Deserialize;
use serde_json::Value;
use crate::EffortValue;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
GenerationPromptMode, ResolvedToolContext,
};
/// Options for constructing a [`ChatRequest`] from a fixture file.
@@ -47,7 +48,7 @@ pub(crate) struct FixtureRequest {
tools: Vec<FixtureTool>,
messages: Vec<FixtureMessage>,
add_generation_prompt: Option<bool>,
reasoning_effort: Option<ReasoningEffort>,
reasoning_effort: Option<EffortValue>,
/// Standard response format passed to model-specific renderers.
#[serde(default)]
response_format: Option<Value>,
rust/src/chat/src/request.rs
+6 -106
View File
@@ -11,9 +11,9 @@ pub use vllm_parser::tool::Tool as ChatTool;
use vllm_text::TextDecodeOptions;
pub use vllm_text::{PromptTruncation, SamplingParams};
use crate::AssistantMessageExt;
use crate::error::{Error, Result};
use crate::event::{AssistantContentBlock, AssistantMessage};
use crate::{AssistantMessageExt, EffortValue};
/// Role label for one text-only chat message.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -362,33 +362,6 @@ pub enum GenerationPromptMode {
NoGenerationPrompt,
}
/// Effort level for reasoning models.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ReasoningEffort {
None,
Minimal,
Low,
Medium,
High,
XHigh,
Max,
}
impl ReasoningEffort {
pub fn as_str(self) -> &'static str {
match self {
Self::None => "none",
Self::Minimal => "minimal",
Self::Low => "low",
Self::Medium => "medium",
Self::High => "high",
Self::XHigh => "xhigh",
Self::Max => "max",
}
}
}
/// Chat-template-related request options.
///
/// These are the small subset of chat controls that currently affect prompt
@@ -404,8 +377,10 @@ pub struct ChatOptions {
/// used instead of the model's default chat template.
pub chat_template: Option<String>,
/// Effort level exposed to chat templates for reasoning models.
pub reasoning_effort: Option<ReasoningEffort>,
/// Model-specific reasoning effort, typically `none`, `minimal`, `low`,
/// `medium`, `high`, `xhigh`, or `max`. Supported names and numeric ranges
/// are validated by the selected renderer or HF template.
pub reasoning_effort: Option<EffortValue>,
/// Standard response format available to model-specific renderers.
#[serde(default)]
@@ -673,29 +648,6 @@ impl ChatRequest {
self.tool_context.parsing_enabled()
}
/// Return the request-level thinking toggle when explicitly requested.
///
/// We currently accept the two request kwargs `thinking` and
/// `enable_thinking`. Both must be booleans when present. If both are
/// present, they must have the same value. If neither key is provided,
/// return `None`.
pub(crate) fn enable_thinking(&self) -> Result<Option<bool>> {
let thinking = self.parse_template_bool("thinking")?;
let enable_thinking = self.parse_template_bool("enable_thinking")?;
match (thinking, enable_thinking) {
(None, None) => Ok(None),
(Some(thinking), Some(enable_thinking)) if thinking != enable_thinking => {
Err(Error::ChatTemplate(
"template kwargs `thinking` and `enable_thinking` must match when both are set"
.to_string(),
))
}
(Some(thinking), _) => Ok(Some(thinking)),
(None, Some(enable_thinking)) => Ok(Some(enable_thinking)),
}
}
pub(crate) fn parse_template_bool(&self, key: &str) -> Result<Option<bool>> {
match self.chat_options.template_kwargs.get(key) {
None => Ok(None),
@@ -726,7 +678,7 @@ mod tests {
use serde_json::{json, to_value};
use super::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
ChatContent, ChatContentPart, ChatMessage, ChatRole, ChatTool, ChatToolChoice,
ResolvedToolContext,
};
use crate::Error;
@@ -919,56 +871,4 @@ mod tests {
Error::ToolChoiceFunctionNotFound { name } if name == "missing"
));
}
#[test]
fn enable_thinking_is_none_when_no_kwargs_are_present() {
let request = ChatRequest::for_test();
assert_eq!(request.enable_thinking().unwrap(), None);
}
#[test]
fn enable_thinking_accepts_matching_duplicate_kwargs() {
let mut request = ChatRequest::for_test();
request.chat_options.template_kwargs.insert("thinking".to_string(), json!(true));
request
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), json!(true));
assert_eq!(request.enable_thinking().unwrap(), Some(true));
}
#[test]
fn enable_thinking_rejects_non_boolean_kwargs() {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("thinking".to_string(), json!("yes"));
assert!(matches!(
request.enable_thinking(),
Err(Error::ChatTemplate(message))
if message.contains("`thinking` must be a boolean")
));
}
#[test]
fn enable_thinking_rejects_conflicting_duplicate_kwargs() {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("thinking".to_string(), json!(false));
request
.chat_options
.template_kwargs
.insert("enable_thinking".to_string(), json!(true));
assert!(matches!(
request.enable_thinking(),
Err(Error::ChatTemplate(message))
if message.contains("`thinking` and `enable_thinking` must match")
));
}
}
rust/src/server/src/error.rs
+16 -10
View File
@@ -175,16 +175,22 @@ mod tests {
}
#[test]
fn invalid_reasoning_effort_maps_to_invalid_request() {
let error = vllm_chat::Error::InvalidReasoningEffort(
"DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(),
);
let api_error = chat_submit_error("failed to submit chat request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
assert_eq!(
api_error.to_error_response().error.error_type,
"invalid_request_error"
);
fn invalid_reasoning_parameters_map_to_invalid_request() {
for error in [
vllm_chat::Error::InvalidReasoningEffort(
"DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(),
),
vllm_chat::Error::InvalidReasoningControl {
message: "template kwarg `thinking` must be a boolean".to_string(),
},
] {
let api_error = chat_submit_error("failed to submit chat request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
assert_eq!(
api_error.to_error_response().error.error_type,
"invalid_request_error"
);
}
}
#[test]
rust/src/server/src/routes/openai/chat_completions/convert.rs
+46 -2
View File
@@ -180,7 +180,7 @@ pub(super) fn prepare_chat_request(
chat_options: ChatOptions {
generation_prompt_mode,
chat_template: request.chat_template,
reasoning_effort: request.reasoning_effort,
reasoning_effort: request.reasoning_effort.map(|effort| effort.as_str().into()),
response_format,
template_kwargs,
},
@@ -488,6 +488,50 @@ mod tests {
}
}
#[test]
fn chat_http_reasoning_effort_preserves_omission_none_and_kwargs() {
for effort in [None, Some(json!(null))].into_iter().chain(
["none", "minimal", "low", "medium", "high", "xhigh", "max"]
.map(|name| Some(json!(name))),
) {
let kwargs = json!({"reasoning_effort": 37, "thinking": true});
let mut value = json!({
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": kwargs,
});
if let Some(effort) = &effort {
value["reasoning_effort"] = effort.clone();
}
let request: ChatCompletionRequest = serde_json::from_value(value).unwrap();
let prepared = prepare_chat_request(
request,
&served(&["test-model"]),
request_context(&HeaderMap::new(), None),
)
.unwrap();
let options = prepared.chat_request.chat_options;
assert_eq!(
serde_json::to_value(options.reasoning_effort).unwrap(),
effort.unwrap_or(serde_json::Value::Null)
);
assert_eq!(
serde_json::to_value(options.template_kwargs).unwrap(),
kwargs
);
}
}
#[test]
fn chat_http_reasoning_effort_rejects_model_extensions_at_top_level() {
for effort in [json!(37), json!("custom")] {
let request = json!({
"messages": [{"role": "user", "content": "hello"}],
"reasoning_effort": effort,
});
assert!(serde_json::from_value::<ChatCompletionRequest>(request).is_err());
}
}
#[test]
fn chat_http_request_defaults_missing_or_null_model() {
for model in [None, Some(serde_json::Value::Null)] {
@@ -747,7 +791,7 @@ mod tests {
);
let tokenizer = Arc::new(TestTokenizer::new());
let prompt = KimiK3ChatRenderer::new(tokenizer.clone())
let prompt = KimiK3ChatRenderer::new(tokenizer.clone(), Default::default())
.render(&prepared.chat_request)
.expect("Kimi K3 rendering succeeds")
.prompt;
rust/src/server/src/routes/openai/chat_completions/types.rs
+4 -4
View File
@@ -9,15 +9,15 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_with::SerializeDisplay;
use validator::Validate;
use vllm_chat::ReasoningEffort;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_text::TruncationSide;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope,
StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage, default_true,
deserialize_request_top_k, validate_messages, validate_stop, validate_top_p_value,
ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, ReasoningEffort, StreamOptions,
StreamResponseEnvelope, StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage,
default_true, deserialize_request_top_k, validate_messages, validate_stop,
validate_top_p_value,
};
/// vLLM-compatible request type for the Chat Completions API.
rust/src/server/src/routes/openai/chat_completions/validate.rs
+1 -1
View File
@@ -125,8 +125,8 @@ fn validate_function_tools(tools: &[Tool], param: &'static str) -> Result<(), Ap
mod tests {
use std::collections::HashMap;
use crate::routes::openai::utils::types::ReasoningEffort;
use serde_json::json;
use vllm_chat::ReasoningEffort;
use super::validate_request_compat;
use crate::routes::openai::chat_completions::types::ChatCompletionRequest;
rust/src/server/src/routes/openai/utils/types.rs
+30
View File
@@ -39,6 +39,36 @@ where
}
}
/// Effort level for reasoning models.
///
/// Fixed OpenAI HTTP request grades. Request conversion maps these names to
/// the model-independent `vllm_chat::EffortValue` before renderer validation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ReasoningEffort {
None,
Minimal,
Low,
Medium,
High,
XHigh,
Max,
}
impl ReasoningEffort {
pub fn as_str(self) -> &'static str {
match self {
Self::None => "none",
Self::Minimal => "minimal",
Self::Low => "low",
Self::Medium => "medium",
Self::High => "high",
Self::XHigh => "xhigh",
Self::Max => "max",
}
}
}
// ============================================================================
// String/Array Utilities
// ============================================================================