diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e1c051a6fe..7820a7b676 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5049,6 +5049,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "thiserror-ext", + "time", "tokio", "tracing", "tracing-subscriber", diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index cb28b1e9c1..40498bac3f 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -25,6 +25,7 @@ strum.workspace = true subenum.workspace = true thiserror.workspace = true thiserror-ext.workspace = true +time.workspace = true tokio.workspace = true tracing.workspace = true trait-set.workspace = true diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 77ed24de85..9dff25ea49 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -15,7 +15,9 @@ use crate::output::{ DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides, }; use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo}; -use crate::renderer::{DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer}; +use crate::renderer::{ + DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer, +}; use crate::request::ChatRequest; use crate::{DynChatOutputProcessor, RendererSelection}; @@ -61,6 +63,7 @@ impl HfChatBackend { )?), RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()), RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()), + RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?), }; info!( @@ -148,13 +151,15 @@ mod tests { use std::sync::Arc; use tempfile::tempdir; + use thiserror_ext::AsReport as _; + use vllm_text::Prompt; use vllm_text::backend::hf::TokenizerSource; use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; use super::HfChatBackend; - use crate::RendererSelection; - use crate::backend::{ChatBackend, LoadModelBackendsOptions}; + use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions}; use crate::request::{ChatContent, ChatMessage, ChatRequest}; + use crate::{ParserSelection, RendererSelection}; fn request_with_user_text(text: &str) -> ChatRequest { ChatRequest { @@ -219,12 +224,12 @@ mod tests { Arc::new(TestTokenizer) } - fn render_prompt( + fn backend_for_selection( renderer: RendererSelection, config_json: &str, tokenizer_config_json: &str, - ) -> String { - let backend = HfChatBackend::from_resolved_model_files( + ) -> HfChatBackend { + HfChatBackend::from_resolved_model_files( resolved_files(config_json, tokenizer_config_json), "test-model".to_string(), LoadModelBackendsOptions { @@ -236,9 +241,15 @@ mod tests { }, test_tokenizer(), ) - .unwrap(); + .unwrap() + } - backend + fn render_prompt( + renderer: RendererSelection, + config_json: &str, + tokenizer_config_json: &str, + ) -> String { + backend_for_selection(renderer, config_json, tokenizer_config_json) .chat_renderer() .render(&request_with_user_text("hello")) .unwrap() @@ -272,6 +283,35 @@ mod tests { assert_eq!(prompt, "hello"); } + #[test] + fn auto_uses_harmony_renderer_and_output_processor_for_gpt_oss_model_type() { + let backend = backend_for_selection( + RendererSelection::Auto, + r#"{"model_type":"gpt_oss"}"#, + r#"{"chat_template":"{{ messages[0].content }}"}"#, + ); + + let prompt = + backend.chat_renderer().render(&request_with_user_text("hello")).unwrap().prompt; + assert!(matches!(prompt, Prompt::TokenIds(_))); + + let mut request = request_with_user_text("hello"); + let error = match backend.new_chat_output_processor( + &mut request, + NewChatOutputProcessorOptions { + tool_call_parser: &ParserSelection::Explicit("json".to_string()), + reasoning_parser: &ParserSelection::Auto, + }, + ) { + Ok(_) => panic!("gpt_oss should reject generic parser overrides"), + Err(error) => error, + }; + assert_eq!( + error.to_report_string(), + "gpt_oss uses native Harmony output parsing; generic tool parser override `json` is not supported" + ); + } + #[test] fn language_model_only_skips_multimodal_preprocessor_config() { let mut files = resolved_files( diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 06e46f64b8..c16921ea75 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -29,8 +29,8 @@ pub use parser::reasoning::{ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory}; pub use renderer::hf::ChatTemplateContentFormatOption; pub use renderer::{ - ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, RenderedPrompt, - RendererSelection, + ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, + HarmonyChatRenderer, RenderedPrompt, RendererSelection, }; pub use request::{ ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool, diff --git a/rust/src/chat/src/output/harmony/mod.rs b/rust/src/chat/src/output/harmony/mod.rs index 4209dc0735..597e313379 100644 --- a/rust/src/chat/src/output/harmony/mod.rs +++ b/rust/src/chat/src/output/harmony/mod.rs @@ -4,16 +4,10 @@ //! `DecodedTextEvent` token IDs directly and lets the official `openai-harmony` //! parser recover the structured assistant message shape at token granularity. -use std::sync::LazyLock; - -use anyhow::Context; use asynk_strim_attr::{TryYielder, try_stream}; use futures::StreamExt as _; use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role}; -use openai_harmony::{ - HarmonyEncoding, HarmonyEncodingName, StreamableParser, load_harmony_encoding, -}; -use thiserror_ext::AsReport; +use openai_harmony::{HarmonyEncoding, StreamableParser}; use vllm_text::output::DecodedTextEvent; use crate::Result as ChatResult; @@ -24,6 +18,7 @@ use crate::output::{ generate_tool_call_id, }; use crate::parser::ParserSelection; +use crate::renderer::harmony::encoding::harmony_encoding; use crate::request::ChatRequest; /// Request-scoped Harmony output processor used for `model_type == "gpt_oss"`. @@ -384,18 +379,6 @@ async fn harmony_assistant_event_stream( Ok(()) } -/// Lazily load the shared GPT-OSS Harmony encoding once per process. -fn harmony_encoding() -> Result<&'static HarmonyEncoding> { - static ENCODING: LazyLock> = LazyLock::new(|| { - load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss) - .context("failed to load harmony encoding for gpt-oss") - }); - - ENCODING.as_ref().map_err(|error| Error::HarmonyOutputParsing { - error: error.to_report_string().into(), - }) -} - fn harmony_output_parsing_error( error: impl Into>, ) -> Error { diff --git a/rust/src/chat/src/renderer/harmony/encoding.rs b/rust/src/chat/src/renderer/harmony/encoding.rs new file mode 100644 index 0000000000..3b8030292d --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/encoding.rs @@ -0,0 +1,21 @@ +//! Shared Harmony encoding helper for the GPT-OSS renderer and output parser. + +use std::sync::LazyLock; + +use anyhow::Context as _; +use openai_harmony::{HarmonyEncoding, HarmonyEncodingName, load_harmony_encoding}; +use thiserror_ext::AsReport as _; + +use crate::error::{Error, Result}; + +/// Lazily load the shared GPT-OSS Harmony encoding once per process. +pub(crate) fn harmony_encoding() -> Result<&'static HarmonyEncoding> { + static ENCODING: LazyLock> = LazyLock::new(|| { + load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss) + .context("failed to load harmony encoding for gpt-oss") + }); + + ENCODING.as_ref().map_err(|error| Error::HarmonyOutputParsing { + error: error.to_report_string().into(), + }) +} diff --git a/rust/src/chat/src/renderer/harmony/fixtures/assistant_history.json b/rust/src/chat/src/renderer/harmony/fixtures/assistant_history.json new file mode 100644 index 0000000000..50edd03ee4 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/assistant_history.json @@ -0,0 +1,14 @@ +{ + "add_generation_prompt": false, + "messages": [ + { + "role": "user", + "content": "What is 2 + 2?" + }, + { + "role": "assistant", + "reasoning_content": "Need simple arithmetic.", + "content": "4" + } + ] +} diff --git a/rust/src/chat/src/renderer/harmony/fixtures/assistant_history.txt b/rust/src/chat/src/renderer/harmony/fixtures/assistant_history.txt new file mode 100644 index 0000000000..dc08897e22 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/assistant_history.txt @@ -0,0 +1,7 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>4<|end|> diff --git a/rust/src/chat/src/renderer/harmony/fixtures/developer_tools.json b/rust/src/chat/src/renderer/harmony/fixtures/developer_tools.json new file mode 100644 index 0000000000..4516e3b32b --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/developer_tools.json @@ -0,0 +1,27 @@ +[ + { + "role": "developer", + "content": "Use tools when needed.", + "tools": [ + { + "function": { + "name": "lookup", + "description": "Lookup a record.", + "parameters": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": ["id"] + } + } + } + ] + }, + { + "role": "user", + "content": "Find record abc." + } +] diff --git a/rust/src/chat/src/renderer/harmony/fixtures/developer_tools.txt b/rust/src/chat/src/renderer/harmony/fixtures/developer_tools.txt new file mode 100644 index 0000000000..a63b447f29 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/developer_tools.txt @@ -0,0 +1,23 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message. +Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions + +Use tools when needed. + +# Tools + +## functions + +namespace functions { + +// Lookup a record. +type lookup = (_: { +id: string, +}) => any; + +} // namespace functions<|end|><|start|>user<|message|>Find record abc.<|end|><|start|>assistant diff --git a/rust/src/chat/src/renderer/harmony/fixtures/drop_analysis.json b/rust/src/chat/src/renderer/harmony/fixtures/drop_analysis.json new file mode 100644 index 0000000000..75fd5d8b29 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/drop_analysis.json @@ -0,0 +1,15 @@ +[ + { + "role": "user", + "content": "What is 2 + 2?" + }, + { + "role": "assistant", + "reasoning_content": "This should be dropped.", + "content": "4" + }, + { + "role": "user", + "content": "What is 3 + 5?" + } +] diff --git a/rust/src/chat/src/renderer/harmony/fixtures/drop_analysis.txt b/rust/src/chat/src/renderer/harmony/fixtures/drop_analysis.txt new file mode 100644 index 0000000000..9e967b7956 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/drop_analysis.txt @@ -0,0 +1,7 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>4<|end|><|start|>user<|message|>What is 3 + 5?<|end|><|start|>assistant diff --git a/rust/src/chat/src/renderer/harmony/fixtures/leading_system.json b/rust/src/chat/src/renderer/harmony/fixtures/leading_system.json new file mode 100644 index 0000000000..5ff190d0b8 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/leading_system.json @@ -0,0 +1,13 @@ +{ + "reasoning_effort": "high", + "messages": [ + { + "role": "system", + "content": "Answer tersely." + }, + { + "role": "user", + "content": "What is 2 + 2?" + } + ] +} diff --git a/rust/src/chat/src/renderer/harmony/fixtures/leading_system.txt b/rust/src/chat/src/renderer/harmony/fixtures/leading_system.txt new file mode 100644 index 0000000000..e656a0a0a4 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/leading_system.txt @@ -0,0 +1,9 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: high + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions + +Answer tersely.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant diff --git a/rust/src/chat/src/renderer/harmony/fixtures/request_tools.json b/rust/src/chat/src/renderer/harmony/fixtures/request_tools.json new file mode 100644 index 0000000000..db5988182f --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/request_tools.json @@ -0,0 +1,26 @@ +{ + "tools": [ + { + "function": { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string" + } + }, + "required": ["city"] + }, + "strict": true + } + } + ], + "messages": [ + { + "role": "user", + "content": "Check Hangzhou weather." + } + ] +} diff --git a/rust/src/chat/src/renderer/harmony/fixtures/request_tools.txt b/rust/src/chat/src/renderer/harmony/fixtures/request_tools.txt new file mode 100644 index 0000000000..f31bc449bf --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/request_tools.txt @@ -0,0 +1,19 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message. +Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Tools + +## functions + +namespace functions { + +// Get weather for a city. +type get_weather = (_: { +city: string, +}) => any; + +} // namespace functions<|end|><|start|>user<|message|>Check Hangzhou weather.<|end|><|start|>assistant diff --git a/rust/src/chat/src/renderer/harmony/fixtures/simple_user.json b/rust/src/chat/src/renderer/harmony/fixtures/simple_user.json new file mode 100644 index 0000000000..b8b7f597d6 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/simple_user.json @@ -0,0 +1,6 @@ +[ + { + "role": "user", + "content": "Hello, who are you?" + } +] diff --git a/rust/src/chat/src/renderer/harmony/fixtures/simple_user.txt b/rust/src/chat/src/renderer/harmony/fixtures/simple_user.txt new file mode 100644 index 0000000000..7e44ca314c --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/simple_user.txt @@ -0,0 +1,7 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>Hello, who are you?<|end|><|start|>assistant diff --git a/rust/src/chat/src/renderer/harmony/fixtures/system_instructions_env.txt b/rust/src/chat/src/renderer/harmony/fixtures/system_instructions_env.txt new file mode 100644 index 0000000000..8ad0ac7d0e --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/system_instructions_env.txt @@ -0,0 +1,8 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Answer tersely. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: high + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant diff --git a/rust/src/chat/src/renderer/harmony/fixtures/tool_roundtrip.json b/rust/src/chat/src/renderer/harmony/fixtures/tool_roundtrip.json new file mode 100644 index 0000000000..00ccae641e --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/tool_roundtrip.json @@ -0,0 +1,43 @@ +{ + "tools": [ + { + "function": { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string" + } + }, + "required": ["city"] + } + } + } + ], + "messages": [ + { + "role": "user", + "content": "Check Hangzhou weather." + }, + { + "role": "assistant", + "reasoning_content": "Need current weather.", + "tool_calls": [ + { + "id": "call-weather", + "function": { + "name": "get_weather", + "arguments": "{\"city\":\"Hangzhou\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call-weather", + "content": "{\"temperature\":20}" + } + ] +} diff --git a/rust/src/chat/src/renderer/harmony/fixtures/tool_roundtrip.txt b/rust/src/chat/src/renderer/harmony/fixtures/tool_roundtrip.txt new file mode 100644 index 0000000000..0e06a4d107 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/fixtures/tool_roundtrip.txt @@ -0,0 +1,19 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message. +Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Tools + +## functions + +namespace functions { + +// Get weather for a city. +type get_weather = (_: { +city: string, +}) => any; + +} // namespace functions<|end|><|start|>user<|message|>Check Hangzhou weather.<|end|><|start|>assistant<|channel|>analysis<|message|>Need current weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{"city":"Hangzhou"}<|call|><|start|>functions.get_weather<|channel|>commentary to=assistant<|message|>{"temperature":20}<|end|><|start|>assistant diff --git a/rust/src/chat/src/renderer/harmony/mod.rs b/rust/src/chat/src/renderer/harmony/mod.rs new file mode 100644 index 0000000000..70a1bb063e --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/mod.rs @@ -0,0 +1,487 @@ +//! Native Harmony chat renderer for `gpt_oss`. + +pub(crate) mod encoding; + +use openai_harmony::HarmonyEncoding; +use openai_harmony::chat::{ + Author, Conversation, DeveloperContent, Message, ReasoningEffort as HarmonyReasoningEffort, + Role, SystemContent, ToolDescription, +}; +use thiserror_ext::AsReport as _; +use time::macros::format_description; +use vllm_text::Prompt; + +use self::encoding::harmony_encoding; +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use crate::error::{Error, Result}; +use crate::event::AssistantContentBlock; +use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, GenerationPromptMode}; +use crate::{AssistantMessageExt as _, ReasoningEffort}; + +const SYSTEM_START_DATE_ENV: &str = "VLLM_SYSTEM_START_DATE"; +const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS"; + +/// GPT-OSS renderer backed by the official Harmony encoding. +pub struct HarmonyChatRenderer { + encoding: &'static HarmonyEncoding, + options: Options, +} + +struct Options { + system_start_date: String, + use_system_instructions: bool, +} + +impl HarmonyChatRenderer { + /// Create a Harmony renderer for production use. + /// + /// Environment-derived options are resolved once at construction time: + /// + /// - `VLLM_SYSTEM_START_DATE` pins the Harmony system start date. When it is + /// unset, the renderer uses the current local date with a UTC fallback. + /// - `VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS` moves leading instructions + /// into the system model identity when set to a non-zero integer. + pub fn new() -> Result { + Self::with_options( + env_system_start_date(), + env_use_harmony_system_instructions(), + ) + } + + /// Create a Harmony renderer with explicit preamble options. + /// + /// Tests use this constructor to avoid process-global environment mutation. + /// Production code should call [`Self::new`] so the renderer observes the + /// same environment contract as the Python Harmony path. + pub fn with_options( + system_start_date: impl Into, + use_system_instructions: bool, + ) -> Result { + Ok(Self { + encoding: harmony_encoding()?, + options: Options { + system_start_date: system_start_date.into(), + use_system_instructions, + }, + }) + } + + /// Render a chat request directly to Harmony token IDs. + /// + /// Harmony owns both prompt formatting and tokenization, so the Rust + /// frontend bypasses the generic HF tokenizer path for GPT-OSS input. + fn render_token_ids(&self, request: &ChatRequest) -> Result> { + if request.has_multimodal() { + return Err(Error::UnsupportedMultimodalContent("image_url")); + } + if matches!( + request.chat_options.generation_prompt_mode, + GenerationPromptMode::ContinueFinalAssistant + ) { + return Err(Error::ChatTemplate( + "Harmony renderer does not support continue_final_message".to_string(), + )); + } + + let messages = auto_drop_analysis_messages(to_harmony_messages(request, &self.options)?); + let conversation = Conversation::from_messages(messages); + // Pass `None` so oss-harmony does not apply its narrower built-in + // analysis-drop policy after the Rust-side Python-parity cleanup above. + let token_ids = match request.chat_options.generation_prompt_mode { + GenerationPromptMode::StartNewAssistant => self + .encoding + .render_conversation_for_completion(&conversation, Role::Assistant, None), + GenerationPromptMode::NoGenerationPrompt => { + self.encoding.render_conversation(&conversation, None) + } + GenerationPromptMode::ContinueFinalAssistant => unreachable!("checked above"), + } + .map_err(|error| { + Error::ChatTemplate(format!( + "failed to render Harmony prompt: {}", + error.as_report() + )) + })?; + + Ok(token_ids) + } +} + +impl ChatRenderer for HarmonyChatRenderer { + /// Render a chat request as [`Prompt::TokenIds`] with template kwargs echoed + /// for downstream accounting/debugging. + fn render(&self, request: &ChatRequest) -> Result { + Ok(RenderedPrompt { + prompt: Prompt::TokenIds(self.render_token_ids(request)?), + effective_template_kwargs: request_template_kwargs(request), + }) + } +} + +/// Convert a vLLM chat request into a full Harmony conversation. +/// +/// This adds the Harmony system/developer preamble, peels at most one leading +/// system/developer instruction message, and then lowers the remaining chat +/// history message-by-message. +fn to_harmony_messages(request: &ChatRequest, options: &Options) -> Result> { + let (instructions, leading_developer_tools, remaining_messages) = + peel_leading_instructions(&request.messages)?; + let tool_call_names = tool_call_names(&request.messages); + let mut messages = + build_harmony_preamble(request, instructions, leading_developer_tools, options)?; + + for message in remaining_messages { + messages.extend(to_harmony_message(message, &tool_call_names, options)?); + } + + Ok(messages) +} + +/// Extract the optional leading instruction message used by the Harmony preamble. +/// +/// Python only peels the first leading `system` or `developer` message. Later +/// system/developer messages stay in the conversation and are lowered normally. +#[allow(clippy::type_complexity)] +fn peel_leading_instructions( + messages: &[ChatMessage], +) -> Result<(Option, Option<&[ChatTool]>, &[ChatMessage])> { + let Some(first) = messages.first() else { + return Ok((None, None, messages)); + }; + + match first { + ChatMessage::System { content } => Ok((Some(flatten_text(content)?), None, &messages[1..])), + ChatMessage::Developer { content, tools } => Ok(( + Some(flatten_text(content)?), + tools.as_deref(), + &messages[1..], + )), + ChatMessage::User { .. } + | ChatMessage::Assistant { .. } + | ChatMessage::ToolResponse { .. } => Ok((None, None, messages)), + } +} + +/// Build the Harmony preamble for one request. +/// +/// The preamble always contains a system message with date and reasoning-effort +/// metadata. Leading instructions live either in the system model identity or in +/// a developer message depending on `use_system_instructions`; request-level and +/// leading developer tools are attached to the developer message. +fn build_harmony_preamble( + request: &ChatRequest, + instructions: Option, + leading_developer_tools: Option<&[ChatTool]>, + options: &Options, +) -> Result> { + let mut messages = vec![Message::from_role_and_content( + Role::System, + system_content( + instructions.as_deref().filter(|_| options.use_system_instructions), + request.chat_options.reasoning_effort, + &options.system_start_date, + )?, + )]; + + let mut developer = DeveloperContent::new(); + let mut has_developer_content = false; + + if !options.use_system_instructions + && let Some(instructions) = instructions.as_deref().filter(|text| !text.is_empty()) + { + developer = developer.with_instructions(instructions); + has_developer_content = true; + } + + let tool_descriptions = preamble_tool_descriptions(request, leading_developer_tools); + if !tool_descriptions.is_empty() { + developer = developer.with_function_tools(tool_descriptions); + has_developer_content = true; + } + + if has_developer_content { + messages.push(Message::from_role_and_content(Role::Developer, developer)); + } + + Ok(messages) +} + +/// Collect request-level and leading developer function tools for the preamble. +fn preamble_tool_descriptions( + request: &ChatRequest, + leading_developer_tools: Option<&[ChatTool]>, +) -> Vec { + let mut tools = Vec::new(); + if request.tool_parsing_enabled() { + tools.extend(to_tool_descriptions(&request.tools)); + } + if let Some(leading_developer_tools) = leading_developer_tools { + tools.extend(to_tool_descriptions(leading_developer_tools)); + } + tools +} + +/// Construct the Harmony system content for the request preamble. +/// +/// Harmony defaults the reasoning effort to `medium` when none is provided, so +/// this only sets an explicit effort after validating vLLM's request value. +fn system_content( + instructions: Option<&str>, + reasoning_effort: Option, + system_start_date: &str, +) -> Result { + let mut content = + SystemContent::new().with_conversation_start_date(system_start_date.to_string()); + + if let Some(reasoning_effort) = reasoning_effort { + content = content.with_reasoning_effort(to_harmony_reasoning_effort(reasoning_effort)?); + } + + if let Some(instructions) = instructions.filter(|text| !text.is_empty()) { + let model_identity = match content.model_identity.as_deref() { + Some(identity) if !identity.is_empty() => format!("{identity}\n{instructions}"), + _ => instructions.to_string(), + }; + content = content.with_model_identity(model_identity); + } + + Ok(content) +} + +/// Lower a single vLLM chat message into one or more Harmony messages. +/// +/// Assistant messages can split into separate analysis, final, commentary, and +/// tool-call messages. Tool responses require the earlier assistant tool-call ID +/// map so the Harmony tool author can include `functions.{name}`. +fn to_harmony_message( + message: &ChatMessage, + tool_call_names: &std::collections::HashMap, + options: &Options, +) -> Result> { + Ok(match message { + ChatMessage::System { content } => { + let instructions = flatten_text(content)?; + vec![system_or_developer_message( + "system", + instructions, + None, + options, + )?] + } + ChatMessage::Developer { content, tools } => { + let instructions = flatten_text(content)?; + vec![developer_message(Some(instructions), tools.as_deref())] + } + ChatMessage::User { content } => { + vec![Message::from_role_and_content( + Role::User, + flatten_text(content)?, + )] + } + ChatMessage::Assistant { content } => assistant_messages(content), + ChatMessage::ToolResponse { + content, + tool_call_id, + } => { + let name = tool_call_names.get(tool_call_id).ok_or_else(|| { + Error::ChatTemplate(format!( + "invalid Harmony tool message: unknown tool_call_id `{tool_call_id}`" + )) + })?; + vec![ + Message::from_author_and_content( + Author::new(Role::Tool, format!("functions.{name}")), + flatten_text(content)?, + ) + .with_channel("commentary") + .with_recipient("assistant"), + ] + } + }) +} + +/// Lower a non-leading system/developer message. +/// +/// Harmony treats most extra system/developer messages as developer +/// instructions. When system-instructions mode is enabled, system messages are +/// rendered as system model-identity additions to match Python. +fn system_or_developer_message( + role: &str, + instructions: String, + tools: Option<&[ChatTool]>, + options: &Options, +) -> Result { + if role == "system" && options.use_system_instructions { + return Ok(Message::from_role_and_content( + Role::System, + system_content(Some(&instructions), None, &options.system_start_date)?, + )); + } + + Ok(developer_message(Some(instructions), tools)) +} + +/// Build a Harmony developer message with optional instructions and function tools. +fn developer_message(instructions: Option, tools: Option<&[ChatTool]>) -> Message { + let mut content = DeveloperContent::new(); + if let Some(instructions) = instructions.filter(|text| !text.is_empty()) { + content = content.with_instructions(instructions); + } + if let Some(tools) = tools { + let tools = to_tool_descriptions(tools); + if !tools.is_empty() { + content = content.with_function_tools(tools); + } + } + Message::from_role_and_content(Role::Developer, content) +} + +/// Lower assistant history into Harmony channels. +/// +/// Plain assistant text goes to `final`. When the assistant has tool calls, +/// visible text goes to `commentary`, reasoning goes to `analysis`, and each +/// function call becomes a `commentary` message to `functions.{name}` with JSON +/// constrained content. +fn assistant_messages(content: &[AssistantContentBlock]) -> Vec { + let mut messages = Vec::new(); + let has_tool_calls = content.has_tool_calls(); + + if has_tool_calls { + let text = content.text(); + if !text.is_empty() { + messages.push( + Message::from_role_and_content(Role::Assistant, text).with_channel("commentary"), + ); + } + } + + if let Some(reasoning) = content.reasoning() { + messages.push( + Message::from_role_and_content(Role::Assistant, reasoning).with_channel("analysis"), + ); + } + + if has_tool_calls { + for tool_call in content.tool_calls() { + messages.push( + Message::from_role_and_content(Role::Assistant, tool_call.arguments.clone()) + .with_channel("commentary") + .with_recipient(format!("functions.{}", tool_call.name)) + .with_content_type("<|constrain|>json"), + ); + } + } else { + let text = content.text(); + if !text.is_empty() { + messages + .push(Message::from_role_and_content(Role::Assistant, text).with_channel("final")); + } + } + + messages +} + +/// Build the tool-call ID to function-name map used by later tool responses. +fn tool_call_names(messages: &[ChatMessage]) -> std::collections::HashMap { + let mut names = std::collections::HashMap::new(); + for message in messages { + let ChatMessage::Assistant { content } = message else { + continue; + }; + for tool_call in content.tool_calls() { + names.insert(tool_call.id.clone(), tool_call.name.clone()); + } + } + names +} + +/// Drop stale assistant analysis messages using vLLM Python's policy. +/// +/// Once an assistant final message exists, earlier analysis messages represent +/// chain-of-thought for completed turns and should not be replayed to the model. +fn auto_drop_analysis_messages(messages: Vec) -> Vec { + // Match vLLM Python's Harmony cleanup: once an assistant final message exists, + // previous assistant analysis messages are stale chain-of-thought and should + // be removed. oss-harmony can also drop analysis with `Some(Default::default())`, + // but that built-in path only triggers when the last assistant message is final + // and drops relative to the first final message, which misses longer multi-turn + // histories with later user/tool turns. + let Some(last_assistant_final_index) = messages.iter().rposition(|message| { + message.author.role == Role::Assistant && message.channel.as_deref() == Some("final") + }) else { + return messages; + }; + + messages + .into_iter() + .enumerate() + .filter_map(|(index, message)| { + (index >= last_assistant_final_index || message.channel.as_deref() != Some("analysis")) + .then_some(message) + }) + .collect() +} + +/// Flatten vLLM text content and reject unsupported multimodal parts. +fn flatten_text(content: &ChatContent) -> Result { + content.try_flatten_to_text() +} + +/// Convert vLLM function tool definitions to Harmony tool descriptions. +fn to_tool_descriptions(tools: &[ChatTool]) -> Vec { + tools + .iter() + .map(|tool| { + ToolDescription::new( + tool.name.clone(), + tool.description.clone().unwrap_or_default(), + Some(tool.parameters.clone()), + ) + }) + .collect() +} + +/// Map supported OpenAI reasoning-effort values onto Harmony's enum. +fn to_harmony_reasoning_effort( + reasoning_effort: ReasoningEffort, +) -> Result { + match reasoning_effort { + ReasoningEffort::Low => Ok(HarmonyReasoningEffort::Low), + ReasoningEffort::Medium => Ok(HarmonyReasoningEffort::Medium), + ReasoningEffort::High => Ok(HarmonyReasoningEffort::High), + ReasoningEffort::None + | ReasoningEffort::Minimal + | ReasoningEffort::XHigh + | ReasoningEffort::Max => Err(Error::ChatTemplate(format!( + "reasoning_effort={:?} is not supported by Harmony. Supported values are: low, medium, high.", + reasoning_effort.as_str() + ))), + } +} + +/// Resolve the system start date from the environment or the current date. +fn env_system_start_date() -> String { + std::env::var(SYSTEM_START_DATE_ENV) + .ok() + .filter(|date| !date.is_empty()) + .unwrap_or_else(current_date) +} + +/// Format today's date as `YYYY-MM-DD`, preferring local time. +fn current_date() -> String { + const DATE_FORMAT: &[time::format_description::FormatItem<'static>] = + format_description!("[year]-[month]-[day]"); + let now = time::OffsetDateTime::now_local().unwrap_or_else(|_| time::OffsetDateTime::now_utc()); + now.format(DATE_FORMAT).expect("static date format should be valid") +} + +/// Resolve the env flag that places leading instructions in system identity. +fn env_use_harmony_system_instructions() -> bool { + std::env::var(HARMONY_SYSTEM_INSTRUCTIONS_ENV) + .ok() + .and_then(|value| value.parse::().ok()) + .is_some_and(|value| value != 0) +} + +#[cfg(test)] +mod tests; diff --git a/rust/src/chat/src/renderer/harmony/tests.rs b/rust/src/chat/src/renderer/harmony/tests.rs new file mode 100644 index 0000000000..bcbf97c866 --- /dev/null +++ b/rust/src/chat/src/renderer/harmony/tests.rs @@ -0,0 +1,212 @@ +use std::path::PathBuf; + +use expect_test::{ExpectFile, expect, expect_file}; +use thiserror_ext::AsReport as _; + +use super::HarmonyChatRenderer; +use super::encoding::harmony_encoding; +use crate::ChatRenderer; +use crate::error::Error; +use crate::event::{AssistantContentBlock, AssistantToolCall}; +use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; +use crate::request::{ + ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort, +}; + +const PINNED_DATE: &str = "2025-06-28"; + +fn fixture_request(input_name: &str) -> ChatRequest { + fixture_chat_request( + &fixture_path(input_name), + FixtureRequestOptions { + enable_thinking: false, + no_generation_prompt_when_last_assistant: false, + }, + ) +} + +fn fixture_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/renderer/harmony") + .join("fixtures") + .join(name) +} + +fn test_renderer(use_system_instructions: bool) -> HarmonyChatRenderer { + HarmonyChatRenderer::with_options(PINNED_DATE, use_system_instructions).unwrap() +} + +fn render_token_ids(request: &ChatRequest) -> Vec { + render_token_ids_with(&test_renderer(false), request) +} + +fn render_token_ids_with(renderer: &HarmonyChatRenderer, request: &ChatRequest) -> Vec { + renderer + .render(request) + .unwrap() + .prompt + .into_token_ids() + .expect("Harmony renderer returns token IDs") +} + +fn render_prompt_text(request: &ChatRequest) -> String { + render_prompt_text_with(&test_renderer(false), request) +} + +fn render_prompt_text_with(renderer: &HarmonyChatRenderer, request: &ChatRequest) -> String { + let token_ids = render_token_ids_with(renderer, request); + harmony_encoding().unwrap().tokenizer().decode_utf8(&token_ids).unwrap() +} + +fn assert_fixture(input_name: &str, expected: ExpectFile) { + let request = fixture_request(input_name); + let rendered = format!("{}\n", render_prompt_text(&request)); + expected.assert_eq(&rendered); +} + +#[test] +fn renders_token_ids() { + let request = fixture_request("simple_user.json"); + + assert!(!render_token_ids(&request).is_empty()); +} + +#[test] +fn renders_simple_user_fixture() { + assert_fixture("simple_user.json", expect_file!["fixtures/simple_user.txt"]); +} + +#[test] +fn renders_leading_system_fixture() { + assert_fixture( + "leading_system.json", + expect_file!["fixtures/leading_system.txt"], + ); +} + +#[test] +fn renders_system_instructions_env_fixture() { + let renderer = test_renderer(true); + let request = fixture_request("leading_system.json"); + let rendered = format!("{}\n", render_prompt_text_with(&renderer, &request)); + expect_file!["fixtures/system_instructions_env.txt"].assert_eq(&rendered); +} + +#[test] +fn renders_request_tools_fixture() { + assert_fixture( + "request_tools.json", + expect_file!["fixtures/request_tools.txt"], + ); +} + +#[test] +fn renders_developer_tools_fixture() { + assert_fixture( + "developer_tools.json", + expect_file!["fixtures/developer_tools.txt"], + ); +} + +#[test] +fn renders_assistant_history_fixture() { + assert_fixture( + "assistant_history.json", + expect_file!["fixtures/assistant_history.txt"], + ); +} + +#[test] +fn renders_tool_roundtrip_fixture() { + assert_fixture( + "tool_roundtrip.json", + expect_file!["fixtures/tool_roundtrip.txt"], + ); +} + +#[test] +fn drops_stale_analysis_fixture() { + assert_fixture( + "drop_analysis.json", + expect_file!["fixtures/drop_analysis.txt"], + ); +} + +#[test] +fn rejects_invalid_reasoning_effort() { + let mut request = ChatRequest::for_test(); + request.chat_options.reasoning_effort = Some(ReasoningEffort::None); + + let error = test_renderer(false).render(&request).unwrap_err(); + + expect![[r#"chat template error: reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]] + .assert_eq(&error.to_report_string()); +} + +#[test] +fn rejects_unknown_tool_response_id() { + let request = ChatRequest { + messages: vec![ + ChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall( + AssistantToolCall { + id: "call-known".to_string(), + name: "lookup".to_string(), + arguments: "{}".to_string(), + }, + )]), + ChatMessage::tool_response("{}", "call-unknown"), + ], + ..ChatRequest::for_test() + }; + + let error = test_renderer(false).render(&request).unwrap_err(); + + expect![ + "chat template error: invalid Harmony tool message: unknown tool_call_id `call-unknown`" + ] + .assert_eq(&error.to_report_string()); +} + +#[test] +fn rejects_multimodal_input() { + let request = ChatRequest { + messages: vec![ChatMessage::user(vec![ChatContentPart::image_url( + "data:image/png;base64,test", + )])], + ..ChatRequest::for_test() + }; + + let error = test_renderer(false).render(&request).unwrap_err(); + + assert!(matches!( + error, + Error::UnsupportedMultimodalContent("image_url") + )); +} + +#[test] +fn rejects_continue_final_assistant() { + let mut request = ChatRequest { + messages: vec![ + ChatMessage::user("write"), + ChatMessage::assistant_text("partial"), + ], + ..ChatRequest::for_test() + }; + request.chat_options.generation_prompt_mode = GenerationPromptMode::ContinueFinalAssistant; + + let error = test_renderer(false).render(&request).unwrap_err(); + + expect!["chat template error: Harmony renderer does not support continue_final_message"] + .assert_eq(&error.to_report_string()); +} + +#[test] +fn no_generation_prompt_omits_trailing_assistant_start() { + let mut request = fixture_request("simple_user.json"); + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt; + + let rendered = render_prompt_text(&request); + + assert!(!rendered.ends_with("<|start|>assistant")); +} diff --git a/rust/src/chat/src/renderer/mod.rs b/rust/src/chat/src/renderer/mod.rs index c4ee787c86..f1c510a1b3 100644 --- a/rust/src/chat/src/renderer/mod.rs +++ b/rust/src/chat/src/renderer/mod.rs @@ -9,6 +9,7 @@ use crate::request::{ChatRequest, ReasoningEffort}; pub mod deepseek_v32; pub mod deepseek_v4; +pub mod harmony; pub mod hf; mod selection; #[cfg(test)] @@ -16,6 +17,7 @@ mod test_utils; pub use deepseek_v4::DeepSeekV4ChatRenderer; pub use deepseek_v32::DeepSeekV32ChatRenderer; +pub use harmony::HarmonyChatRenderer; pub use selection::RendererSelection; /// Rendered chat prompt submitted to the text backend. diff --git a/rust/src/chat/src/renderer/selection.rs b/rust/src/chat/src/renderer/selection.rs index 09bdd6b972..837ec7d69c 100644 --- a/rust/src/chat/src/renderer/selection.rs +++ b/rust/src/chat/src/renderer/selection.rs @@ -19,12 +19,16 @@ pub enum RendererSelection { DeepSeekV32, /// Force the DeepSeek V4 renderer. DeepSeekV4, + /// Force the GPT-OSS Harmony renderer. + Harmony, } impl RendererSelection { pub const AUTO_LITERAL: &str = "auto"; pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32"; pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4"; + pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss"; + pub const HARMONY_LITERAL: &str = "harmony"; pub const HF_LITERAL: &str = "hf"; /// Resolve the renderer selection using the given model type string, if @@ -34,6 +38,7 @@ impl RendererSelection { Self::Auto => match model_type { Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32, Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4, + Self::GPT_OSS_MODEL_TYPE => Self::Harmony, _ => Self::Hf, }, selection => selection, @@ -53,6 +58,8 @@ impl FromStr for RendererSelection { Ok(Self::DeepSeekV32) } else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) { Ok(Self::DeepSeekV4) + } else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) { + Ok(Self::Harmony) } else { Err(format!( "unknown renderer `{value}` (expected one of: {})", @@ -69,6 +76,7 @@ impl fmt::Display for RendererSelection { Self::Hf => f.write_str(Self::HF_LITERAL), Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL), Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL), + Self::Harmony => f.write_str(Self::HARMONY_LITERAL), } } } @@ -95,7 +103,7 @@ mod tests { fn renderer_selection_expected_error_message() { let err = RendererSelection::from_str("unknown").unwrap_err(); expect_test::expect![ - "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4)" + "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)" ] .assert_eq(&err); } diff --git a/rust/src/chat/src/renderer/test_utils.rs b/rust/src/chat/src/renderer/test_utils.rs index 0aab3769db..bf560de842 100644 --- a/rust/src/chat/src/renderer/test_utils.rs +++ b/rust/src/chat/src/renderer/test_utils.rs @@ -7,7 +7,7 @@ use serde_json::Value; use crate::event::{AssistantContentBlock, AssistantToolCall}; use crate::request::{ ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice, - GenerationPromptMode, + GenerationPromptMode, ReasoningEffort, }; /// Options for constructing a [`ChatRequest`] from a fixture file. @@ -42,6 +42,7 @@ pub(crate) struct FixtureRequest { tools: Vec, messages: Vec, add_generation_prompt: Option, + reasoning_effort: Option, } impl FixtureFile { @@ -52,6 +53,7 @@ impl FixtureFile { tools: Vec::new(), messages, add_generation_prompt: None, + reasoning_effort: None, }, } } @@ -154,6 +156,7 @@ impl FixtureRequest { if self.add_generation_prompt == Some(false) { request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt; } + request.chat_options.reasoning_effort = self.reasoning_effort; if options.enable_thinking { for key in ["thinking", "enable_thinking"] { request.chat_options.template_kwargs.insert(key.to_string(), Value::Bool(true)); diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 345cc9f60d..a9b11ca18f 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -468,7 +468,7 @@ fn serve_args_reject_unknown_renderer_value() { .unwrap_err(); expect![[r#" - error: invalid value 'definitely_missing' for '--tokenizer-mode ': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4) + error: invalid value 'definitely_missing' for '--tokenizer-mode ': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony) For more information, try '--help'. "#]]