[Rust Frontend] Add Harmony Renderer for GPT-OSS (#46800)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-29 03:39:04 +00:00
committed by GitHub
parent 58d6a6e60a
commit 5274c1181d
27 changed files with 1033 additions and 32 deletions
+1
View File
@@ -25,6 +25,7 @@ strum.workspace = true
subenum.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
time.workspace = true
tokio.workspace = true
tracing.workspace = true
trait-set.workspace = true
+48 -8
View File
@@ -15,7 +15,9 @@ use crate::output::{
DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides,
};
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
use crate::renderer::{DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer};
use crate::renderer::{
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
};
use crate::request::ChatRequest;
use crate::{DynChatOutputProcessor, RendererSelection};
@@ -61,6 +63,7 @@ impl HfChatBackend {
)?),
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
};
info!(
@@ -148,13 +151,15 @@ mod tests {
use std::sync::Arc;
use tempfile::tempdir;
use thiserror_ext::AsReport as _;
use vllm_text::Prompt;
use vllm_text::backend::hf::TokenizerSource;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use super::HfChatBackend;
use crate::RendererSelection;
use crate::backend::{ChatBackend, LoadModelBackendsOptions};
use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};
use crate::request::{ChatContent, ChatMessage, ChatRequest};
use crate::{ParserSelection, RendererSelection};
fn request_with_user_text(text: &str) -> ChatRequest {
ChatRequest {
@@ -219,12 +224,12 @@ mod tests {
Arc::new(TestTokenizer)
}
fn render_prompt(
fn backend_for_selection(
renderer: RendererSelection,
config_json: &str,
tokenizer_config_json: &str,
) -> String {
let backend = HfChatBackend::from_resolved_model_files(
) -> HfChatBackend {
HfChatBackend::from_resolved_model_files(
resolved_files(config_json, tokenizer_config_json),
"test-model".to_string(),
LoadModelBackendsOptions {
@@ -236,9 +241,15 @@ mod tests {
},
test_tokenizer(),
)
.unwrap();
.unwrap()
}
backend
fn render_prompt(
renderer: RendererSelection,
config_json: &str,
tokenizer_config_json: &str,
) -> String {
backend_for_selection(renderer, config_json, tokenizer_config_json)
.chat_renderer()
.render(&request_with_user_text("hello"))
.unwrap()
@@ -272,6 +283,35 @@ mod tests {
assert_eq!(prompt, "hello");
}
#[test]
fn auto_uses_harmony_renderer_and_output_processor_for_gpt_oss_model_type() {
let backend = backend_for_selection(
RendererSelection::Auto,
r#"{"model_type":"gpt_oss"}"#,
r#"{"chat_template":"{{ messages[0].content }}"}"#,
);
let prompt =
backend.chat_renderer().render(&request_with_user_text("hello")).unwrap().prompt;
assert!(matches!(prompt, Prompt::TokenIds(_)));
let mut request = request_with_user_text("hello");
let error = match backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
reasoning_parser: &ParserSelection::Auto,
},
) {
Ok(_) => panic!("gpt_oss should reject generic parser overrides"),
Err(error) => error,
};
assert_eq!(
error.to_report_string(),
"gpt_oss uses native Harmony output parsing; generic tool parser override `json` is not supported"
);
}
#[test]
fn language_model_only_skips_multimodal_preprocessor_config() {
let mut files = resolved_files(
+2 -2
View File
@@ -29,8 +29,8 @@ pub use parser::reasoning::{
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, RenderedPrompt,
RendererSelection,
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
HarmonyChatRenderer, RenderedPrompt, RendererSelection,
};
pub use request::{
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
+2 -19
View File
@@ -4,16 +4,10 @@
//! `DecodedTextEvent` token IDs directly and lets the official `openai-harmony`
//! parser recover the structured assistant message shape at token granularity.
use std::sync::LazyLock;
use anyhow::Context;
use asynk_strim_attr::{TryYielder, try_stream};
use futures::StreamExt as _;
use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role};
use openai_harmony::{
HarmonyEncoding, HarmonyEncodingName, StreamableParser, load_harmony_encoding,
};
use thiserror_ext::AsReport;
use openai_harmony::{HarmonyEncoding, StreamableParser};
use vllm_text::output::DecodedTextEvent;
use crate::Result as ChatResult;
@@ -24,6 +18,7 @@ use crate::output::{
generate_tool_call_id,
};
use crate::parser::ParserSelection;
use crate::renderer::harmony::encoding::harmony_encoding;
use crate::request::ChatRequest;
/// Request-scoped Harmony output processor used for `model_type == "gpt_oss"`.
@@ -384,18 +379,6 @@ async fn harmony_assistant_event_stream(
Ok(())
}
/// Lazily load the shared GPT-OSS Harmony encoding once per process.
fn harmony_encoding() -> Result<&'static HarmonyEncoding> {
static ENCODING: LazyLock<anyhow::Result<HarmonyEncoding>> = LazyLock::new(|| {
load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss)
.context("failed to load harmony encoding for gpt-oss")
});
ENCODING.as_ref().map_err(|error| Error::HarmonyOutputParsing {
error: error.to_report_string().into(),
})
}
fn harmony_output_parsing_error(
error: impl Into<Box<dyn std::error::Error + Send + Sync>>,
) -> Error {
@@ -0,0 +1,21 @@
//! Shared Harmony encoding helper for the GPT-OSS renderer and output parser.
use std::sync::LazyLock;
use anyhow::Context as _;
use openai_harmony::{HarmonyEncoding, HarmonyEncodingName, load_harmony_encoding};
use thiserror_ext::AsReport as _;
use crate::error::{Error, Result};
/// Lazily load the shared GPT-OSS Harmony encoding once per process.
pub(crate) fn harmony_encoding() -> Result<&'static HarmonyEncoding> {
static ENCODING: LazyLock<anyhow::Result<HarmonyEncoding>> = LazyLock::new(|| {
load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss)
.context("failed to load harmony encoding for gpt-oss")
});
ENCODING.as_ref().map_err(|error| Error::HarmonyOutputParsing {
error: error.to_report_string().into(),
})
}
@@ -0,0 +1,14 @@
{
"add_generation_prompt": false,
"messages": [
{
"role": "user",
"content": "What is 2 + 2?"
},
{
"role": "assistant",
"reasoning_content": "Need simple arithmetic.",
"content": "4"
}
]
}
@@ -0,0 +1,7 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: medium
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>4<|end|>
@@ -0,0 +1,27 @@
[
{
"role": "developer",
"content": "Use tools when needed.",
"tools": [
{
"function": {
"name": "lookup",
"description": "Lookup a record.",
"parameters": {
"type": "object",
"properties": {
"id": {
"type": "string"
}
},
"required": ["id"]
}
}
}
]
},
{
"role": "user",
"content": "Find record abc."
}
]
@@ -0,0 +1,23 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: medium
# Valid channels: analysis, commentary, final. Channel must be included for every message.
Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions
Use tools when needed.
# Tools
## functions
namespace functions {
// Lookup a record.
type lookup = (_: {
id: string,
}) => any;
} // namespace functions<|end|><|start|>user<|message|>Find record abc.<|end|><|start|>assistant
@@ -0,0 +1,15 @@
[
{
"role": "user",
"content": "What is 2 + 2?"
},
{
"role": "assistant",
"reasoning_content": "This should be dropped.",
"content": "4"
},
{
"role": "user",
"content": "What is 3 + 5?"
}
]
@@ -0,0 +1,7 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: medium
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>4<|end|><|start|>user<|message|>What is 3 + 5?<|end|><|start|>assistant
@@ -0,0 +1,13 @@
{
"reasoning_effort": "high",
"messages": [
{
"role": "system",
"content": "Answer tersely."
},
{
"role": "user",
"content": "What is 2 + 2?"
}
]
}
@@ -0,0 +1,9 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: high
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions
Answer tersely.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant
@@ -0,0 +1,26 @@
{
"tools": [
{
"function": {
"name": "get_weather",
"description": "Get weather for a city.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string"
}
},
"required": ["city"]
},
"strict": true
}
}
],
"messages": [
{
"role": "user",
"content": "Check Hangzhou weather."
}
]
}
@@ -0,0 +1,19 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: medium
# Valid channels: analysis, commentary, final. Channel must be included for every message.
Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Tools
## functions
namespace functions {
// Get weather for a city.
type get_weather = (_: {
city: string,
}) => any;
} // namespace functions<|end|><|start|>user<|message|>Check Hangzhou weather.<|end|><|start|>assistant
@@ -0,0 +1,6 @@
[
{
"role": "user",
"content": "Hello, who are you?"
}
]
@@ -0,0 +1,7 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: medium
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>Hello, who are you?<|end|><|start|>assistant
@@ -0,0 +1,8 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Answer tersely.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: high
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant
@@ -0,0 +1,43 @@
{
"tools": [
{
"function": {
"name": "get_weather",
"description": "Get weather for a city.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string"
}
},
"required": ["city"]
}
}
}
],
"messages": [
{
"role": "user",
"content": "Check Hangzhou weather."
},
{
"role": "assistant",
"reasoning_content": "Need current weather.",
"tool_calls": [
{
"id": "call-weather",
"function": {
"name": "get_weather",
"arguments": "{\"city\":\"Hangzhou\"}"
}
}
]
},
{
"role": "tool",
"tool_call_id": "call-weather",
"content": "{\"temperature\":20}"
}
]
}
@@ -0,0 +1,19 @@
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
Knowledge cutoff: 2024-06
Current date: 2025-06-28
Reasoning: medium
# Valid channels: analysis, commentary, final. Channel must be included for every message.
Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Tools
## functions
namespace functions {
// Get weather for a city.
type get_weather = (_: {
city: string,
}) => any;
} // namespace functions<|end|><|start|>user<|message|>Check Hangzhou weather.<|end|><|start|>assistant<|channel|>analysis<|message|>Need current weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{"city":"Hangzhou"}<|call|><|start|>functions.get_weather<|channel|>commentary to=assistant<|message|>{"temperature":20}<|end|><|start|>assistant
+487
View File
@@ -0,0 +1,487 @@
//! Native Harmony chat renderer for `gpt_oss`.
pub(crate) mod encoding;
use openai_harmony::HarmonyEncoding;
use openai_harmony::chat::{
Author, Conversation, DeveloperContent, Message, ReasoningEffort as HarmonyReasoningEffort,
Role, SystemContent, ToolDescription,
};
use thiserror_ext::AsReport as _;
use time::macros::format_description;
use vllm_text::Prompt;
use self::encoding::harmony_encoding;
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::error::{Error, Result};
use crate::event::AssistantContentBlock;
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, GenerationPromptMode};
use crate::{AssistantMessageExt as _, ReasoningEffort};
const SYSTEM_START_DATE_ENV: &str = "VLLM_SYSTEM_START_DATE";
const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS";
/// GPT-OSS renderer backed by the official Harmony encoding.
pub struct HarmonyChatRenderer {
encoding: &'static HarmonyEncoding,
options: Options,
}
struct Options {
system_start_date: String,
use_system_instructions: bool,
}
impl HarmonyChatRenderer {
/// Create a Harmony renderer for production use.
///
/// Environment-derived options are resolved once at construction time:
///
/// - `VLLM_SYSTEM_START_DATE` pins the Harmony system start date. When it is
/// unset, the renderer uses the current local date with a UTC fallback.
/// - `VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS` moves leading instructions
/// into the system model identity when set to a non-zero integer.
pub fn new() -> Result<Self> {
Self::with_options(
env_system_start_date(),
env_use_harmony_system_instructions(),
)
}
/// Create a Harmony renderer with explicit preamble options.
///
/// Tests use this constructor to avoid process-global environment mutation.
/// Production code should call [`Self::new`] so the renderer observes the
/// same environment contract as the Python Harmony path.
pub fn with_options(
system_start_date: impl Into<String>,
use_system_instructions: bool,
) -> Result<Self> {
Ok(Self {
encoding: harmony_encoding()?,
options: Options {
system_start_date: system_start_date.into(),
use_system_instructions,
},
})
}
/// Render a chat request directly to Harmony token IDs.
///
/// Harmony owns both prompt formatting and tokenization, so the Rust
/// frontend bypasses the generic HF tokenizer path for GPT-OSS input.
fn render_token_ids(&self, request: &ChatRequest) -> Result<Vec<u32>> {
if request.has_multimodal() {
return Err(Error::UnsupportedMultimodalContent("image_url"));
}
if matches!(
request.chat_options.generation_prompt_mode,
GenerationPromptMode::ContinueFinalAssistant
) {
return Err(Error::ChatTemplate(
"Harmony renderer does not support continue_final_message".to_string(),
));
}
let messages = auto_drop_analysis_messages(to_harmony_messages(request, &self.options)?);
let conversation = Conversation::from_messages(messages);
// Pass `None` so oss-harmony does not apply its narrower built-in
// analysis-drop policy after the Rust-side Python-parity cleanup above.
let token_ids = match request.chat_options.generation_prompt_mode {
GenerationPromptMode::StartNewAssistant => self
.encoding
.render_conversation_for_completion(&conversation, Role::Assistant, None),
GenerationPromptMode::NoGenerationPrompt => {
self.encoding.render_conversation(&conversation, None)
}
GenerationPromptMode::ContinueFinalAssistant => unreachable!("checked above"),
}
.map_err(|error| {
Error::ChatTemplate(format!(
"failed to render Harmony prompt: {}",
error.as_report()
))
})?;
Ok(token_ids)
}
}
impl ChatRenderer for HarmonyChatRenderer {
/// Render a chat request as [`Prompt::TokenIds`] with template kwargs echoed
/// for downstream accounting/debugging.
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
Ok(RenderedPrompt {
prompt: Prompt::TokenIds(self.render_token_ids(request)?),
effective_template_kwargs: request_template_kwargs(request),
})
}
}
/// Convert a vLLM chat request into a full Harmony conversation.
///
/// This adds the Harmony system/developer preamble, peels at most one leading
/// system/developer instruction message, and then lowers the remaining chat
/// history message-by-message.
fn to_harmony_messages(request: &ChatRequest, options: &Options) -> Result<Vec<Message>> {
let (instructions, leading_developer_tools, remaining_messages) =
peel_leading_instructions(&request.messages)?;
let tool_call_names = tool_call_names(&request.messages);
let mut messages =
build_harmony_preamble(request, instructions, leading_developer_tools, options)?;
for message in remaining_messages {
messages.extend(to_harmony_message(message, &tool_call_names, options)?);
}
Ok(messages)
}
/// Extract the optional leading instruction message used by the Harmony preamble.
///
/// Python only peels the first leading `system` or `developer` message. Later
/// system/developer messages stay in the conversation and are lowered normally.
#[allow(clippy::type_complexity)]
fn peel_leading_instructions(
messages: &[ChatMessage],
) -> Result<(Option<String>, Option<&[ChatTool]>, &[ChatMessage])> {
let Some(first) = messages.first() else {
return Ok((None, None, messages));
};
match first {
ChatMessage::System { content } => Ok((Some(flatten_text(content)?), None, &messages[1..])),
ChatMessage::Developer { content, tools } => Ok((
Some(flatten_text(content)?),
tools.as_deref(),
&messages[1..],
)),
ChatMessage::User { .. }
| ChatMessage::Assistant { .. }
| ChatMessage::ToolResponse { .. } => Ok((None, None, messages)),
}
}
/// Build the Harmony preamble for one request.
///
/// The preamble always contains a system message with date and reasoning-effort
/// metadata. Leading instructions live either in the system model identity or in
/// a developer message depending on `use_system_instructions`; request-level and
/// leading developer tools are attached to the developer message.
fn build_harmony_preamble(
request: &ChatRequest,
instructions: Option<String>,
leading_developer_tools: Option<&[ChatTool]>,
options: &Options,
) -> Result<Vec<Message>> {
let mut messages = vec![Message::from_role_and_content(
Role::System,
system_content(
instructions.as_deref().filter(|_| options.use_system_instructions),
request.chat_options.reasoning_effort,
&options.system_start_date,
)?,
)];
let mut developer = DeveloperContent::new();
let mut has_developer_content = false;
if !options.use_system_instructions
&& let Some(instructions) = instructions.as_deref().filter(|text| !text.is_empty())
{
developer = developer.with_instructions(instructions);
has_developer_content = true;
}
let tool_descriptions = preamble_tool_descriptions(request, leading_developer_tools);
if !tool_descriptions.is_empty() {
developer = developer.with_function_tools(tool_descriptions);
has_developer_content = true;
}
if has_developer_content {
messages.push(Message::from_role_and_content(Role::Developer, developer));
}
Ok(messages)
}
/// Collect request-level and leading developer function tools for the preamble.
fn preamble_tool_descriptions(
request: &ChatRequest,
leading_developer_tools: Option<&[ChatTool]>,
) -> Vec<ToolDescription> {
let mut tools = Vec::new();
if request.tool_parsing_enabled() {
tools.extend(to_tool_descriptions(&request.tools));
}
if let Some(leading_developer_tools) = leading_developer_tools {
tools.extend(to_tool_descriptions(leading_developer_tools));
}
tools
}
/// Construct the Harmony system content for the request preamble.
///
/// Harmony defaults the reasoning effort to `medium` when none is provided, so
/// this only sets an explicit effort after validating vLLM's request value.
fn system_content(
instructions: Option<&str>,
reasoning_effort: Option<ReasoningEffort>,
system_start_date: &str,
) -> Result<SystemContent> {
let mut content =
SystemContent::new().with_conversation_start_date(system_start_date.to_string());
if let Some(reasoning_effort) = reasoning_effort {
content = content.with_reasoning_effort(to_harmony_reasoning_effort(reasoning_effort)?);
}
if let Some(instructions) = instructions.filter(|text| !text.is_empty()) {
let model_identity = match content.model_identity.as_deref() {
Some(identity) if !identity.is_empty() => format!("{identity}\n{instructions}"),
_ => instructions.to_string(),
};
content = content.with_model_identity(model_identity);
}
Ok(content)
}
/// Lower a single vLLM chat message into one or more Harmony messages.
///
/// Assistant messages can split into separate analysis, final, commentary, and
/// tool-call messages. Tool responses require the earlier assistant tool-call ID
/// map so the Harmony tool author can include `functions.{name}`.
fn to_harmony_message(
message: &ChatMessage,
tool_call_names: &std::collections::HashMap<String, String>,
options: &Options,
) -> Result<Vec<Message>> {
Ok(match message {
ChatMessage::System { content } => {
let instructions = flatten_text(content)?;
vec![system_or_developer_message(
"system",
instructions,
None,
options,
)?]
}
ChatMessage::Developer { content, tools } => {
let instructions = flatten_text(content)?;
vec![developer_message(Some(instructions), tools.as_deref())]
}
ChatMessage::User { content } => {
vec![Message::from_role_and_content(
Role::User,
flatten_text(content)?,
)]
}
ChatMessage::Assistant { content } => assistant_messages(content),
ChatMessage::ToolResponse {
content,
tool_call_id,
} => {
let name = tool_call_names.get(tool_call_id).ok_or_else(|| {
Error::ChatTemplate(format!(
"invalid Harmony tool message: unknown tool_call_id `{tool_call_id}`"
))
})?;
vec![
Message::from_author_and_content(
Author::new(Role::Tool, format!("functions.{name}")),
flatten_text(content)?,
)
.with_channel("commentary")
.with_recipient("assistant"),
]
}
})
}
/// Lower a non-leading system/developer message.
///
/// Harmony treats most extra system/developer messages as developer
/// instructions. When system-instructions mode is enabled, system messages are
/// rendered as system model-identity additions to match Python.
fn system_or_developer_message(
role: &str,
instructions: String,
tools: Option<&[ChatTool]>,
options: &Options,
) -> Result<Message> {
if role == "system" && options.use_system_instructions {
return Ok(Message::from_role_and_content(
Role::System,
system_content(Some(&instructions), None, &options.system_start_date)?,
));
}
Ok(developer_message(Some(instructions), tools))
}
/// Build a Harmony developer message with optional instructions and function tools.
fn developer_message(instructions: Option<String>, tools: Option<&[ChatTool]>) -> Message {
let mut content = DeveloperContent::new();
if let Some(instructions) = instructions.filter(|text| !text.is_empty()) {
content = content.with_instructions(instructions);
}
if let Some(tools) = tools {
let tools = to_tool_descriptions(tools);
if !tools.is_empty() {
content = content.with_function_tools(tools);
}
}
Message::from_role_and_content(Role::Developer, content)
}
/// Lower assistant history into Harmony channels.
///
/// Plain assistant text goes to `final`. When the assistant has tool calls,
/// visible text goes to `commentary`, reasoning goes to `analysis`, and each
/// function call becomes a `commentary` message to `functions.{name}` with JSON
/// constrained content.
fn assistant_messages(content: &[AssistantContentBlock]) -> Vec<Message> {
let mut messages = Vec::new();
let has_tool_calls = content.has_tool_calls();
if has_tool_calls {
let text = content.text();
if !text.is_empty() {
messages.push(
Message::from_role_and_content(Role::Assistant, text).with_channel("commentary"),
);
}
}
if let Some(reasoning) = content.reasoning() {
messages.push(
Message::from_role_and_content(Role::Assistant, reasoning).with_channel("analysis"),
);
}
if has_tool_calls {
for tool_call in content.tool_calls() {
messages.push(
Message::from_role_and_content(Role::Assistant, tool_call.arguments.clone())
.with_channel("commentary")
.with_recipient(format!("functions.{}", tool_call.name))
.with_content_type("<|constrain|>json"),
);
}
} else {
let text = content.text();
if !text.is_empty() {
messages
.push(Message::from_role_and_content(Role::Assistant, text).with_channel("final"));
}
}
messages
}
/// Build the tool-call ID to function-name map used by later tool responses.
fn tool_call_names(messages: &[ChatMessage]) -> std::collections::HashMap<String, String> {
let mut names = std::collections::HashMap::new();
for message in messages {
let ChatMessage::Assistant { content } = message else {
continue;
};
for tool_call in content.tool_calls() {
names.insert(tool_call.id.clone(), tool_call.name.clone());
}
}
names
}
/// Drop stale assistant analysis messages using vLLM Python's policy.
///
/// Once an assistant final message exists, earlier analysis messages represent
/// chain-of-thought for completed turns and should not be replayed to the model.
fn auto_drop_analysis_messages(messages: Vec<Message>) -> Vec<Message> {
// Match vLLM Python's Harmony cleanup: once an assistant final message exists,
// previous assistant analysis messages are stale chain-of-thought and should
// be removed. oss-harmony can also drop analysis with `Some(Default::default())`,
// but that built-in path only triggers when the last assistant message is final
// and drops relative to the first final message, which misses longer multi-turn
// histories with later user/tool turns.
let Some(last_assistant_final_index) = messages.iter().rposition(|message| {
message.author.role == Role::Assistant && message.channel.as_deref() == Some("final")
}) else {
return messages;
};
messages
.into_iter()
.enumerate()
.filter_map(|(index, message)| {
(index >= last_assistant_final_index || message.channel.as_deref() != Some("analysis"))
.then_some(message)
})
.collect()
}
/// Flatten vLLM text content and reject unsupported multimodal parts.
fn flatten_text(content: &ChatContent) -> Result<String> {
content.try_flatten_to_text()
}
/// Convert vLLM function tool definitions to Harmony tool descriptions.
fn to_tool_descriptions(tools: &[ChatTool]) -> Vec<ToolDescription> {
tools
.iter()
.map(|tool| {
ToolDescription::new(
tool.name.clone(),
tool.description.clone().unwrap_or_default(),
Some(tool.parameters.clone()),
)
})
.collect()
}
/// Map supported OpenAI reasoning-effort values onto Harmony's enum.
fn to_harmony_reasoning_effort(
reasoning_effort: ReasoningEffort,
) -> Result<HarmonyReasoningEffort> {
match reasoning_effort {
ReasoningEffort::Low => Ok(HarmonyReasoningEffort::Low),
ReasoningEffort::Medium => Ok(HarmonyReasoningEffort::Medium),
ReasoningEffort::High => Ok(HarmonyReasoningEffort::High),
ReasoningEffort::None
| ReasoningEffort::Minimal
| ReasoningEffort::XHigh
| ReasoningEffort::Max => Err(Error::ChatTemplate(format!(
"reasoning_effort={:?} is not supported by Harmony. Supported values are: low, medium, high.",
reasoning_effort.as_str()
))),
}
}
/// Resolve the system start date from the environment or the current date.
fn env_system_start_date() -> String {
std::env::var(SYSTEM_START_DATE_ENV)
.ok()
.filter(|date| !date.is_empty())
.unwrap_or_else(current_date)
}
/// Format today's date as `YYYY-MM-DD`, preferring local time.
fn current_date() -> String {
const DATE_FORMAT: &[time::format_description::FormatItem<'static>] =
format_description!("[year]-[month]-[day]");
let now = time::OffsetDateTime::now_local().unwrap_or_else(|_| time::OffsetDateTime::now_utc());
now.format(DATE_FORMAT).expect("static date format should be valid")
}
/// Resolve the env flag that places leading instructions in system identity.
fn env_use_harmony_system_instructions() -> bool {
std::env::var(HARMONY_SYSTEM_INSTRUCTIONS_ENV)
.ok()
.and_then(|value| value.parse::<i32>().ok())
.is_some_and(|value| value != 0)
}
#[cfg(test)]
mod tests;
+212
View File
@@ -0,0 +1,212 @@
use std::path::PathBuf;
use expect_test::{ExpectFile, expect, expect_file};
use thiserror_ext::AsReport as _;
use super::HarmonyChatRenderer;
use super::encoding::harmony_encoding;
use crate::ChatRenderer;
use crate::error::Error;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use crate::request::{
ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort,
};
const PINNED_DATE: &str = "2025-06-28";
fn fixture_request(input_name: &str) -> ChatRequest {
fixture_chat_request(
&fixture_path(input_name),
FixtureRequestOptions {
enable_thinking: false,
no_generation_prompt_when_last_assistant: false,
},
)
}
fn fixture_path(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("src/renderer/harmony")
.join("fixtures")
.join(name)
}
fn test_renderer(use_system_instructions: bool) -> HarmonyChatRenderer {
HarmonyChatRenderer::with_options(PINNED_DATE, use_system_instructions).unwrap()
}
fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
render_token_ids_with(&test_renderer(false), request)
}
fn render_token_ids_with(renderer: &HarmonyChatRenderer, request: &ChatRequest) -> Vec<u32> {
renderer
.render(request)
.unwrap()
.prompt
.into_token_ids()
.expect("Harmony renderer returns token IDs")
}
fn render_prompt_text(request: &ChatRequest) -> String {
render_prompt_text_with(&test_renderer(false), request)
}
fn render_prompt_text_with(renderer: &HarmonyChatRenderer, request: &ChatRequest) -> String {
let token_ids = render_token_ids_with(renderer, request);
harmony_encoding().unwrap().tokenizer().decode_utf8(&token_ids).unwrap()
}
fn assert_fixture(input_name: &str, expected: ExpectFile) {
let request = fixture_request(input_name);
let rendered = format!("{}\n", render_prompt_text(&request));
expected.assert_eq(&rendered);
}
#[test]
fn renders_token_ids() {
let request = fixture_request("simple_user.json");
assert!(!render_token_ids(&request).is_empty());
}
#[test]
fn renders_simple_user_fixture() {
assert_fixture("simple_user.json", expect_file!["fixtures/simple_user.txt"]);
}
#[test]
fn renders_leading_system_fixture() {
assert_fixture(
"leading_system.json",
expect_file!["fixtures/leading_system.txt"],
);
}
#[test]
fn renders_system_instructions_env_fixture() {
let renderer = test_renderer(true);
let request = fixture_request("leading_system.json");
let rendered = format!("{}\n", render_prompt_text_with(&renderer, &request));
expect_file!["fixtures/system_instructions_env.txt"].assert_eq(&rendered);
}
#[test]
fn renders_request_tools_fixture() {
assert_fixture(
"request_tools.json",
expect_file!["fixtures/request_tools.txt"],
);
}
#[test]
fn renders_developer_tools_fixture() {
assert_fixture(
"developer_tools.json",
expect_file!["fixtures/developer_tools.txt"],
);
}
#[test]
fn renders_assistant_history_fixture() {
assert_fixture(
"assistant_history.json",
expect_file!["fixtures/assistant_history.txt"],
);
}
#[test]
fn renders_tool_roundtrip_fixture() {
assert_fixture(
"tool_roundtrip.json",
expect_file!["fixtures/tool_roundtrip.txt"],
);
}
#[test]
fn drops_stale_analysis_fixture() {
assert_fixture(
"drop_analysis.json",
expect_file!["fixtures/drop_analysis.txt"],
);
}
#[test]
fn rejects_invalid_reasoning_effort() {
let mut request = ChatRequest::for_test();
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
let error = test_renderer(false).render(&request).unwrap_err();
expect![[r#"chat template error: reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]]
.assert_eq(&error.to_report_string());
}
#[test]
fn rejects_unknown_tool_response_id() {
let request = ChatRequest {
messages: vec![
ChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall(
AssistantToolCall {
id: "call-known".to_string(),
name: "lookup".to_string(),
arguments: "{}".to_string(),
},
)]),
ChatMessage::tool_response("{}", "call-unknown"),
],
..ChatRequest::for_test()
};
let error = test_renderer(false).render(&request).unwrap_err();
expect![
"chat template error: invalid Harmony tool message: unknown tool_call_id `call-unknown`"
]
.assert_eq(&error.to_report_string());
}
#[test]
fn rejects_multimodal_input() {
let request = ChatRequest {
messages: vec![ChatMessage::user(vec![ChatContentPart::image_url(
"data:image/png;base64,test",
)])],
..ChatRequest::for_test()
};
let error = test_renderer(false).render(&request).unwrap_err();
assert!(matches!(
error,
Error::UnsupportedMultimodalContent("image_url")
));
}
#[test]
fn rejects_continue_final_assistant() {
let mut request = ChatRequest {
messages: vec![
ChatMessage::user("write"),
ChatMessage::assistant_text("partial"),
],
..ChatRequest::for_test()
};
request.chat_options.generation_prompt_mode = GenerationPromptMode::ContinueFinalAssistant;
let error = test_renderer(false).render(&request).unwrap_err();
expect!["chat template error: Harmony renderer does not support continue_final_message"]
.assert_eq(&error.to_report_string());
}
#[test]
fn no_generation_prompt_omits_trailing_assistant_start() {
let mut request = fixture_request("simple_user.json");
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
let rendered = render_prompt_text(&request);
assert!(!rendered.ends_with("<|start|>assistant"));
}
+2
View File
@@ -9,6 +9,7 @@ use crate::request::{ChatRequest, ReasoningEffort};
pub mod deepseek_v32;
pub mod deepseek_v4;
pub mod harmony;
pub mod hf;
mod selection;
#[cfg(test)]
@@ -16,6 +17,7 @@ mod test_utils;
pub use deepseek_v4::DeepSeekV4ChatRenderer;
pub use deepseek_v32::DeepSeekV32ChatRenderer;
pub use harmony::HarmonyChatRenderer;
pub use selection::RendererSelection;
/// Rendered chat prompt submitted to the text backend.
+9 -1
View File
@@ -19,12 +19,16 @@ pub enum RendererSelection {
DeepSeekV32,
/// Force the DeepSeek V4 renderer.
DeepSeekV4,
/// Force the GPT-OSS Harmony renderer.
Harmony,
}
impl RendererSelection {
pub const AUTO_LITERAL: &str = "auto";
pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32";
pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4";
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
pub const HARMONY_LITERAL: &str = "harmony";
pub const HF_LITERAL: &str = "hf";
/// Resolve the renderer selection using the given model type string, if
@@ -34,6 +38,7 @@ impl RendererSelection {
Self::Auto => match model_type {
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
_ => Self::Hf,
},
selection => selection,
@@ -53,6 +58,8 @@ impl FromStr for RendererSelection {
Ok(Self::DeepSeekV32)
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) {
Ok(Self::DeepSeekV4)
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
Ok(Self::Harmony)
} else {
Err(format!(
"unknown renderer `{value}` (expected one of: {})",
@@ -69,6 +76,7 @@ impl fmt::Display for RendererSelection {
Self::Hf => f.write_str(Self::HF_LITERAL),
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
}
}
}
@@ -95,7 +103,7 @@ mod tests {
fn renderer_selection_expected_error_message() {
let err = RendererSelection::from_str("unknown").unwrap_err();
expect_test::expect![
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4)"
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)"
]
.assert_eq(&err);
}
+4 -1
View File
@@ -7,7 +7,7 @@ use serde_json::Value;
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::request::{
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
GenerationPromptMode,
GenerationPromptMode, ReasoningEffort,
};
/// Options for constructing a [`ChatRequest`] from a fixture file.
@@ -42,6 +42,7 @@ pub(crate) struct FixtureRequest {
tools: Vec<FixtureTool>,
messages: Vec<FixtureMessage>,
add_generation_prompt: Option<bool>,
reasoning_effort: Option<ReasoningEffort>,
}
impl FixtureFile {
@@ -52,6 +53,7 @@ impl FixtureFile {
tools: Vec::new(),
messages,
add_generation_prompt: None,
reasoning_effort: None,
},
}
}
@@ -154,6 +156,7 @@ impl FixtureRequest {
if self.add_generation_prompt == Some(false) {
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
}
request.chat_options.reasoning_effort = self.reasoning_effort;
if options.enable_thinking {
for key in ["thinking", "enable_thinking"] {
request.chat_options.template_kwargs.insert(key.to_string(), Value::Bool(true));
+1 -1
View File
@@ -468,7 +468,7 @@ fn serve_args_reject_unknown_renderer_value() {
.unwrap_err();
expect![[r#"
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4)
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)
For more information, try '--help'.
"#]]