[Rust Frontend] Add Harmony Renderer for GPT-OSS (#46800)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Generated
+1
@@ -5049,6 +5049,7 @@ dependencies = [
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"thiserror-ext",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
|
||||
@@ -25,6 +25,7 @@ strum.workspace = true
|
||||
subenum.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
time.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
trait-set.workspace = true
|
||||
|
||||
@@ -15,7 +15,9 @@ use crate::output::{
|
||||
DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides,
|
||||
};
|
||||
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
|
||||
use crate::renderer::{DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer};
|
||||
use crate::renderer::{
|
||||
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
|
||||
};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{DynChatOutputProcessor, RendererSelection};
|
||||
|
||||
@@ -61,6 +63,7 @@ impl HfChatBackend {
|
||||
)?),
|
||||
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
|
||||
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
|
||||
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
|
||||
};
|
||||
|
||||
info!(
|
||||
@@ -148,13 +151,15 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use tempfile::tempdir;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::backend::hf::TokenizerSource;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
|
||||
use super::HfChatBackend;
|
||||
use crate::RendererSelection;
|
||||
use crate::backend::{ChatBackend, LoadModelBackendsOptions};
|
||||
use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};
|
||||
use crate::request::{ChatContent, ChatMessage, ChatRequest};
|
||||
use crate::{ParserSelection, RendererSelection};
|
||||
|
||||
fn request_with_user_text(text: &str) -> ChatRequest {
|
||||
ChatRequest {
|
||||
@@ -219,12 +224,12 @@ mod tests {
|
||||
Arc::new(TestTokenizer)
|
||||
}
|
||||
|
||||
fn render_prompt(
|
||||
fn backend_for_selection(
|
||||
renderer: RendererSelection,
|
||||
config_json: &str,
|
||||
tokenizer_config_json: &str,
|
||||
) -> String {
|
||||
let backend = HfChatBackend::from_resolved_model_files(
|
||||
) -> HfChatBackend {
|
||||
HfChatBackend::from_resolved_model_files(
|
||||
resolved_files(config_json, tokenizer_config_json),
|
||||
"test-model".to_string(),
|
||||
LoadModelBackendsOptions {
|
||||
@@ -236,9 +241,15 @@ mod tests {
|
||||
},
|
||||
test_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
backend
|
||||
fn render_prompt(
|
||||
renderer: RendererSelection,
|
||||
config_json: &str,
|
||||
tokenizer_config_json: &str,
|
||||
) -> String {
|
||||
backend_for_selection(renderer, config_json, tokenizer_config_json)
|
||||
.chat_renderer()
|
||||
.render(&request_with_user_text("hello"))
|
||||
.unwrap()
|
||||
@@ -272,6 +283,35 @@ mod tests {
|
||||
assert_eq!(prompt, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_uses_harmony_renderer_and_output_processor_for_gpt_oss_model_type() {
|
||||
let backend = backend_for_selection(
|
||||
RendererSelection::Auto,
|
||||
r#"{"model_type":"gpt_oss"}"#,
|
||||
r#"{"chat_template":"{{ messages[0].content }}"}"#,
|
||||
);
|
||||
|
||||
let prompt =
|
||||
backend.chat_renderer().render(&request_with_user_text("hello")).unwrap().prompt;
|
||||
assert!(matches!(prompt, Prompt::TokenIds(_)));
|
||||
|
||||
let mut request = request_with_user_text("hello");
|
||||
let error = match backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
|
||||
reasoning_parser: &ParserSelection::Auto,
|
||||
},
|
||||
) {
|
||||
Ok(_) => panic!("gpt_oss should reject generic parser overrides"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(
|
||||
error.to_report_string(),
|
||||
"gpt_oss uses native Harmony output parsing; generic tool parser override `json` is not supported"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn language_model_only_skips_multimodal_preprocessor_config() {
|
||||
let mut files = resolved_files(
|
||||
|
||||
@@ -29,8 +29,8 @@ pub use parser::reasoning::{
|
||||
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, RenderedPrompt,
|
||||
RendererSelection,
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
|
||||
HarmonyChatRenderer, RenderedPrompt, RendererSelection,
|
||||
};
|
||||
pub use request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
|
||||
|
||||
@@ -4,16 +4,10 @@
|
||||
//! `DecodedTextEvent` token IDs directly and lets the official `openai-harmony`
|
||||
//! parser recover the structured assistant message shape at token granularity.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use anyhow::Context;
|
||||
use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use futures::StreamExt as _;
|
||||
use openai_harmony::chat::{Content as HarmonyContent, Message as HarmonyMessage, Role};
|
||||
use openai_harmony::{
|
||||
HarmonyEncoding, HarmonyEncodingName, StreamableParser, load_harmony_encoding,
|
||||
};
|
||||
use thiserror_ext::AsReport;
|
||||
use openai_harmony::{HarmonyEncoding, StreamableParser};
|
||||
use vllm_text::output::DecodedTextEvent;
|
||||
|
||||
use crate::Result as ChatResult;
|
||||
@@ -24,6 +18,7 @@ use crate::output::{
|
||||
generate_tool_call_id,
|
||||
};
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::renderer::harmony::encoding::harmony_encoding;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
/// Request-scoped Harmony output processor used for `model_type == "gpt_oss"`.
|
||||
@@ -384,18 +379,6 @@ async fn harmony_assistant_event_stream(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Lazily load the shared GPT-OSS Harmony encoding once per process.
|
||||
fn harmony_encoding() -> Result<&'static HarmonyEncoding> {
|
||||
static ENCODING: LazyLock<anyhow::Result<HarmonyEncoding>> = LazyLock::new(|| {
|
||||
load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss)
|
||||
.context("failed to load harmony encoding for gpt-oss")
|
||||
});
|
||||
|
||||
ENCODING.as_ref().map_err(|error| Error::HarmonyOutputParsing {
|
||||
error: error.to_report_string().into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn harmony_output_parsing_error(
|
||||
error: impl Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
) -> Error {
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
//! Shared Harmony encoding helper for the GPT-OSS renderer and output parser.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use anyhow::Context as _;
|
||||
use openai_harmony::{HarmonyEncoding, HarmonyEncodingName, load_harmony_encoding};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// Lazily load the shared GPT-OSS Harmony encoding once per process.
|
||||
pub(crate) fn harmony_encoding() -> Result<&'static HarmonyEncoding> {
|
||||
static ENCODING: LazyLock<anyhow::Result<HarmonyEncoding>> = LazyLock::new(|| {
|
||||
load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss)
|
||||
.context("failed to load harmony encoding for gpt-oss")
|
||||
});
|
||||
|
||||
ENCODING.as_ref().map_err(|error| Error::HarmonyOutputParsing {
|
||||
error: error.to_report_string().into(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"add_generation_prompt": false,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 2 + 2?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "Need simple arithmetic.",
|
||||
"content": "4"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: medium
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>4<|end|>
|
||||
@@ -0,0 +1,27 @@
|
||||
[
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Use tools when needed.",
|
||||
"tools": [
|
||||
{
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"description": "Lookup a record.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["id"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Find record abc."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,23 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: medium
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.
|
||||
Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions
|
||||
|
||||
Use tools when needed.
|
||||
|
||||
# Tools
|
||||
|
||||
## functions
|
||||
|
||||
namespace functions {
|
||||
|
||||
// Lookup a record.
|
||||
type lookup = (_: {
|
||||
id: string,
|
||||
}) => any;
|
||||
|
||||
} // namespace functions<|end|><|start|>user<|message|>Find record abc.<|end|><|start|>assistant
|
||||
@@ -0,0 +1,15 @@
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 2 + 2?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "This should be dropped.",
|
||||
"content": "4"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 3 + 5?"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,7 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: medium
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>4<|end|><|start|>user<|message|>What is 3 + 5?<|end|><|start|>assistant
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"reasoning_effort": "high",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Answer tersely."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 2 + 2?"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: high
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions
|
||||
|
||||
Answer tersely.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["city"]
|
||||
},
|
||||
"strict": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Check Hangzhou weather."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: medium
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.
|
||||
Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Tools
|
||||
|
||||
## functions
|
||||
|
||||
namespace functions {
|
||||
|
||||
// Get weather for a city.
|
||||
type get_weather = (_: {
|
||||
city: string,
|
||||
}) => any;
|
||||
|
||||
} // namespace functions<|end|><|start|>user<|message|>Check Hangzhou weather.<|end|><|start|>assistant
|
||||
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, who are you?"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,7 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: medium
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>Hello, who are you?<|end|><|start|>assistant
|
||||
@@ -0,0 +1,8 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Answer tersely.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: high
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["city"]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Check Hangzhou weather."
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "Need current weather.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-weather",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"city\":\"Hangzhou\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call-weather",
|
||||
"content": "{\"temperature\":20}"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
|
||||
Knowledge cutoff: 2024-06
|
||||
Current date: 2025-06-28
|
||||
|
||||
Reasoning: medium
|
||||
|
||||
# Valid channels: analysis, commentary, final. Channel must be included for every message.
|
||||
Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Tools
|
||||
|
||||
## functions
|
||||
|
||||
namespace functions {
|
||||
|
||||
// Get weather for a city.
|
||||
type get_weather = (_: {
|
||||
city: string,
|
||||
}) => any;
|
||||
|
||||
} // namespace functions<|end|><|start|>user<|message|>Check Hangzhou weather.<|end|><|start|>assistant<|channel|>analysis<|message|>Need current weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{"city":"Hangzhou"}<|call|><|start|>functions.get_weather<|channel|>commentary to=assistant<|message|>{"temperature":20}<|end|><|start|>assistant
|
||||
@@ -0,0 +1,487 @@
|
||||
//! Native Harmony chat renderer for `gpt_oss`.
|
||||
|
||||
pub(crate) mod encoding;
|
||||
|
||||
use openai_harmony::HarmonyEncoding;
|
||||
use openai_harmony::chat::{
|
||||
Author, Conversation, DeveloperContent, Message, ReasoningEffort as HarmonyReasoningEffort,
|
||||
Role, SystemContent, ToolDescription,
|
||||
};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use time::macros::format_description;
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use self::encoding::harmony_encoding;
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::event::AssistantContentBlock;
|
||||
use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, GenerationPromptMode};
|
||||
use crate::{AssistantMessageExt as _, ReasoningEffort};
|
||||
|
||||
const SYSTEM_START_DATE_ENV: &str = "VLLM_SYSTEM_START_DATE";
|
||||
const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS";
|
||||
|
||||
/// GPT-OSS renderer backed by the official Harmony encoding.
|
||||
pub struct HarmonyChatRenderer {
|
||||
encoding: &'static HarmonyEncoding,
|
||||
options: Options,
|
||||
}
|
||||
|
||||
struct Options {
|
||||
system_start_date: String,
|
||||
use_system_instructions: bool,
|
||||
}
|
||||
|
||||
impl HarmonyChatRenderer {
|
||||
/// Create a Harmony renderer for production use.
|
||||
///
|
||||
/// Environment-derived options are resolved once at construction time:
|
||||
///
|
||||
/// - `VLLM_SYSTEM_START_DATE` pins the Harmony system start date. When it is
|
||||
/// unset, the renderer uses the current local date with a UTC fallback.
|
||||
/// - `VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS` moves leading instructions
|
||||
/// into the system model identity when set to a non-zero integer.
|
||||
pub fn new() -> Result<Self> {
|
||||
Self::with_options(
|
||||
env_system_start_date(),
|
||||
env_use_harmony_system_instructions(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Create a Harmony renderer with explicit preamble options.
|
||||
///
|
||||
/// Tests use this constructor to avoid process-global environment mutation.
|
||||
/// Production code should call [`Self::new`] so the renderer observes the
|
||||
/// same environment contract as the Python Harmony path.
|
||||
pub fn with_options(
|
||||
system_start_date: impl Into<String>,
|
||||
use_system_instructions: bool,
|
||||
) -> Result<Self> {
|
||||
Ok(Self {
|
||||
encoding: harmony_encoding()?,
|
||||
options: Options {
|
||||
system_start_date: system_start_date.into(),
|
||||
use_system_instructions,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Render a chat request directly to Harmony token IDs.
|
||||
///
|
||||
/// Harmony owns both prompt formatting and tokenization, so the Rust
|
||||
/// frontend bypasses the generic HF tokenizer path for GPT-OSS input.
|
||||
fn render_token_ids(&self, request: &ChatRequest) -> Result<Vec<u32>> {
|
||||
if request.has_multimodal() {
|
||||
return Err(Error::UnsupportedMultimodalContent("image_url"));
|
||||
}
|
||||
if matches!(
|
||||
request.chat_options.generation_prompt_mode,
|
||||
GenerationPromptMode::ContinueFinalAssistant
|
||||
) {
|
||||
return Err(Error::ChatTemplate(
|
||||
"Harmony renderer does not support continue_final_message".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let messages = auto_drop_analysis_messages(to_harmony_messages(request, &self.options)?);
|
||||
let conversation = Conversation::from_messages(messages);
|
||||
// Pass `None` so oss-harmony does not apply its narrower built-in
|
||||
// analysis-drop policy after the Rust-side Python-parity cleanup above.
|
||||
let token_ids = match request.chat_options.generation_prompt_mode {
|
||||
GenerationPromptMode::StartNewAssistant => self
|
||||
.encoding
|
||||
.render_conversation_for_completion(&conversation, Role::Assistant, None),
|
||||
GenerationPromptMode::NoGenerationPrompt => {
|
||||
self.encoding.render_conversation(&conversation, None)
|
||||
}
|
||||
GenerationPromptMode::ContinueFinalAssistant => unreachable!("checked above"),
|
||||
}
|
||||
.map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"failed to render Harmony prompt: {}",
|
||||
error.as_report()
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok(token_ids)
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for HarmonyChatRenderer {
|
||||
/// Render a chat request as [`Prompt::TokenIds`] with template kwargs echoed
|
||||
/// for downstream accounting/debugging.
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::TokenIds(self.render_token_ids(request)?),
|
||||
effective_template_kwargs: request_template_kwargs(request),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a vLLM chat request into a full Harmony conversation.
|
||||
///
|
||||
/// This adds the Harmony system/developer preamble, peels at most one leading
|
||||
/// system/developer instruction message, and then lowers the remaining chat
|
||||
/// history message-by-message.
|
||||
fn to_harmony_messages(request: &ChatRequest, options: &Options) -> Result<Vec<Message>> {
|
||||
let (instructions, leading_developer_tools, remaining_messages) =
|
||||
peel_leading_instructions(&request.messages)?;
|
||||
let tool_call_names = tool_call_names(&request.messages);
|
||||
let mut messages =
|
||||
build_harmony_preamble(request, instructions, leading_developer_tools, options)?;
|
||||
|
||||
for message in remaining_messages {
|
||||
messages.extend(to_harmony_message(message, &tool_call_names, options)?);
|
||||
}
|
||||
|
||||
Ok(messages)
|
||||
}
|
||||
|
||||
/// Extract the optional leading instruction message used by the Harmony preamble.
|
||||
///
|
||||
/// Python only peels the first leading `system` or `developer` message. Later
|
||||
/// system/developer messages stay in the conversation and are lowered normally.
|
||||
#[allow(clippy::type_complexity)]
|
||||
fn peel_leading_instructions(
|
||||
messages: &[ChatMessage],
|
||||
) -> Result<(Option<String>, Option<&[ChatTool]>, &[ChatMessage])> {
|
||||
let Some(first) = messages.first() else {
|
||||
return Ok((None, None, messages));
|
||||
};
|
||||
|
||||
match first {
|
||||
ChatMessage::System { content } => Ok((Some(flatten_text(content)?), None, &messages[1..])),
|
||||
ChatMessage::Developer { content, tools } => Ok((
|
||||
Some(flatten_text(content)?),
|
||||
tools.as_deref(),
|
||||
&messages[1..],
|
||||
)),
|
||||
ChatMessage::User { .. }
|
||||
| ChatMessage::Assistant { .. }
|
||||
| ChatMessage::ToolResponse { .. } => Ok((None, None, messages)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the Harmony preamble for one request.
|
||||
///
|
||||
/// The preamble always contains a system message with date and reasoning-effort
|
||||
/// metadata. Leading instructions live either in the system model identity or in
|
||||
/// a developer message depending on `use_system_instructions`; request-level and
|
||||
/// leading developer tools are attached to the developer message.
|
||||
fn build_harmony_preamble(
|
||||
request: &ChatRequest,
|
||||
instructions: Option<String>,
|
||||
leading_developer_tools: Option<&[ChatTool]>,
|
||||
options: &Options,
|
||||
) -> Result<Vec<Message>> {
|
||||
let mut messages = vec![Message::from_role_and_content(
|
||||
Role::System,
|
||||
system_content(
|
||||
instructions.as_deref().filter(|_| options.use_system_instructions),
|
||||
request.chat_options.reasoning_effort,
|
||||
&options.system_start_date,
|
||||
)?,
|
||||
)];
|
||||
|
||||
let mut developer = DeveloperContent::new();
|
||||
let mut has_developer_content = false;
|
||||
|
||||
if !options.use_system_instructions
|
||||
&& let Some(instructions) = instructions.as_deref().filter(|text| !text.is_empty())
|
||||
{
|
||||
developer = developer.with_instructions(instructions);
|
||||
has_developer_content = true;
|
||||
}
|
||||
|
||||
let tool_descriptions = preamble_tool_descriptions(request, leading_developer_tools);
|
||||
if !tool_descriptions.is_empty() {
|
||||
developer = developer.with_function_tools(tool_descriptions);
|
||||
has_developer_content = true;
|
||||
}
|
||||
|
||||
if has_developer_content {
|
||||
messages.push(Message::from_role_and_content(Role::Developer, developer));
|
||||
}
|
||||
|
||||
Ok(messages)
|
||||
}
|
||||
|
||||
/// Collect request-level and leading developer function tools for the preamble.
|
||||
fn preamble_tool_descriptions(
|
||||
request: &ChatRequest,
|
||||
leading_developer_tools: Option<&[ChatTool]>,
|
||||
) -> Vec<ToolDescription> {
|
||||
let mut tools = Vec::new();
|
||||
if request.tool_parsing_enabled() {
|
||||
tools.extend(to_tool_descriptions(&request.tools));
|
||||
}
|
||||
if let Some(leading_developer_tools) = leading_developer_tools {
|
||||
tools.extend(to_tool_descriptions(leading_developer_tools));
|
||||
}
|
||||
tools
|
||||
}
|
||||
|
||||
/// Construct the Harmony system content for the request preamble.
|
||||
///
|
||||
/// Harmony defaults the reasoning effort to `medium` when none is provided, so
|
||||
/// this only sets an explicit effort after validating vLLM's request value.
|
||||
fn system_content(
|
||||
instructions: Option<&str>,
|
||||
reasoning_effort: Option<ReasoningEffort>,
|
||||
system_start_date: &str,
|
||||
) -> Result<SystemContent> {
|
||||
let mut content =
|
||||
SystemContent::new().with_conversation_start_date(system_start_date.to_string());
|
||||
|
||||
if let Some(reasoning_effort) = reasoning_effort {
|
||||
content = content.with_reasoning_effort(to_harmony_reasoning_effort(reasoning_effort)?);
|
||||
}
|
||||
|
||||
if let Some(instructions) = instructions.filter(|text| !text.is_empty()) {
|
||||
let model_identity = match content.model_identity.as_deref() {
|
||||
Some(identity) if !identity.is_empty() => format!("{identity}\n{instructions}"),
|
||||
_ => instructions.to_string(),
|
||||
};
|
||||
content = content.with_model_identity(model_identity);
|
||||
}
|
||||
|
||||
Ok(content)
|
||||
}
|
||||
|
||||
/// Lower a single vLLM chat message into one or more Harmony messages.
|
||||
///
|
||||
/// Assistant messages can split into separate analysis, final, commentary, and
|
||||
/// tool-call messages. Tool responses require the earlier assistant tool-call ID
|
||||
/// map so the Harmony tool author can include `functions.{name}`.
|
||||
fn to_harmony_message(
|
||||
message: &ChatMessage,
|
||||
tool_call_names: &std::collections::HashMap<String, String>,
|
||||
options: &Options,
|
||||
) -> Result<Vec<Message>> {
|
||||
Ok(match message {
|
||||
ChatMessage::System { content } => {
|
||||
let instructions = flatten_text(content)?;
|
||||
vec![system_or_developer_message(
|
||||
"system",
|
||||
instructions,
|
||||
None,
|
||||
options,
|
||||
)?]
|
||||
}
|
||||
ChatMessage::Developer { content, tools } => {
|
||||
let instructions = flatten_text(content)?;
|
||||
vec![developer_message(Some(instructions), tools.as_deref())]
|
||||
}
|
||||
ChatMessage::User { content } => {
|
||||
vec![Message::from_role_and_content(
|
||||
Role::User,
|
||||
flatten_text(content)?,
|
||||
)]
|
||||
}
|
||||
ChatMessage::Assistant { content } => assistant_messages(content),
|
||||
ChatMessage::ToolResponse {
|
||||
content,
|
||||
tool_call_id,
|
||||
} => {
|
||||
let name = tool_call_names.get(tool_call_id).ok_or_else(|| {
|
||||
Error::ChatTemplate(format!(
|
||||
"invalid Harmony tool message: unknown tool_call_id `{tool_call_id}`"
|
||||
))
|
||||
})?;
|
||||
vec![
|
||||
Message::from_author_and_content(
|
||||
Author::new(Role::Tool, format!("functions.{name}")),
|
||||
flatten_text(content)?,
|
||||
)
|
||||
.with_channel("commentary")
|
||||
.with_recipient("assistant"),
|
||||
]
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Lower a non-leading system/developer message.
|
||||
///
|
||||
/// Harmony treats most extra system/developer messages as developer
|
||||
/// instructions. When system-instructions mode is enabled, system messages are
|
||||
/// rendered as system model-identity additions to match Python.
|
||||
fn system_or_developer_message(
|
||||
role: &str,
|
||||
instructions: String,
|
||||
tools: Option<&[ChatTool]>,
|
||||
options: &Options,
|
||||
) -> Result<Message> {
|
||||
if role == "system" && options.use_system_instructions {
|
||||
return Ok(Message::from_role_and_content(
|
||||
Role::System,
|
||||
system_content(Some(&instructions), None, &options.system_start_date)?,
|
||||
));
|
||||
}
|
||||
|
||||
Ok(developer_message(Some(instructions), tools))
|
||||
}
|
||||
|
||||
/// Build a Harmony developer message with optional instructions and function tools.
|
||||
fn developer_message(instructions: Option<String>, tools: Option<&[ChatTool]>) -> Message {
|
||||
let mut content = DeveloperContent::new();
|
||||
if let Some(instructions) = instructions.filter(|text| !text.is_empty()) {
|
||||
content = content.with_instructions(instructions);
|
||||
}
|
||||
if let Some(tools) = tools {
|
||||
let tools = to_tool_descriptions(tools);
|
||||
if !tools.is_empty() {
|
||||
content = content.with_function_tools(tools);
|
||||
}
|
||||
}
|
||||
Message::from_role_and_content(Role::Developer, content)
|
||||
}
|
||||
|
||||
/// Lower assistant history into Harmony channels.
|
||||
///
|
||||
/// Plain assistant text goes to `final`. When the assistant has tool calls,
|
||||
/// visible text goes to `commentary`, reasoning goes to `analysis`, and each
|
||||
/// function call becomes a `commentary` message to `functions.{name}` with JSON
|
||||
/// constrained content.
|
||||
fn assistant_messages(content: &[AssistantContentBlock]) -> Vec<Message> {
|
||||
let mut messages = Vec::new();
|
||||
let has_tool_calls = content.has_tool_calls();
|
||||
|
||||
if has_tool_calls {
|
||||
let text = content.text();
|
||||
if !text.is_empty() {
|
||||
messages.push(
|
||||
Message::from_role_and_content(Role::Assistant, text).with_channel("commentary"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(reasoning) = content.reasoning() {
|
||||
messages.push(
|
||||
Message::from_role_and_content(Role::Assistant, reasoning).with_channel("analysis"),
|
||||
);
|
||||
}
|
||||
|
||||
if has_tool_calls {
|
||||
for tool_call in content.tool_calls() {
|
||||
messages.push(
|
||||
Message::from_role_and_content(Role::Assistant, tool_call.arguments.clone())
|
||||
.with_channel("commentary")
|
||||
.with_recipient(format!("functions.{}", tool_call.name))
|
||||
.with_content_type("<|constrain|>json"),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
let text = content.text();
|
||||
if !text.is_empty() {
|
||||
messages
|
||||
.push(Message::from_role_and_content(Role::Assistant, text).with_channel("final"));
|
||||
}
|
||||
}
|
||||
|
||||
messages
|
||||
}
|
||||
|
||||
/// Build the tool-call ID to function-name map used by later tool responses.
|
||||
fn tool_call_names(messages: &[ChatMessage]) -> std::collections::HashMap<String, String> {
|
||||
let mut names = std::collections::HashMap::new();
|
||||
for message in messages {
|
||||
let ChatMessage::Assistant { content } = message else {
|
||||
continue;
|
||||
};
|
||||
for tool_call in content.tool_calls() {
|
||||
names.insert(tool_call.id.clone(), tool_call.name.clone());
|
||||
}
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
/// Drop stale assistant analysis messages using vLLM Python's policy.
|
||||
///
|
||||
/// Once an assistant final message exists, earlier analysis messages represent
|
||||
/// chain-of-thought for completed turns and should not be replayed to the model.
|
||||
fn auto_drop_analysis_messages(messages: Vec<Message>) -> Vec<Message> {
|
||||
// Match vLLM Python's Harmony cleanup: once an assistant final message exists,
|
||||
// previous assistant analysis messages are stale chain-of-thought and should
|
||||
// be removed. oss-harmony can also drop analysis with `Some(Default::default())`,
|
||||
// but that built-in path only triggers when the last assistant message is final
|
||||
// and drops relative to the first final message, which misses longer multi-turn
|
||||
// histories with later user/tool turns.
|
||||
let Some(last_assistant_final_index) = messages.iter().rposition(|message| {
|
||||
message.author.role == Role::Assistant && message.channel.as_deref() == Some("final")
|
||||
}) else {
|
||||
return messages;
|
||||
};
|
||||
|
||||
messages
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, message)| {
|
||||
(index >= last_assistant_final_index || message.channel.as_deref() != Some("analysis"))
|
||||
.then_some(message)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Flatten vLLM text content and reject unsupported multimodal parts.
|
||||
fn flatten_text(content: &ChatContent) -> Result<String> {
|
||||
content.try_flatten_to_text()
|
||||
}
|
||||
|
||||
/// Convert vLLM function tool definitions to Harmony tool descriptions.
|
||||
fn to_tool_descriptions(tools: &[ChatTool]) -> Vec<ToolDescription> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
ToolDescription::new(
|
||||
tool.name.clone(),
|
||||
tool.description.clone().unwrap_or_default(),
|
||||
Some(tool.parameters.clone()),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Map supported OpenAI reasoning-effort values onto Harmony's enum.
|
||||
fn to_harmony_reasoning_effort(
|
||||
reasoning_effort: ReasoningEffort,
|
||||
) -> Result<HarmonyReasoningEffort> {
|
||||
match reasoning_effort {
|
||||
ReasoningEffort::Low => Ok(HarmonyReasoningEffort::Low),
|
||||
ReasoningEffort::Medium => Ok(HarmonyReasoningEffort::Medium),
|
||||
ReasoningEffort::High => Ok(HarmonyReasoningEffort::High),
|
||||
ReasoningEffort::None
|
||||
| ReasoningEffort::Minimal
|
||||
| ReasoningEffort::XHigh
|
||||
| ReasoningEffort::Max => Err(Error::ChatTemplate(format!(
|
||||
"reasoning_effort={:?} is not supported by Harmony. Supported values are: low, medium, high.",
|
||||
reasoning_effort.as_str()
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the system start date from the environment or the current date.
|
||||
fn env_system_start_date() -> String {
|
||||
std::env::var(SYSTEM_START_DATE_ENV)
|
||||
.ok()
|
||||
.filter(|date| !date.is_empty())
|
||||
.unwrap_or_else(current_date)
|
||||
}
|
||||
|
||||
/// Format today's date as `YYYY-MM-DD`, preferring local time.
|
||||
fn current_date() -> String {
|
||||
const DATE_FORMAT: &[time::format_description::FormatItem<'static>] =
|
||||
format_description!("[year]-[month]-[day]");
|
||||
let now = time::OffsetDateTime::now_local().unwrap_or_else(|_| time::OffsetDateTime::now_utc());
|
||||
now.format(DATE_FORMAT).expect("static date format should be valid")
|
||||
}
|
||||
|
||||
/// Resolve the env flag that places leading instructions in system identity.
|
||||
fn env_use_harmony_system_instructions() -> bool {
|
||||
std::env::var(HARMONY_SYSTEM_INSTRUCTIONS_ENV)
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<i32>().ok())
|
||||
.is_some_and(|value| value != 0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,212 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use expect_test::{ExpectFile, expect, expect_file};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use super::HarmonyChatRenderer;
|
||||
use super::encoding::harmony_encoding;
|
||||
use crate::ChatRenderer;
|
||||
use crate::error::Error;
|
||||
use crate::event::{AssistantContentBlock, AssistantToolCall};
|
||||
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
|
||||
use crate::request::{
|
||||
ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort,
|
||||
};
|
||||
|
||||
const PINNED_DATE: &str = "2025-06-28";
|
||||
|
||||
fn fixture_request(input_name: &str) -> ChatRequest {
|
||||
fixture_chat_request(
|
||||
&fixture_path(input_name),
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: false,
|
||||
no_generation_prompt_when_last_assistant: false,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn fixture_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("src/renderer/harmony")
|
||||
.join("fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
fn test_renderer(use_system_instructions: bool) -> HarmonyChatRenderer {
|
||||
HarmonyChatRenderer::with_options(PINNED_DATE, use_system_instructions).unwrap()
|
||||
}
|
||||
|
||||
fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
|
||||
render_token_ids_with(&test_renderer(false), request)
|
||||
}
|
||||
|
||||
fn render_token_ids_with(renderer: &HarmonyChatRenderer, request: &ChatRequest) -> Vec<u32> {
|
||||
renderer
|
||||
.render(request)
|
||||
.unwrap()
|
||||
.prompt
|
||||
.into_token_ids()
|
||||
.expect("Harmony renderer returns token IDs")
|
||||
}
|
||||
|
||||
fn render_prompt_text(request: &ChatRequest) -> String {
|
||||
render_prompt_text_with(&test_renderer(false), request)
|
||||
}
|
||||
|
||||
fn render_prompt_text_with(renderer: &HarmonyChatRenderer, request: &ChatRequest) -> String {
|
||||
let token_ids = render_token_ids_with(renderer, request);
|
||||
harmony_encoding().unwrap().tokenizer().decode_utf8(&token_ids).unwrap()
|
||||
}
|
||||
|
||||
fn assert_fixture(input_name: &str, expected: ExpectFile) {
|
||||
let request = fixture_request(input_name);
|
||||
let rendered = format!("{}\n", render_prompt_text(&request));
|
||||
expected.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_token_ids() {
|
||||
let request = fixture_request("simple_user.json");
|
||||
|
||||
assert!(!render_token_ids(&request).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_simple_user_fixture() {
|
||||
assert_fixture("simple_user.json", expect_file!["fixtures/simple_user.txt"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_leading_system_fixture() {
|
||||
assert_fixture(
|
||||
"leading_system.json",
|
||||
expect_file!["fixtures/leading_system.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_system_instructions_env_fixture() {
|
||||
let renderer = test_renderer(true);
|
||||
let request = fixture_request("leading_system.json");
|
||||
let rendered = format!("{}\n", render_prompt_text_with(&renderer, &request));
|
||||
expect_file!["fixtures/system_instructions_env.txt"].assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_request_tools_fixture() {
|
||||
assert_fixture(
|
||||
"request_tools.json",
|
||||
expect_file!["fixtures/request_tools.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_developer_tools_fixture() {
|
||||
assert_fixture(
|
||||
"developer_tools.json",
|
||||
expect_file!["fixtures/developer_tools.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_assistant_history_fixture() {
|
||||
assert_fixture(
|
||||
"assistant_history.json",
|
||||
expect_file!["fixtures/assistant_history.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_tool_roundtrip_fixture() {
|
||||
assert_fixture(
|
||||
"tool_roundtrip.json",
|
||||
expect_file!["fixtures/tool_roundtrip.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drops_stale_analysis_fixture() {
|
||||
assert_fixture(
|
||||
"drop_analysis.json",
|
||||
expect_file!["fixtures/drop_analysis.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_reasoning_effort() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
|
||||
|
||||
let error = test_renderer(false).render(&request).unwrap_err();
|
||||
|
||||
expect![[r#"chat template error: reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_tool_response_id() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::assistant_blocks(vec![AssistantContentBlock::ToolCall(
|
||||
AssistantToolCall {
|
||||
id: "call-known".to_string(),
|
||||
name: "lookup".to_string(),
|
||||
arguments: "{}".to_string(),
|
||||
},
|
||||
)]),
|
||||
ChatMessage::tool_response("{}", "call-unknown"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let error = test_renderer(false).render(&request).unwrap_err();
|
||||
|
||||
expect![
|
||||
"chat template error: invalid Harmony tool message: unknown tool_call_id `call-unknown`"
|
||||
]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_multimodal_input() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::user(vec![ChatContentPart::image_url(
|
||||
"data:image/png;base64,test",
|
||||
)])],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let error = test_renderer(false).render(&request).unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::UnsupportedMultimodalContent("image_url")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_continue_final_assistant() {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::user("write"),
|
||||
ChatMessage::assistant_text("partial"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::ContinueFinalAssistant;
|
||||
|
||||
let error = test_renderer(false).render(&request).unwrap_err();
|
||||
|
||||
expect!["chat template error: Harmony renderer does not support continue_final_message"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_generation_prompt_omits_trailing_assistant_start() {
|
||||
let mut request = fixture_request("simple_user.json");
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
|
||||
let rendered = render_prompt_text(&request);
|
||||
|
||||
assert!(!rendered.ends_with("<|start|>assistant"));
|
||||
}
|
||||
@@ -9,6 +9,7 @@ use crate::request::{ChatRequest, ReasoningEffort};
|
||||
|
||||
pub mod deepseek_v32;
|
||||
pub mod deepseek_v4;
|
||||
pub mod harmony;
|
||||
pub mod hf;
|
||||
mod selection;
|
||||
#[cfg(test)]
|
||||
@@ -16,6 +17,7 @@ mod test_utils;
|
||||
|
||||
pub use deepseek_v4::DeepSeekV4ChatRenderer;
|
||||
pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use harmony::HarmonyChatRenderer;
|
||||
pub use selection::RendererSelection;
|
||||
|
||||
/// Rendered chat prompt submitted to the text backend.
|
||||
|
||||
@@ -19,12 +19,16 @@ pub enum RendererSelection {
|
||||
DeepSeekV32,
|
||||
/// Force the DeepSeek V4 renderer.
|
||||
DeepSeekV4,
|
||||
/// Force the GPT-OSS Harmony renderer.
|
||||
Harmony,
|
||||
}
|
||||
|
||||
impl RendererSelection {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const DEEPSEEK_V32_LITERAL: &str = "deepseek_v32";
|
||||
pub const DEEPSEEK_V4_LITERAL: &str = "deepseek_v4";
|
||||
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
|
||||
pub const HARMONY_LITERAL: &str = "harmony";
|
||||
pub const HF_LITERAL: &str = "hf";
|
||||
|
||||
/// Resolve the renderer selection using the given model type string, if
|
||||
@@ -34,6 +38,7 @@ impl RendererSelection {
|
||||
Self::Auto => match model_type {
|
||||
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
|
||||
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
|
||||
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
|
||||
_ => Self::Hf,
|
||||
},
|
||||
selection => selection,
|
||||
@@ -53,6 +58,8 @@ impl FromStr for RendererSelection {
|
||||
Ok(Self::DeepSeekV32)
|
||||
} else if value.eq_ignore_ascii_case(Self::DEEPSEEK_V4_LITERAL) {
|
||||
Ok(Self::DeepSeekV4)
|
||||
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
|
||||
Ok(Self::Harmony)
|
||||
} else {
|
||||
Err(format!(
|
||||
"unknown renderer `{value}` (expected one of: {})",
|
||||
@@ -69,6 +76,7 @@ impl fmt::Display for RendererSelection {
|
||||
Self::Hf => f.write_str(Self::HF_LITERAL),
|
||||
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
|
||||
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
|
||||
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -95,7 +103,7 @@ mod tests {
|
||||
fn renderer_selection_expected_error_message() {
|
||||
let err = RendererSelection::from_str("unknown").unwrap_err();
|
||||
expect_test::expect![
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4)"
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)"
|
||||
]
|
||||
.assert_eq(&err);
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use serde_json::Value;
|
||||
use crate::event::{AssistantContentBlock, AssistantToolCall};
|
||||
use crate::request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
|
||||
GenerationPromptMode,
|
||||
GenerationPromptMode, ReasoningEffort,
|
||||
};
|
||||
|
||||
/// Options for constructing a [`ChatRequest`] from a fixture file.
|
||||
@@ -42,6 +42,7 @@ pub(crate) struct FixtureRequest {
|
||||
tools: Vec<FixtureTool>,
|
||||
messages: Vec<FixtureMessage>,
|
||||
add_generation_prompt: Option<bool>,
|
||||
reasoning_effort: Option<ReasoningEffort>,
|
||||
}
|
||||
|
||||
impl FixtureFile {
|
||||
@@ -52,6 +53,7 @@ impl FixtureFile {
|
||||
tools: Vec::new(),
|
||||
messages,
|
||||
add_generation_prompt: None,
|
||||
reasoning_effort: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -154,6 +156,7 @@ impl FixtureRequest {
|
||||
if self.add_generation_prompt == Some(false) {
|
||||
request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
|
||||
}
|
||||
request.chat_options.reasoning_effort = self.reasoning_effort;
|
||||
if options.enable_thinking {
|
||||
for key in ["thinking", "enable_thinking"] {
|
||||
request.chat_options.template_kwargs.insert(key.to_string(), Value::Bool(true));
|
||||
|
||||
@@ -468,7 +468,7 @@ fn serve_args_reject_unknown_renderer_value() {
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4)
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
|
||||
Reference in New Issue
Block a user