[Bugfix][DeepSeek V4] Handle trailing system messages in prompt rendering (#51262)
Signed-off-by: jiahaoliang <gzliangjiahao@gmail.com> Signed-off-by: Jiahao Liang <gzliangjiahao@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -110,7 +110,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
|
||||
}
|
||||
}
|
||||
|
||||
if is_user_like_entry(message)
|
||||
if (is_user_like_entry(message) || matches!(message, ChatMessage::System { .. }))
|
||||
&& next_rendered_entry_is_assistant_or_end(request.messages.as_slice(), message_index)
|
||||
{
|
||||
write_assistant_transition(
|
||||
@@ -423,7 +423,7 @@ fn write_tool_result(out: &mut String, content: &ChatContent) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append the assistant transition token after a user-like turn.
|
||||
/// Append the assistant transition token after a user-like or system turn.
|
||||
fn write_assistant_transition(
|
||||
out: &mut String,
|
||||
thinking_mode: ThinkingMode,
|
||||
|
||||
@@ -195,6 +195,68 @@ fn reasoning_effort_none_disables_thinking() {
|
||||
.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_to_assistant_transitions_in_chat_and_thinking_modes() {
|
||||
let rendered = [false, true].map(|enable_thinking| {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::system("rules"),
|
||||
ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::Reasoning {
|
||||
text: "reason".to_string(),
|
||||
},
|
||||
AssistantContentBlock::Text {
|
||||
text: "answer".to_string(),
|
||||
},
|
||||
]),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
|
||||
render_request(&request)
|
||||
});
|
||||
|
||||
expect![[r#"
|
||||
[
|
||||
"<|begin▁of▁sentence|>rules<|Assistant|></think>answer<|end▁of▁sentence|>",
|
||||
"<|begin▁of▁sentence|>rules<|Assistant|><think>reason</think>answer<|end▁of▁sentence|>",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_system_transitions_in_chat_and_thinking_modes() {
|
||||
let rendered = [false, true].map(|enable_thinking| {
|
||||
let mut request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::system("rules"),
|
||||
ChatMessage::user("question"),
|
||||
ChatMessage::system("final rules"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
|
||||
request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
|
||||
render_request(&request)
|
||||
});
|
||||
|
||||
expect![[r#"
|
||||
[
|
||||
"<|begin▁of▁sentence|>rules<|User|>questionfinal rules<|Assistant|></think>",
|
||||
"<|begin▁of▁sentence|>rules<|User|>questionfinal rules<|Assistant|><think>",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_effort_template_kwarg_is_ignored() {
|
||||
let mut request = ChatRequest {
|
||||
|
||||
@@ -113,6 +113,83 @@ def test_deepseek_v4_explicitly_disables_thinking(kwargs):
|
||||
assert prompt == ("<|begin▁of▁sentence|><|User|>Hello<|Assistant|></think>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enable_thinking", "thinking_token"),
|
||||
[(False, "</think>"), (True, "<think>")],
|
||||
)
|
||||
def test_deepseek_v4_appends_assistant_transition_after_trailing_system(
|
||||
enable_thinking, thinking_token
|
||||
):
|
||||
prompt = _tokenizer().apply_chat_template(
|
||||
[
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{"role": "system", "content": "Be concise."},
|
||||
],
|
||||
tokenize=False,
|
||||
enable_thinking=enable_thinking,
|
||||
reasoning_effort="low",
|
||||
)
|
||||
|
||||
assert prompt == (
|
||||
"<|begin▁of▁sentence|>You are helpful."
|
||||
f"<|User|>What is 2+2?Be concise.<|Assistant|>{thinking_token}"
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_v4_does_not_transition_after_mid_conversation_system():
|
||||
prompt = _tokenizer().apply_chat_template(
|
||||
[
|
||||
{"role": "user", "content": "What is 3+3?"},
|
||||
{"role": "assistant", "content": "6"},
|
||||
{"role": "system", "content": "Answer in French."},
|
||||
{"role": "user", "content": "What is 5+5?"},
|
||||
],
|
||||
tokenize=False,
|
||||
enable_thinking=False,
|
||||
)
|
||||
|
||||
assert prompt == (
|
||||
"<|begin▁of▁sentence|><|User|>What is 3+3?"
|
||||
"<|Assistant|></think>6<|end▁of▁sentence|>"
|
||||
"Answer in French.<|User|>What is 5+5?"
|
||||
"<|Assistant|></think>"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enable_thinking", "expected"),
|
||||
[
|
||||
(
|
||||
False,
|
||||
"<|begin▁of▁sentence|>rules<|Assistant|></think>answer"
|
||||
"<|end▁of▁sentence|>",
|
||||
),
|
||||
(
|
||||
True,
|
||||
"<|begin▁of▁sentence|>rules<|Assistant|><think>reason</think>answer"
|
||||
"<|end▁of▁sentence|>",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_transitions_from_system_to_assistant(enable_thinking, expected):
|
||||
prompt = _tokenizer().apply_chat_template(
|
||||
[
|
||||
{"role": "system", "content": "rules"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "reason",
|
||||
"content": "answer",
|
||||
},
|
||||
],
|
||||
tokenize=False,
|
||||
enable_thinking=enable_thinking,
|
||||
reasoning_effort="low",
|
||||
)
|
||||
|
||||
assert prompt == expected
|
||||
|
||||
|
||||
def test_deepseek_v4_unknown_role_raises_value_error():
|
||||
# Invalid roles are client errors: they must surface as ValueError
|
||||
# (mapped to HTTP 400 by the OpenAI serving layer), not
|
||||
|
||||
@@ -374,7 +374,15 @@ def render_message(
|
||||
prompt += thinking_end_token if thinking_mode != "thinking" else thinking_start_token
|
||||
prompt += task_sp_token
|
||||
|
||||
elif messages[index].get("role") in ["user", "developer"]:
|
||||
# A trailing system message opens generation, while a system message
|
||||
# followed by assistant opens that assistant history turn.
|
||||
elif role in ["user", "developer"] or (
|
||||
role == "system"
|
||||
and (
|
||||
index == len(messages) - 1
|
||||
or messages[index + 1].get("role") == "assistant"
|
||||
)
|
||||
):
|
||||
# Normal generation: append Assistant + thinking token
|
||||
prompt += ASSISTANT_SP_TOKEN
|
||||
if not drop_thinking and thinking_mode == "thinking":
|
||||
|
||||
Reference in New Issue
Block a user