[Bugfix][Rust Frontend] Support raise_exception in chat templates (#59020)
Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yashasvi Asthana <yashasvi.asthana@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude
Bugen Zhao
parent
77d9cda688
commit
9af952c555
@@ -17,6 +17,8 @@ pub enum Error {
|
||||
MissingChatTemplate,
|
||||
#[error("chat template error: {0}")]
|
||||
ChatTemplate(String),
|
||||
#[error("{message}")]
|
||||
ChatTemplateThrown { message: String },
|
||||
#[error("{0}")]
|
||||
InvalidReasoningEffort(String),
|
||||
#[error("{message}")]
|
||||
@@ -112,6 +114,7 @@ impl Error {
|
||||
pub fn is_request_validation_error(&self) -> bool {
|
||||
match self {
|
||||
Self::PromptTooLong { .. }
|
||||
| Self::ChatTemplateThrown { .. }
|
||||
| Self::InvalidReasoningEffort(_)
|
||||
| Self::InvalidReasoningControl { .. }
|
||||
| Self::DuplicateToolName { .. }
|
||||
|
||||
@@ -226,7 +226,12 @@ impl HfChatRenderer {
|
||||
template_kwargs: Some(&effective_template_kwargs),
|
||||
special_tokens: self.special_tokens.as_ref(),
|
||||
})
|
||||
.map_err(|error| Error::ChatTemplate(error.to_report_string()))?;
|
||||
.map_err(|error| match error.thrown_message() {
|
||||
Some(message) => Error::ChatTemplateThrown {
|
||||
message: message.to_owned(),
|
||||
},
|
||||
None => Error::ChatTemplate(error.to_report_string()),
|
||||
})?;
|
||||
|
||||
let prompt = match &final_message_text {
|
||||
Some(final_message_text) => {
|
||||
@@ -973,6 +978,23 @@ mod tests {
|
||||
assert!(matches!(error, Error::ChatTemplate(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raise_exception_in_template_is_a_request_validation_error() {
|
||||
let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hi")]);
|
||||
|
||||
let error = render(
|
||||
Some("{{- raise_exception('No user query found in messages.') }}"),
|
||||
&request,
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(&error, Error::ChatTemplateThrown { message } if message == "No user query found in messages."),
|
||||
"expected an exception thrown by the template, got: {error:?}"
|
||||
);
|
||||
assert!(error.is_request_validation_error());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_flattens_text_parts_for_string_templates() {
|
||||
let request = sample_request(vec![ChatMessage::user(vec![
|
||||
|
||||
@@ -8,12 +8,15 @@
|
||||
//! - special tokens are wired to `vllm_text::backends::hf::HfSpecialTokens`
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error as _;
|
||||
use std::fs;
|
||||
use std::iter;
|
||||
use std::path::Path;
|
||||
|
||||
use minijinja::Environment;
|
||||
use minijinja::{Environment, Error as MinijinjaError, ErrorKind, Value};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{self};
|
||||
use thiserror::Error as ThisError;
|
||||
use vllm_text::backend::hf::HfSpecialTokens;
|
||||
|
||||
use super::error::TemplateError;
|
||||
@@ -38,10 +41,44 @@ fn build_environment(template: String) -> Result<Environment<'static>> {
|
||||
env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback);
|
||||
env.add_filter("tojson", hf_tojson_filter);
|
||||
env.add_function("__hf_generation", render_generation);
|
||||
env.add_function("raise_exception", raise_exception);
|
||||
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Error source attached by [`raise_exception`], so that an exception thrown
|
||||
/// by the template can be told apart from a template that fails to render.
|
||||
#[derive(Debug, ThisError)]
|
||||
#[error("{message}")]
|
||||
struct ThrownTemplateException {
|
||||
message: String,
|
||||
}
|
||||
|
||||
/// Throw an exception with the template's own message, aborting rendering.
|
||||
fn raise_exception(message: Value) -> std::result::Result<Value, MinijinjaError> {
|
||||
let thrown = ThrownTemplateException {
|
||||
message: message.to_string(),
|
||||
};
|
||||
let error = MinijinjaError::new(
|
||||
ErrorKind::InvalidOperation,
|
||||
"chat template threw an exception",
|
||||
);
|
||||
Err(error.with_source(thrown))
|
||||
}
|
||||
|
||||
impl TemplateError {
|
||||
/// Return the template's own message if rendering was aborted by
|
||||
/// [`raise_exception`].
|
||||
pub(super) fn thrown_message(&self) -> Option<&str> {
|
||||
let Self::Jinja(error) = self else {
|
||||
return None;
|
||||
};
|
||||
iter::successors(error.source(), |&error| error.source())
|
||||
.find_map(|error| error.downcast_ref::<ThrownTemplateException>())
|
||||
.map(|thrown| thrown.message.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Default, Serialize)]
|
||||
pub(super) struct TemplateContext<'a> {
|
||||
@@ -189,6 +226,59 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_raise_exception_returns_template_message() {
|
||||
let template = CompiledChatTemplate::new(
|
||||
"{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '.') }}"
|
||||
.to_string(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
let mut kwargs = HashMap::new();
|
||||
kwargs.insert("reasoning_effort".to_string(), serde_json::json!("high"));
|
||||
|
||||
let error = template
|
||||
.apply(TemplateContext {
|
||||
template_kwargs: Some(&kwargs),
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(
|
||||
error.thrown_message(),
|
||||
Some("Unexpected reasoning effort high."),
|
||||
"expected an exception thrown by the template, got: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_raise_exception_in_untaken_branch_renders() {
|
||||
let template = CompiledChatTemplate::new(
|
||||
"{% if messages %}{{ raise_exception('unreachable') }}{% endif %}ok".to_string(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(template.apply(TemplateContext::default()).unwrap(), "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_function_is_not_reported_as_thrown() {
|
||||
let template = CompiledChatTemplate::new(
|
||||
"{{ undefined_helper('not registered') }}".to_string(),
|
||||
ChatTemplateContentFormatOption::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let error = template.apply(TemplateContext::default()).unwrap_err();
|
||||
|
||||
assert_eq!(
|
||||
error.thrown_message(),
|
||||
None,
|
||||
"expected a jinja render error, got: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_special_tokens_injected_into_context() {
|
||||
let template = "{{ bos_token }}hello{{ eos_token }}";
|
||||
|
||||
@@ -193,6 +193,27 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_throw_maps_to_invalid_request() {
|
||||
let error = vllm_chat::Error::ChatTemplateThrown {
|
||||
message: "Unexpected reasoning effort high.".to_string(),
|
||||
};
|
||||
let api_error = chat_submit_error("failed to submit chat request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert_eq!(response.error.message, "Unexpected reasoning effort high.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_template_render_failure_stays_internal() {
|
||||
let error = vllm_chat::Error::ChatTemplate(
|
||||
"failed to render jinja template: unknown function".to_string(),
|
||||
);
|
||||
let api_error = chat_submit_error("failed to submit chat request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn llm_wrapped_empty_prompt_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds {
|
||||
|
||||
Reference in New Issue
Block a user