[Bugfix][Rust Frontend] Return 400 for prompt-validation submit errors (#45286)
Signed-off-by: xiaguan <751080330@qq.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
2043258dec
commit
1ae1051b4b
@@ -1,6 +1,7 @@
|
||||
use axum::Json;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use thiserror_ext::{Construct, Macro};
|
||||
|
||||
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
|
||||
@@ -72,3 +73,84 @@ impl IntoResponse for ApiError {
|
||||
(self.status_code(), Json(self.to_error_response())).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a text-pipeline submit failure: tokenized-prompt validation
|
||||
/// failures (the prompt is too long for the model, or empty after
|
||||
/// tokenization) are the client's fault and map to HTTP 400, mirroring the
|
||||
/// Python frontend. Everything else stays an internal 500.
|
||||
pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
|
||||
if is_prompt_validation_error(&error) {
|
||||
return invalid_request!("{error}");
|
||||
}
|
||||
server_error!("{}: {}", context, error.to_report_string())
|
||||
}
|
||||
|
||||
/// Like [`text_submit_error`], for the chat pipeline (which both wraps the
|
||||
/// text errors and raises its own prompt-length variant).
|
||||
pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
|
||||
match &error {
|
||||
vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
|
||||
vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => {
|
||||
invalid_request!("{error}")
|
||||
}
|
||||
_ => server_error!("{}: {}", context, error.to_report_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_prompt_validation_error(error: &vllm_text::Error) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
vllm_text::Error::PromptTooLong { .. }
|
||||
| vllm_text::Error::EmptyPromptTokenIds { .. }
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn prompt_too_long_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::PromptTooLong {
|
||||
max_model_len: 8192,
|
||||
prompt_len: 9000,
|
||||
};
|
||||
let api_error = text_submit_error("failed to submit completion request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert!(response.error.message.contains("8192"));
|
||||
assert!(response.error.message.contains("9000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_wrapped_prompt_too_long_maps_to_invalid_request() {
|
||||
let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong {
|
||||
max_model_len: 8192,
|
||||
prompt_len: 9000,
|
||||
});
|
||||
let api_error = chat_submit_error("failed to submit chat request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn llm_wrapped_empty_prompt_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds {
|
||||
request_id: "req-1".to_string(),
|
||||
});
|
||||
let api_error = text_submit_error("failed to submit completion request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_submit_errors_stay_internal() {
|
||||
let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
|
||||
let api_error = text_submit_error("failed to submit completion request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let response = api_error.to_error_response();
|
||||
assert!(response.error.message.starts_with("failed to submit completion request:"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ use self::types::{
|
||||
GenerateResponseStreamChoice, GenerateStreamResponse,
|
||||
};
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
@@ -65,11 +65,8 @@ pub async fn generate(
|
||||
{
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
return server_error!(
|
||||
"failed to submit raw generate request: {}",
|
||||
error.to_report_string()
|
||||
)
|
||||
.into_response();
|
||||
return text_submit_error("failed to submit raw generate request", error)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ use vllm_engine_core_client::protocol::StopReason;
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_chat_request};
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error};
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
|
||||
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
|
||||
@@ -77,11 +77,7 @@ pub async fn chat_completions(
|
||||
match state.chat.chat(prepared.chat_request).instrument(request_span.clone()).await {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
return server_error!(
|
||||
"failed to submit chat request: {}",
|
||||
error.to_report_string()
|
||||
)
|
||||
.into_response();
|
||||
return chat_submit_error("failed to submit chat request", error).into_response();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ use super::utils::logprobs::{
|
||||
};
|
||||
use super::utils::types::Usage;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::routes::openai::completions::types::{
|
||||
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
|
||||
CompletionStreamChoice, CompletionStreamResponse,
|
||||
@@ -75,11 +75,7 @@ pub async fn completions(
|
||||
{
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
return server_error!(
|
||||
"failed to submit completion request: {}",
|
||||
error.to_report_string()
|
||||
)
|
||||
.into_response();
|
||||
return text_submit_error("failed to submit completion request", error).into_response();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user