From 1ae1051b4bf6e7e98d61b15527040f63eda73a0b Mon Sep 17 00:00:00 2001 From: JinYan Su Date: Fri, 12 Jun 2026 15:53:11 +0800 Subject: [PATCH] [Bugfix][Rust Frontend] Return 400 for prompt-validation submit errors (#45286) Signed-off-by: xiaguan <751080330@qq.com> Co-authored-by: Claude Fable 5 --- rust/src/server/src/error.rs | 82 +++++++++++++++++++ .../server/src/routes/inference/generate.rs | 9 +- .../src/routes/openai/chat_completions.rs | 8 +- .../server/src/routes/openai/completions.rs | 8 +- 4 files changed, 89 insertions(+), 18 deletions(-) diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index cc425ca076..ce716bb65f 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -1,6 +1,7 @@ use axum::Json; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; +use thiserror_ext::AsReport as _; use thiserror_ext::{Construct, Macro}; use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse}; @@ -72,3 +73,84 @@ impl IntoResponse for ApiError { (self.status_code(), Json(self.to_error_response())).into_response() } } + +/// Classify a text-pipeline submit failure: tokenized-prompt validation +/// failures (the prompt is too long for the model, or empty after +/// tokenization) are the client's fault and map to HTTP 400, mirroring the +/// Python frontend. Everything else stays an internal 500. +pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError { + if is_prompt_validation_error(&error) { + return invalid_request!("{error}"); + } + server_error!("{}: {}", context, error.to_report_string()) +} + +/// Like [`text_submit_error`], for the chat pipeline (which both wraps the +/// text errors and raises its own prompt-length variant). +pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError { + match &error { + vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"), + vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => { + invalid_request!("{error}") + } + _ => server_error!("{}: {}", context, error.to_report_string()), + } +} + +fn is_prompt_validation_error(error: &vllm_text::Error) -> bool { + matches!( + error, + vllm_text::Error::PromptTooLong { .. } + | vllm_text::Error::EmptyPromptTokenIds { .. } + // An empty tokenized prompt detected later, at request prepare + // time, surfaces through the transparent Llm wrapper. + | vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. }) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prompt_too_long_maps_to_invalid_request() { + let error = vllm_text::Error::PromptTooLong { + max_model_len: 8192, + prompt_len: 9000, + }; + let api_error = text_submit_error("failed to submit completion request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + let response = api_error.to_error_response(); + assert_eq!(response.error.error_type, "invalid_request_error"); + assert!(response.error.message.contains("8192")); + assert!(response.error.message.contains("9000")); + } + + #[test] + fn chat_wrapped_prompt_too_long_maps_to_invalid_request() { + let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong { + max_model_len: 8192, + prompt_len: 9000, + }); + let api_error = chat_submit_error("failed to submit chat request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + } + + #[test] + fn llm_wrapped_empty_prompt_maps_to_invalid_request() { + let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { + request_id: "req-1".to_string(), + }); + let api_error = text_submit_error("failed to submit completion request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + } + + #[test] + fn other_submit_errors_stay_internal() { + let error = vllm_text::Error::Tokenizer("backend exploded".to_string()); + let api_error = text_submit_error("failed to submit completion request", error); + assert_eq!(api_error.status_code(), StatusCode::INTERNAL_SERVER_ERROR); + let response = api_error.to_error_response(); + assert!(response.error.message.starts_with("failed to submit completion request:")); + } +} diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index ffbf28048d..c11e4c79ca 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -28,7 +28,7 @@ use self::types::{ GenerateResponseStreamChoice, GenerateStreamResponse, }; use crate::config::ApiServerOptions; -use crate::error::{ApiError, bail_server_error, server_error}; +use crate::error::{ApiError, bail_server_error, server_error, text_submit_error}; use crate::routes::openai::utils::logprobs::clamp_logprob; use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage}; use crate::routes::openai::utils::validated_json::ValidatedJson; @@ -65,11 +65,8 @@ pub async fn generate( { Ok(stream) => stream, Err(error) => { - return server_error!( - "failed to submit raw generate request: {}", - error.to_report_string() - ) - .into_response(); + return text_submit_error("failed to submit raw generate request", error) + .into_response(); } }; diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 6274a4e98a..a8c70d273d 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -25,7 +25,7 @@ use vllm_engine_core_client::protocol::StopReason; use self::convert::{ResponseOptions, prepare_chat_request}; use crate::config::ApiServerOptions; -use crate::error::{ApiError, bail_server_error, server_error}; +use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error}; use crate::routes::openai::chat_completions::types::{ AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse, @@ -77,11 +77,7 @@ pub async fn chat_completions( match state.chat.chat(prepared.chat_request).instrument(request_span.clone()).await { Ok(stream) => stream, Err(error) => { - return server_error!( - "failed to submit chat request: {}", - error.to_report_string() - ) - .into_response(); + return chat_submit_error("failed to submit chat request", error).into_response(); } }; diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index 9dc2e19154..3dc3bbff6f 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -25,7 +25,7 @@ use super::utils::logprobs::{ }; use super::utils::types::Usage; use crate::config::ApiServerOptions; -use crate::error::{ApiError, bail_server_error, server_error}; +use crate::error::{ApiError, bail_server_error, server_error, text_submit_error}; use crate::routes::openai::completions::types::{ CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk, CompletionStreamChoice, CompletionStreamResponse, @@ -75,11 +75,7 @@ pub async fn completions( { Ok(stream) => stream, Err(error) => { - return server_error!( - "failed to submit completion request: {}", - error.to_report_string() - ) - .into_response(); + return text_submit_error("failed to submit completion request", error).into_response(); } };