From d350fa8dddc6ed1a3a5710473d485ad0d02a6127 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:41:33 +0800 Subject: [PATCH] [Bugfix][Rust Frontend] Reject min_tokens above max_tokens (#46733) Co-authored-by: Bugen Zhao Signed-off-by: reidliu41 Signed-off-by: Bugen Zhao --- rust/src/chat/src/error.rs | 11 ++++++ rust/src/server/src/error.rs | 41 +++++++++++---------- rust/src/server/src/grpc/mod.rs | 20 +++++++---- rust/src/server/src/grpc/tests.rs | 59 +++++++++++++++++++++++++++++++ rust/src/text/src/error.rs | 23 ++++++++++++ rust/src/text/src/lower.rs | 27 ++++++++++++++ 6 files changed, 153 insertions(+), 28 deletions(-) diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index c472a65601..da2396c219 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -74,6 +74,17 @@ pub enum Error { pub type Result = std::result::Result; +impl Error { + /// Whether this error represents invalid user request parameters. + pub fn is_request_validation_error(&self) -> bool { + match self { + Self::PromptTooLong { .. } => true, + Self::Text(error) => error.is_request_validation_error(), + _ => false, + } + } +} + /// Format the available-parser suffix used in user-facing error messages. fn available_parser_hint(available_names: &[String]) -> String { if available_names.is_empty() { diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index ede83748f3..3eba278267 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -78,7 +78,7 @@ impl IntoResponse for ApiError { /// the client's fault and map to HTTP 400, mirroring the Python frontend. /// Everything else stays an internal 500. pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError { - if is_request_validation_error(&error) { + if error.is_request_validation_error() { return invalid_request!("{error}"); } server_error!("{}: {}", context, error.to_report_string()) @@ -87,27 +87,10 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE /// Like [`text_submit_error`], for the chat pipeline (which both wraps the /// text errors and raises its own prompt-length variant). pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError { - match &error { - vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"), - vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => { - invalid_request!("{error}") - } - _ => server_error!("{}: {}", context, error.to_report_string()), + if error.is_request_validation_error() { + return invalid_request!("{error}"); } -} - -fn is_request_validation_error(error: &vllm_text::Error) -> bool { - matches!( - error, - vllm_text::Error::PromptTooLong { .. } - | vllm_text::Error::EmptyPromptTokenIds { .. } - | vllm_text::Error::Logprobs(_) - | vllm_text::Error::TokenIds(_) - | vllm_text::Error::InvalidThinkingTokenBudget - // An empty tokenized prompt detected later, at request prepare - // time, surfaces through the transparent Llm wrapper. - | vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. }) - ) + server_error!("{}: {}", context, error.to_report_string()) } #[cfg(test)] @@ -140,6 +123,22 @@ mod tests { assert!(response.error.message.contains("thinking_token_budget")); } + #[test] + fn min_tokens_above_max_tokens_maps_to_invalid_request() { + let api_error = text_submit_error( + "failed to submit completion request", + vllm_text::Error::MinTokensExceedsMaxTokens { + min_tokens: 5, + max_tokens: 4, + }, + ); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + let response = api_error.to_error_response(); + assert_eq!(response.error.error_type, "invalid_request_error"); + assert!(response.error.message.contains("min_tokens=5")); + assert!(response.error.message.contains("max_tokens=4")); + } + #[test] fn chat_wrapped_prompt_too_long_maps_to_invalid_request() { let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong { diff --git a/rust/src/server/src/grpc/mod.rs b/rust/src/server/src/grpc/mod.rs index 62ee860766..1fcb8674fe 100644 --- a/rust/src/server/src/grpc/mod.rs +++ b/rust/src/server/src/grpc/mod.rs @@ -56,12 +56,9 @@ impl pb::generate_server::Generate for GenerateServiceImpl { info!(%request_id, "grpc generate (unary)"); let stream = self.state.chat.text().generate(text_request).await; - let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?; + let stream = stream.map_err(text_error_to_status)?; - let collected = stream - .collect_output() - .await - .map_err(|e| Status::internal(e.to_report_string()))?; + let collected = stream.collect_output().await.map_err(text_error_to_status)?; // Build the single aggregated response. let prompt_info = convert::to_prompt_info( @@ -104,7 +101,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl { info!(%request_id, "grpc generate (stream)"); let stream = self.state.chat.text().generate(text_request).await; - let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?; + let stream = stream.map_err(text_error_to_status)?; let (tx, rx) = mpsc::channel(32); @@ -112,7 +109,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl { futures::pin_mut!(stream); while let Some(event) = stream.next().await { let response = match event { - Err(e) => Err(Status::internal(e.to_report_string())), + Err(e) => Err(text_error_to_status(e)), Ok(DecodedTextEvent::Start { prompt_token_ids, prompt_logprobs, @@ -155,3 +152,12 @@ impl pb::generate_server::Generate for GenerateServiceImpl { Ok(Response::new(Box::pin(response_stream))) } } + +fn text_error_to_status(error: vllm_text::Error) -> Status { + let message = error.to_report_string(); + if error.is_request_validation_error() { + Status::invalid_argument(message) + } else { + Status::internal(message) + } +} diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 58bf894c92..14156a4104 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -425,6 +425,34 @@ async fn unary_generate_missing_prompt_returns_invalid_argument() { server_task.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() { + let (mut client, server_task, _engine_task) = + grpc_test_server(b"engine-grpc-min-above-max", default_stream_output_specs()).await; + + let status = client + .generate(pb::GenerateRequest { + request_id: "test-min-above-max".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())), + stopping: Some(pb::StoppingCriteria { + max_new_tokens: 4, + min_new_tokens: 5, + ..Default::default() + }), + ..Default::default() + }) + .await + .expect_err("should fail when min_new_tokens exceeds max_new_tokens"); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert!(status.message().contains("min_tokens=5")); + assert!(status.message().contains("max_tokens=4")); + + server_task.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn streaming_generate_yields_incremental_responses() { @@ -513,6 +541,37 @@ async fn streaming_generate_missing_prompt_returns_invalid_argument() { server_task.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn streaming_generate_min_tokens_above_max_tokens_returns_invalid_argument() { + let (mut client, server_task, _engine_task) = grpc_test_server( + b"engine-grpc-stream-min-above-max", + default_stream_output_specs(), + ) + .await; + + let status = client + .generate_stream(pb::GenerateRequest { + request_id: "test-stream-min-above-max".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())), + stopping: Some(pb::StoppingCriteria { + max_new_tokens: 4, + min_new_tokens: 5, + ..Default::default() + }), + ..Default::default() + }) + .await + .expect_err("should fail when min_new_tokens exceeds max_new_tokens"); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert!(status.message().contains("min_tokens=5")); + assert!(status.message().contains("max_tokens=4")); + + server_task.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn unary_generate_with_sampling_params() { diff --git a/rust/src/text/src/error.rs b/rust/src/text/src/error.rs index 2c9e69ca15..96ddced584 100644 --- a/rust/src/text/src/error.rs +++ b/rust/src/text/src/error.rs @@ -20,6 +20,11 @@ pub enum Error { Logprobs(#[from] LogprobsError), #[error(transparent)] TokenIds(#[from] TokenIdsError), + #[error( + "`min_tokens` must be less than or equal to `max_tokens`, \ + got min_tokens={min_tokens}, max_tokens={max_tokens}" + )] + MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 }, #[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")] InvalidThinkingTokenBudget, #[error("text request stream `{request_id}` closed before terminal output")] @@ -32,6 +37,24 @@ pub enum Error { pub type Result = std::result::Result; +impl Error { + /// Whether this error represents invalid user request parameters. + pub fn is_request_validation_error(&self) -> bool { + match self { + Self::PromptTooLong { .. } + | Self::EmptyPromptTokenIds { .. } + | Self::Logprobs(_) + | Self::TokenIds(_) + | Self::MinTokensExceedsMaxTokens { .. } + | Self::InvalidThinkingTokenBudget + // An empty tokenized prompt detected later, at request prepare + // time, surfaces through the transparent Llm wrapper. + | Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true, + _ => false, + } + } +} + impl From for Error { fn from(error: vllm_tokenizer::TokenizerError) -> Self { Self::Tokenizer(error.0) diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 38528114e5..6cd18195bb 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -128,6 +128,12 @@ pub fn lower_sampling_params( prompt_len, )?; let min_tokens = min_tokens.unwrap_or(0); + if min_tokens > max_tokens { + return Err(Error::MinTokensExceedsMaxTokens { + min_tokens, + max_tokens, + }); + } let thinking_token_budget = normalize_thinking_token_budget(thinking_token_budget)?; let frequency_penalty = frequency_penalty.unwrap_or(0.0); let presence_penalty = presence_penalty.unwrap_or(0.0); @@ -414,6 +420,27 @@ mod tests { )); } + #[test] + fn lower_sampling_params_rejects_min_tokens_above_resolved_max_tokens() { + let error = lower_sampling_params_with_limits( + SamplingParams { + max_tokens: Some(4), + min_tokens: Some(5), + ..SamplingParams::default() + }, + sample_sampling_limits(), + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::MinTokensExceedsMaxTokens { + min_tokens: 5, + max_tokens: 4, + } + )); + } + #[test] fn lower_text_request_applies_python_style_eos_hints() { let prepared = lower_text_request(