[Bugfix][Rust Frontend] Reject min_tokens above max_tokens (#46733)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Reid
2026-06-26 03:41:33 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent dbc49b6b99
commit d350fa8ddd
6 changed files with 153 additions and 28 deletions
+20 -21
View File
@@ -78,7 +78,7 @@ impl IntoResponse for ApiError {
/// the client's fault and map to HTTP 400, mirroring the Python frontend.
/// Everything else stays an internal 500.
pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
if is_request_validation_error(&error) {
if error.is_request_validation_error() {
return invalid_request!("{error}");
}
server_error!("{}: {}", context, error.to_report_string())
@@ -87,27 +87,10 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE
/// Like [`text_submit_error`], for the chat pipeline (which both wraps the
/// text errors and raises its own prompt-length variant).
pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
match &error {
vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => {
invalid_request!("{error}")
}
_ => server_error!("{}: {}", context, error.to_report_string()),
if error.is_request_validation_error() {
return invalid_request!("{error}");
}
}
fn is_request_validation_error(error: &vllm_text::Error) -> bool {
matches!(
error,
vllm_text::Error::PromptTooLong { .. }
| vllm_text::Error::EmptyPromptTokenIds { .. }
| vllm_text::Error::Logprobs(_)
| vllm_text::Error::TokenIds(_)
| vllm_text::Error::InvalidThinkingTokenBudget
// An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper.
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
)
server_error!("{}: {}", context, error.to_report_string())
}
#[cfg(test)]
@@ -140,6 +123,22 @@ mod tests {
assert!(response.error.message.contains("thinking_token_budget"));
}
#[test]
fn min_tokens_above_max_tokens_maps_to_invalid_request() {
let api_error = text_submit_error(
"failed to submit completion request",
vllm_text::Error::MinTokensExceedsMaxTokens {
min_tokens: 5,
max_tokens: 4,
},
);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
let response = api_error.to_error_response();
assert_eq!(response.error.error_type, "invalid_request_error");
assert!(response.error.message.contains("min_tokens=5"));
assert!(response.error.message.contains("max_tokens=4"));
}
#[test]
fn chat_wrapped_prompt_too_long_maps_to_invalid_request() {
let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong {
+13 -7
View File
@@ -56,12 +56,9 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
info!(%request_id, "grpc generate (unary)");
let stream = self.state.chat.text().generate(text_request).await;
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
let stream = stream.map_err(text_error_to_status)?;
let collected = stream
.collect_output()
.await
.map_err(|e| Status::internal(e.to_report_string()))?;
let collected = stream.collect_output().await.map_err(text_error_to_status)?;
// Build the single aggregated response.
let prompt_info = convert::to_prompt_info(
@@ -104,7 +101,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
info!(%request_id, "grpc generate (stream)");
let stream = self.state.chat.text().generate(text_request).await;
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
let stream = stream.map_err(text_error_to_status)?;
let (tx, rx) = mpsc::channel(32);
@@ -112,7 +109,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
futures::pin_mut!(stream);
while let Some(event) = stream.next().await {
let response = match event {
Err(e) => Err(Status::internal(e.to_report_string())),
Err(e) => Err(text_error_to_status(e)),
Ok(DecodedTextEvent::Start {
prompt_token_ids,
prompt_logprobs,
@@ -155,3 +152,12 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
Ok(Response::new(Box::pin(response_stream)))
}
}
fn text_error_to_status(error: vllm_text::Error) -> Status {
let message = error.to_report_string();
if error.is_request_validation_error() {
Status::invalid_argument(message)
} else {
Status::internal(message)
}
}
+59
View File
@@ -425,6 +425,34 @@ async fn unary_generate_missing_prompt_returns_invalid_argument() {
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
let (mut client, server_task, _engine_task) =
grpc_test_server(b"engine-grpc-min-above-max", default_stream_output_specs()).await;
let status = client
.generate(pb::GenerateRequest {
request_id: "test-min-above-max".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 4,
min_new_tokens: 5,
..Default::default()
}),
..Default::default()
})
.await
.expect_err("should fail when min_new_tokens exceeds max_new_tokens");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert!(status.message().contains("min_tokens=5"));
assert!(status.message().contains("max_tokens=4"));
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_yields_incremental_responses() {
@@ -513,6 +541,37 @@ async fn streaming_generate_missing_prompt_returns_invalid_argument() {
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
let (mut client, server_task, _engine_task) = grpc_test_server(
b"engine-grpc-stream-min-above-max",
default_stream_output_specs(),
)
.await;
let status = client
.generate_stream(pb::GenerateRequest {
request_id: "test-stream-min-above-max".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 4,
min_new_tokens: 5,
..Default::default()
}),
..Default::default()
})
.await
.expect_err("should fail when min_new_tokens exceeds max_new_tokens");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
assert!(status.message().contains("min_tokens=5"));
assert!(status.message().contains("max_tokens=4"));
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_with_sampling_params() {