[Bugfix][Rust Frontend] Reject min_tokens above max_tokens (#46733)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -74,6 +74,17 @@ pub enum Error {
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl Error {
|
||||
/// Whether this error represents invalid user request parameters.
|
||||
pub fn is_request_validation_error(&self) -> bool {
|
||||
match self {
|
||||
Self::PromptTooLong { .. } => true,
|
||||
Self::Text(error) => error.is_request_validation_error(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Format the available-parser suffix used in user-facing error messages.
|
||||
fn available_parser_hint(available_names: &[String]) -> String {
|
||||
if available_names.is_empty() {
|
||||
|
||||
@@ -78,7 +78,7 @@ impl IntoResponse for ApiError {
|
||||
/// the client's fault and map to HTTP 400, mirroring the Python frontend.
|
||||
/// Everything else stays an internal 500.
|
||||
pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
|
||||
if is_request_validation_error(&error) {
|
||||
if error.is_request_validation_error() {
|
||||
return invalid_request!("{error}");
|
||||
}
|
||||
server_error!("{}: {}", context, error.to_report_string())
|
||||
@@ -87,27 +87,10 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE
|
||||
/// Like [`text_submit_error`], for the chat pipeline (which both wraps the
|
||||
/// text errors and raises its own prompt-length variant).
|
||||
pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
|
||||
match &error {
|
||||
vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
|
||||
vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => {
|
||||
invalid_request!("{error}")
|
||||
}
|
||||
_ => server_error!("{}: {}", context, error.to_report_string()),
|
||||
if error.is_request_validation_error() {
|
||||
return invalid_request!("{error}");
|
||||
}
|
||||
}
|
||||
|
||||
fn is_request_validation_error(error: &vllm_text::Error) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
vllm_text::Error::PromptTooLong { .. }
|
||||
| vllm_text::Error::EmptyPromptTokenIds { .. }
|
||||
| vllm_text::Error::Logprobs(_)
|
||||
| vllm_text::Error::TokenIds(_)
|
||||
| vllm_text::Error::InvalidThinkingTokenBudget
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
|
||||
)
|
||||
server_error!("{}: {}", context, error.to_report_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -140,6 +123,22 @@ mod tests {
|
||||
assert!(response.error.message.contains("thinking_token_budget"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_tokens_above_max_tokens_maps_to_invalid_request() {
|
||||
let api_error = text_submit_error(
|
||||
"failed to submit completion request",
|
||||
vllm_text::Error::MinTokensExceedsMaxTokens {
|
||||
min_tokens: 5,
|
||||
max_tokens: 4,
|
||||
},
|
||||
);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert!(response.error.message.contains("min_tokens=5"));
|
||||
assert!(response.error.message.contains("max_tokens=4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_wrapped_prompt_too_long_maps_to_invalid_request() {
|
||||
let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong {
|
||||
|
||||
@@ -56,12 +56,9 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
|
||||
info!(%request_id, "grpc generate (unary)");
|
||||
|
||||
let stream = self.state.chat.text().generate(text_request).await;
|
||||
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
|
||||
let stream = stream.map_err(text_error_to_status)?;
|
||||
|
||||
let collected = stream
|
||||
.collect_output()
|
||||
.await
|
||||
.map_err(|e| Status::internal(e.to_report_string()))?;
|
||||
let collected = stream.collect_output().await.map_err(text_error_to_status)?;
|
||||
|
||||
// Build the single aggregated response.
|
||||
let prompt_info = convert::to_prompt_info(
|
||||
@@ -104,7 +101,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
|
||||
info!(%request_id, "grpc generate (stream)");
|
||||
|
||||
let stream = self.state.chat.text().generate(text_request).await;
|
||||
let stream = stream.map_err(|e| Status::internal(e.to_report_string()))?;
|
||||
let stream = stream.map_err(text_error_to_status)?;
|
||||
|
||||
let (tx, rx) = mpsc::channel(32);
|
||||
|
||||
@@ -112,7 +109,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
|
||||
futures::pin_mut!(stream);
|
||||
while let Some(event) = stream.next().await {
|
||||
let response = match event {
|
||||
Err(e) => Err(Status::internal(e.to_report_string())),
|
||||
Err(e) => Err(text_error_to_status(e)),
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
@@ -155,3 +152,12 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
|
||||
Ok(Response::new(Box::pin(response_stream)))
|
||||
}
|
||||
}
|
||||
|
||||
fn text_error_to_status(error: vllm_text::Error) -> Status {
|
||||
let message = error.to_report_string();
|
||||
if error.is_request_validation_error() {
|
||||
Status::invalid_argument(message)
|
||||
} else {
|
||||
Status::internal(message)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -425,6 +425,34 @@ async fn unary_generate_missing_prompt_returns_invalid_argument() {
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
|
||||
let (mut client, server_task, _engine_task) =
|
||||
grpc_test_server(b"engine-grpc-min-above-max", default_stream_output_specs()).await;
|
||||
|
||||
let status = client
|
||||
.generate(pb::GenerateRequest {
|
||||
request_id: "test-min-above-max".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 4,
|
||||
min_new_tokens: 5,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("should fail when min_new_tokens exceeds max_new_tokens");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
assert!(status.message().contains("min_tokens=5"));
|
||||
assert!(status.message().contains("max_tokens=4"));
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_generate_yields_incremental_responses() {
|
||||
@@ -513,6 +541,37 @@ async fn streaming_generate_missing_prompt_returns_invalid_argument() {
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
|
||||
let (mut client, server_task, _engine_task) = grpc_test_server(
|
||||
b"engine-grpc-stream-min-above-max",
|
||||
default_stream_output_specs(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = client
|
||||
.generate_stream(pb::GenerateRequest {
|
||||
request_id: "test-stream-min-above-max".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 4,
|
||||
min_new_tokens: 5,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("should fail when min_new_tokens exceeds max_new_tokens");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
assert!(status.message().contains("min_tokens=5"));
|
||||
assert!(status.message().contains("max_tokens=4"));
|
||||
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_with_sampling_params() {
|
||||
|
||||
@@ -20,6 +20,11 @@ pub enum Error {
|
||||
Logprobs(#[from] LogprobsError),
|
||||
#[error(transparent)]
|
||||
TokenIds(#[from] TokenIdsError),
|
||||
#[error(
|
||||
"`min_tokens` must be less than or equal to `max_tokens`, \
|
||||
got min_tokens={min_tokens}, max_tokens={max_tokens}"
|
||||
)]
|
||||
MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 },
|
||||
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
|
||||
InvalidThinkingTokenBudget,
|
||||
#[error("text request stream `{request_id}` closed before terminal output")]
|
||||
@@ -32,6 +37,24 @@ pub enum Error {
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl Error {
|
||||
/// Whether this error represents invalid user request parameters.
|
||||
pub fn is_request_validation_error(&self) -> bool {
|
||||
match self {
|
||||
Self::PromptTooLong { .. }
|
||||
| Self::EmptyPromptTokenIds { .. }
|
||||
| Self::Logprobs(_)
|
||||
| Self::TokenIds(_)
|
||||
| Self::MinTokensExceedsMaxTokens { .. }
|
||||
| Self::InvalidThinkingTokenBudget
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
| Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<vllm_tokenizer::TokenizerError> for Error {
|
||||
fn from(error: vllm_tokenizer::TokenizerError) -> Self {
|
||||
Self::Tokenizer(error.0)
|
||||
|
||||
@@ -128,6 +128,12 @@ pub fn lower_sampling_params(
|
||||
prompt_len,
|
||||
)?;
|
||||
let min_tokens = min_tokens.unwrap_or(0);
|
||||
if min_tokens > max_tokens {
|
||||
return Err(Error::MinTokensExceedsMaxTokens {
|
||||
min_tokens,
|
||||
max_tokens,
|
||||
});
|
||||
}
|
||||
let thinking_token_budget = normalize_thinking_token_budget(thinking_token_budget)?;
|
||||
let frequency_penalty = frequency_penalty.unwrap_or(0.0);
|
||||
let presence_penalty = presence_penalty.unwrap_or(0.0);
|
||||
@@ -414,6 +420,27 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_min_tokens_above_resolved_max_tokens() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
max_tokens: Some(4),
|
||||
min_tokens: Some(5),
|
||||
..SamplingParams::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::MinTokensExceedsMaxTokens {
|
||||
min_tokens: 5,
|
||||
max_tokens: 4,
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_applies_python_style_eos_hints() {
|
||||
let prepared = lower_text_request(
|
||||
|
||||
Reference in New Issue
Block a user