[Bugfix][Frontend] Reject empty gRPC stop strings (#50746)
Signed-off-by: zq <zhouquan1511@163.com> Co-authored-by: zq <zhouquan1511@163.com>
This commit is contained in:
@@ -650,6 +650,32 @@ async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
|
|||||||
server_task.abort();
|
server_task.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn unary_generate_empty_stop_string_returns_invalid_argument() {
|
||||||
|
let (mut client, server_task, _engine_task) =
|
||||||
|
grpc_test_server(b"engine-grpc-empty-stop", default_stream_output_specs()).await;
|
||||||
|
|
||||||
|
let status = client
|
||||||
|
.generate(pb::GenerateRequest {
|
||||||
|
request_id: "test-empty-stop".to_string(),
|
||||||
|
model: "test-model".to_string(),
|
||||||
|
prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
|
||||||
|
stopping: Some(pb::StoppingCriteria {
|
||||||
|
stop_strings: vec!["".to_string()],
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect_err("should fail when stop_strings contains an empty string");
|
||||||
|
|
||||||
|
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||||
|
assert!(status.message().contains("stop strings cannot be empty"));
|
||||||
|
|
||||||
|
server_task.abort();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn unary_generate_invalid_sampling_params_returns_invalid_argument() {
|
async fn unary_generate_invalid_sampling_params_returns_invalid_argument() {
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ pub enum Error {
|
|||||||
Tokenizer(String),
|
Tokenizer(String),
|
||||||
#[error("text request `{request_id}` must contain at least one prompt token ID")]
|
#[error("text request `{request_id}` must contain at least one prompt token ID")]
|
||||||
EmptyPromptTokenIds { request_id: String },
|
EmptyPromptTokenIds { request_id: String },
|
||||||
|
#[error("text request `{request_id}` stop strings cannot be empty")]
|
||||||
|
EmptyStopString { request_id: String },
|
||||||
#[error(
|
#[error(
|
||||||
"this model's maximum context length is {max_model_len} tokens, \
|
"this model's maximum context length is {max_model_len} tokens, \
|
||||||
but the prompt contains {prompt_len} input tokens"
|
but the prompt contains {prompt_len} input tokens"
|
||||||
@@ -51,6 +53,7 @@ impl Error {
|
|||||||
match self {
|
match self {
|
||||||
Self::PromptTooLong { .. }
|
Self::PromptTooLong { .. }
|
||||||
| Self::EmptyPromptTokenIds { .. }
|
| Self::EmptyPromptTokenIds { .. }
|
||||||
|
| Self::EmptyStopString { .. }
|
||||||
| Self::Logprobs(_)
|
| Self::Logprobs(_)
|
||||||
| Self::TokenIds(_)
|
| Self::TokenIds(_)
|
||||||
| Self::SamplingParams(_)
|
| Self::SamplingParams(_)
|
||||||
|
|||||||
@@ -126,14 +126,12 @@ pub async fn decoded_text_event_stream(
|
|||||||
// when streaming the outputs.
|
// when streaming the outputs.
|
||||||
match decode_options.include_stop_str_in_output {
|
match decode_options.include_stop_str_in_output {
|
||||||
true => 0,
|
true => 0,
|
||||||
false => {
|
false => decode_options
|
||||||
decode_options
|
.stop_strings
|
||||||
.stop_strings
|
.as_ref()
|
||||||
.as_ref()
|
.and_then(|stops| stops.iter().map(|ss| ss.len()).max())
|
||||||
.and_then(|stops| stops.iter().map(|ss| ss.len()).max())
|
.unwrap_or(1)
|
||||||
.unwrap_or(1)
|
.saturating_sub(1),
|
||||||
- 1
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
decoder = Some(dec);
|
decoder = Some(dec);
|
||||||
@@ -321,6 +319,9 @@ fn matches_stop_string(stops: &[String], output: &str, new_bytes: usize) -> Opti
|
|||||||
.map(|ss| (ss.as_bytes(), ss.len(), next_off.saturating_sub(ss.len())))
|
.map(|ss| (ss.as_bytes(), ss.len(), next_off.saturating_sub(ss.len())))
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.filter_map(|(ss_idx, (ss, len, start_off))| {
|
.filter_map(|(ss_idx, (ss, len, start_off))| {
|
||||||
|
if len == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
output[start_off..]
|
output[start_off..]
|
||||||
.windows(len)
|
.windows(len)
|
||||||
.position(|w| w == ss)
|
.position(|w| w == ss)
|
||||||
@@ -459,6 +460,13 @@ mod tests {
|
|||||||
assert_eq!(output.finish_reason, FinishReason::Length);
|
assert_eq!(output.finish_reason, FinishReason::Length);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stream_empty_stop_string_does_not_underflow_buffer() {
|
||||||
|
let output = run_to_completion(ascii_tokens("hello"), opts(&[""], 0)).await;
|
||||||
|
assert_eq!(output.text, "hello");
|
||||||
|
assert_eq!(output.finish_reason, FinishReason::Length);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn stream_stop_string_multi_char() {
|
async fn stream_stop_string_multi_char() {
|
||||||
let output = run_to_completion(ascii_tokens("say hello world"), opts(&["lo"], 0)).await;
|
let output = run_to_completion(ascii_tokens("say hello world"), opts(&["lo"], 0)).await;
|
||||||
@@ -636,6 +644,13 @@ mod tests {
|
|||||||
assert_eq!(result, None);
|
assert_eq!(result, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_stop_string_is_ignored_by_matcher() {
|
||||||
|
let stops = vec!["".to_string(), "world".to_string()];
|
||||||
|
let result = matches_stop_string(&stops, "hello", 1);
|
||||||
|
assert_eq!(result, None);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn stop_string_multibyte_utf8() {
|
fn stop_string_multibyte_utf8() {
|
||||||
let stops = vec!["世界".to_string()];
|
let stops = vec!["世界".to_string()];
|
||||||
|
|||||||
@@ -225,6 +225,34 @@ impl TextRequest {
|
|||||||
request_id: self.request_id.clone(),
|
request_id: self.request_id.clone(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
if self
|
||||||
|
.decode_options
|
||||||
|
.stop_strings
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|stops| stops.iter().any(String::is_empty))
|
||||||
|
{
|
||||||
|
return Err(Error::EmptyStopString {
|
||||||
|
request_id: self.request_id.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_rejects_empty_stop_string_at_shared_chokepoint() {
|
||||||
|
let mut request = TextRequest::for_test();
|
||||||
|
request.request_id = "req-empty-stop".to_string();
|
||||||
|
request.decode_options.stop_strings = Some(vec!["valid".to_string(), String::new()]);
|
||||||
|
|
||||||
|
let err = request.validate().expect_err("empty stop strings should be rejected");
|
||||||
|
|
||||||
|
assert!(err.is_request_validation_error());
|
||||||
|
assert!(err.to_string().contains("stop strings cannot be empty"));
|
||||||
|
assert!(err.to_string().contains("req-empty-stop"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user