[Bugfix][Rust Frontend] Detokenizer: avoid leaking prompt on zero-generated-token completions (#47707)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: xiaguan <751080330@qq.com>
This commit is contained in:
JinYan Su
2026-07-16 09:11:21 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 85e296950c
commit cc706b05a5
+34 -5
View File
@@ -158,17 +158,19 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
}
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)> {
if !self.ids.is_empty() {
// If the prefix was never seeded (no push_token was called), `ids`
// holds only prompt context — decoding it would re-emit prompt text.
if self.prefix_seeded && !self.ids.is_empty() {
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
self.ids.clear();
self.prefix.clear();
self.prefix_index = 0;
self.prefix_seeded = true;
// Ensure we split at a utf-8 char boundary.
self.cumulative_output
.push_str(&string[string.floor_char_boundary(prefix_len)..]);
}
self.ids.clear();
self.prefix.clear();
self.prefix_index = 0;
self.prefix_seeded = true;
if let Some(truncate_output_to) = truncate_output_to {
self.cumulative_output.truncate(truncate_output_to);
}
@@ -489,4 +491,31 @@ mod tests {
assert_eq!(full_text, "你好A");
assert_eq!(out, "你好A");
}
#[test]
fn flush_without_push_token_does_not_leak_prompt() {
let backend = Utf8Backend;
let prompt: Vec<u32> = b"The quick brown fox jumps over the lazy dog. "
.iter()
.cycle()
.take(7001)
.map(|&b| b as u32)
.collect();
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "");
}
#[test]
fn flush_without_push_token_does_not_leak_undecodable_prompt_tail() {
let backend = Utf8Backend;
let prompt = vec![0xe4, 0xbd];
let mut decoder = backend.create_decode_stream(&prompt, false, 0);
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "");
}
}