[Bugfix][Rust Frontend] Skip undefined token ids in decode and anchor them zero-width (#55240)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Feathbow <feathbow@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
Codex
parent
474839f846
commit
c7e9816c6a
@@ -54,11 +54,10 @@ fn decode_fastokens_byte_level(
|
||||
let tokens: Vec<&str> = token_ids
|
||||
.iter()
|
||||
.filter(|&&id| !(skip_special_tokens && t.is_special_token(id)))
|
||||
.map(|&id| {
|
||||
t.id_to_token(id)
|
||||
.ok_or_else(|| tokenizer_error!("decoding failed: unknown token ID: {id}"))
|
||||
})
|
||||
.collect::<Result<_>>()?;
|
||||
// Match HF and fastokens: model vocabularies may contain undefined
|
||||
// tokenizer IDs, which contribute no decoded text.
|
||||
.filter_map(|&id| t.id_to_token(id))
|
||||
.collect();
|
||||
Ok(decode_byte_level(tokens))
|
||||
}
|
||||
|
||||
@@ -337,6 +336,7 @@ mod tests {
|
||||
use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
|
||||
|
||||
use super::{HuggingFaceTokenizer, Tokenizer};
|
||||
use crate::{TokenAnchor, TokenAttribution};
|
||||
|
||||
const REGULAR_TOKEN: &str = "<|regular|>";
|
||||
const SPECIAL_TOKEN: &str = "<|special|>";
|
||||
@@ -717,10 +717,50 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fast_byte_level_errors_on_unknown_id() {
|
||||
fn fast_byte_level_skips_undefined_ids() {
|
||||
let t = tiny_byte_level_bpe();
|
||||
let err = super::decode_fastokens_byte_level(&t, &[999], false)
|
||||
.expect_err("unknown id must error");
|
||||
assert!(format!("{err:?}").contains("999"));
|
||||
assert_eq!(
|
||||
super::decode_fastokens_byte_level(&t, &[1, 2, 999, 3, 3, 4], false)
|
||||
.expect("with the id"),
|
||||
super::decode_fastokens_byte_level(&t, &[1, 2, 3, 3, 4], false)
|
||||
.expect("without the id")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_stream_anchors_undefined_ids_zero_width() {
|
||||
let wrapper = HuggingFaceTokenizer::from_fastokens_backend(tiny_byte_level_bpe());
|
||||
assert!(matches!(
|
||||
wrapper.backend,
|
||||
super::Backend::FastokensByteLevel(_)
|
||||
));
|
||||
|
||||
let mut stream = wrapper.create_decode_stream(&[], false, 0);
|
||||
for id in [999, 1, 999, 2] {
|
||||
stream.push_token(id).expect("push token");
|
||||
}
|
||||
let (_, full) = stream.flush(None).expect("flush");
|
||||
assert_eq!(full.text, "He");
|
||||
assert_eq!(
|
||||
full.attributions.as_slice(),
|
||||
[
|
||||
TokenAttribution {
|
||||
token_id: 999,
|
||||
anchor: TokenAnchor::ZeroWidth { byte_offset: 0 }
|
||||
},
|
||||
TokenAttribution {
|
||||
token_id: 1,
|
||||
anchor: TokenAnchor::Visible { byte_offset: 0 }
|
||||
},
|
||||
TokenAttribution {
|
||||
token_id: 999,
|
||||
anchor: TokenAnchor::ZeroWidth { byte_offset: 1 }
|
||||
},
|
||||
TokenAttribution {
|
||||
token_id: 2,
|
||||
anchor: TokenAnchor::Visible { byte_offset: 1 }
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,21 +72,11 @@ const SAFE_SUFFIX_MIN: usize = 4;
|
||||
const SAFE_SUFFIX_MAX: usize = 6;
|
||||
|
||||
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
|
||||
fn record_token(&mut self, token_id: u32) {
|
||||
if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) {
|
||||
self.decoded.record_zero_width_token(token_id);
|
||||
} else {
|
||||
self.decoded.record_pending_token(token_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode prompt-only context for prefix seeding.
|
||||
///
|
||||
/// Prompt ids may come from the model vocabulary rather than the local
|
||||
/// tokenizer vocabulary. For this seeding path, ids that cannot be mapped
|
||||
/// back to raw token text are dropped before retrying strict decode. This
|
||||
/// tolerance is intentionally limited to prompt context; generated ids are
|
||||
/// decoded later through the normal strict path.
|
||||
/// back to raw token text are dropped before retrying strict decode.
|
||||
fn decode_prompt_context(&self, ids: &[u32]) -> Result<(String, Vec<u32>)> {
|
||||
match self.tokenizer.decode(ids, self.skip_special_tokens) {
|
||||
Ok(decoded) => Ok((decoded, ids.to_vec())),
|
||||
@@ -144,10 +134,22 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
|
||||
}
|
||||
|
||||
self.ids.push(token_id);
|
||||
self.record_token(token_id);
|
||||
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
let prefix_len = self.prefix.len();
|
||||
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
|
||||
let produced_text = string.len() > prefix_len && !string.ends_with('\u{FFFD}');
|
||||
|
||||
// Skipped undefined IDs leave the decoded text unchanged. Check only
|
||||
// non-emitting steps to avoid id_to_token's lookup and String allocation
|
||||
// on the normal path.
|
||||
let zero_width = (self.skip_special_tokens && self.tokenizer.is_special_id(token_id))
|
||||
|| (!produced_text && self.tokenizer.id_to_token(token_id).is_none());
|
||||
if zero_width {
|
||||
self.decoded.record_zero_width_token(token_id);
|
||||
} else {
|
||||
self.decoded.record_pending_token(token_id);
|
||||
}
|
||||
|
||||
if !produced_text {
|
||||
return Ok(0);
|
||||
}
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
@@ -224,8 +226,8 @@ mod tests {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn id_to_token(&self, _id: u32) -> Option<String> {
|
||||
unreachable!()
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
Some(id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,8 +458,8 @@ mod tests {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn id_to_token(&self, _id: u32) -> Option<String> {
|
||||
unreachable!()
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
Some(id.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user