[Bugfix][Rust Frontend] Skip undefined token ids in decode and anchor them zero-width (#55240)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Feathbow <feathbow@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Salt Sato
2026-09-09 09:55:04 +00:00
committed by GitHub
co-authored by Bugen Zhao Codex
parent 474839f846
commit c7e9816c6a
2 changed files with 68 additions and 26 deletions
+49 -9
View File
@@ -54,11 +54,10 @@ fn decode_fastokens_byte_level(
let tokens: Vec<&str> = token_ids
.iter()
.filter(|&&id| !(skip_special_tokens && t.is_special_token(id)))
.map(|&id| {
t.id_to_token(id)
.ok_or_else(|| tokenizer_error!("decoding failed: unknown token ID: {id}"))
})
.collect::<Result<_>>()?;
// Match HF and fastokens: model vocabularies may contain undefined
// tokenizer IDs, which contribute no decoded text.
.filter_map(|&id| t.id_to_token(id))
.collect();
Ok(decode_byte_level(tokens))
}
@@ -337,6 +336,7 @@ mod tests {
use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
use super::{HuggingFaceTokenizer, Tokenizer};
use crate::{TokenAnchor, TokenAttribution};
const REGULAR_TOKEN: &str = "<|regular|>";
const SPECIAL_TOKEN: &str = "<|special|>";
@@ -717,10 +717,50 @@ mod tests {
}
#[test]
fn fast_byte_level_errors_on_unknown_id() {
fn fast_byte_level_skips_undefined_ids() {
let t = tiny_byte_level_bpe();
let err = super::decode_fastokens_byte_level(&t, &[999], false)
.expect_err("unknown id must error");
assert!(format!("{err:?}").contains("999"));
assert_eq!(
super::decode_fastokens_byte_level(&t, &[1, 2, 999, 3, 3, 4], false)
.expect("with the id"),
super::decode_fastokens_byte_level(&t, &[1, 2, 3, 3, 4], false)
.expect("without the id")
);
}
#[test]
fn decode_stream_anchors_undefined_ids_zero_width() {
let wrapper = HuggingFaceTokenizer::from_fastokens_backend(tiny_byte_level_bpe());
assert!(matches!(
wrapper.backend,
super::Backend::FastokensByteLevel(_)
));
let mut stream = wrapper.create_decode_stream(&[], false, 0);
for id in [999, 1, 999, 2] {
stream.push_token(id).expect("push token");
}
let (_, full) = stream.flush(None).expect("flush");
assert_eq!(full.text, "He");
assert_eq!(
full.attributions.as_slice(),
[
TokenAttribution {
token_id: 999,
anchor: TokenAnchor::ZeroWidth { byte_offset: 0 }
},
TokenAttribution {
token_id: 1,
anchor: TokenAnchor::Visible { byte_offset: 0 }
},
TokenAttribution {
token_id: 999,
anchor: TokenAnchor::ZeroWidth { byte_offset: 1 }
},
TokenAttribution {
token_id: 2,
anchor: TokenAnchor::Visible { byte_offset: 1 }
},
]
);
}
}
+19 -17
View File
@@ -72,21 +72,11 @@ const SAFE_SUFFIX_MIN: usize = 4;
const SAFE_SUFFIX_MAX: usize = 6;
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
fn record_token(&mut self, token_id: u32) {
if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) {
self.decoded.record_zero_width_token(token_id);
} else {
self.decoded.record_pending_token(token_id);
}
}
/// Decode prompt-only context for prefix seeding.
///
/// Prompt ids may come from the model vocabulary rather than the local
/// tokenizer vocabulary. For this seeding path, ids that cannot be mapped
/// back to raw token text are dropped before retrying strict decode. This
/// tolerance is intentionally limited to prompt context; generated ids are
/// decoded later through the normal strict path.
/// back to raw token text are dropped before retrying strict decode.
fn decode_prompt_context(&self, ids: &[u32]) -> Result<(String, Vec<u32>)> {
match self.tokenizer.decode(ids, self.skip_special_tokens) {
Ok(decoded) => Ok((decoded, ids.to_vec())),
@@ -144,10 +134,22 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
}
self.ids.push(token_id);
self.record_token(token_id);
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
let produced_text = string.len() > prefix_len && !string.ends_with('\u{FFFD}');
// Skipped undefined IDs leave the decoded text unchanged. Check only
// non-emitting steps to avoid id_to_token's lookup and String allocation
// on the normal path.
let zero_width = (self.skip_special_tokens && self.tokenizer.is_special_id(token_id))
|| (!produced_text && self.tokenizer.id_to_token(token_id).is_none());
if zero_width {
self.decoded.record_zero_width_token(token_id);
} else {
self.decoded.record_pending_token(token_id);
}
if !produced_text {
return Ok(0);
}
// Ensure we split at a utf-8 char boundary.
@@ -224,8 +226,8 @@ mod tests {
unreachable!()
}
fn id_to_token(&self, _id: u32) -> Option<String> {
unreachable!()
fn id_to_token(&self, id: u32) -> Option<String> {
Some(id.to_string())
}
}
@@ -456,8 +458,8 @@ mod tests {
unreachable!()
}
fn id_to_token(&self, _id: u32) -> Option<String> {
unreachable!()
fn id_to_token(&self, id: u32) -> Option<String> {
Some(id.to_string())
}
}