From c7e9816c6ab0731165a134fd0a9defed6ab1d748 Mon Sep 17 00:00:00 2001 From: Salt Sato Date: Wed, 9 Sep 2026 10:55:04 +0100 Subject: [PATCH] [Bugfix][Rust Frontend] Skip undefined token ids in decode and anchor them zero-width (#55240) Co-authored-by: Bugen Zhao Co-authored-by: Codex Signed-off-by: Feathbow Signed-off-by: Bugen Zhao --- rust/src/tokenizer/src/hf.rs | 58 ++++++++++++++++++++++----- rust/src/tokenizer/src/incremental.rs | 36 +++++++++-------- 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/rust/src/tokenizer/src/hf.rs b/rust/src/tokenizer/src/hf.rs index 0d7769ce66..44ddc6cb27 100644 --- a/rust/src/tokenizer/src/hf.rs +++ b/rust/src/tokenizer/src/hf.rs @@ -54,11 +54,10 @@ fn decode_fastokens_byte_level( let tokens: Vec<&str> = token_ids .iter() .filter(|&&id| !(skip_special_tokens && t.is_special_token(id))) - .map(|&id| { - t.id_to_token(id) - .ok_or_else(|| tokenizer_error!("decoding failed: unknown token ID: {id}")) - }) - .collect::>()?; + // Match HF and fastokens: model vocabularies may contain undefined + // tokenizer IDs, which contribute no decoded text. + .filter_map(|&id| t.id_to_token(id)) + .collect(); Ok(decode_byte_level(tokens)) } @@ -337,6 +336,7 @@ mod tests { use tokenizers::{AddedToken, Tokenizer as HfTokenizer}; use super::{HuggingFaceTokenizer, Tokenizer}; + use crate::{TokenAnchor, TokenAttribution}; const REGULAR_TOKEN: &str = "<|regular|>"; const SPECIAL_TOKEN: &str = "<|special|>"; @@ -717,10 +717,50 @@ mod tests { } #[test] - fn fast_byte_level_errors_on_unknown_id() { + fn fast_byte_level_skips_undefined_ids() { let t = tiny_byte_level_bpe(); - let err = super::decode_fastokens_byte_level(&t, &[999], false) - .expect_err("unknown id must error"); - assert!(format!("{err:?}").contains("999")); + assert_eq!( + super::decode_fastokens_byte_level(&t, &[1, 2, 999, 3, 3, 4], false) + .expect("with the id"), + super::decode_fastokens_byte_level(&t, &[1, 2, 3, 3, 4], false) + .expect("without the id") + ); + } + + #[test] + fn decode_stream_anchors_undefined_ids_zero_width() { + let wrapper = HuggingFaceTokenizer::from_fastokens_backend(tiny_byte_level_bpe()); + assert!(matches!( + wrapper.backend, + super::Backend::FastokensByteLevel(_) + )); + + let mut stream = wrapper.create_decode_stream(&[], false, 0); + for id in [999, 1, 999, 2] { + stream.push_token(id).expect("push token"); + } + let (_, full) = stream.flush(None).expect("flush"); + assert_eq!(full.text, "He"); + assert_eq!( + full.attributions.as_slice(), + [ + TokenAttribution { + token_id: 999, + anchor: TokenAnchor::ZeroWidth { byte_offset: 0 } + }, + TokenAttribution { + token_id: 1, + anchor: TokenAnchor::Visible { byte_offset: 0 } + }, + TokenAttribution { + token_id: 999, + anchor: TokenAnchor::ZeroWidth { byte_offset: 1 } + }, + TokenAttribution { + token_id: 2, + anchor: TokenAnchor::Visible { byte_offset: 1 } + }, + ] + ); } } diff --git a/rust/src/tokenizer/src/incremental.rs b/rust/src/tokenizer/src/incremental.rs index 3c9c164798..bdc50d175c 100644 --- a/rust/src/tokenizer/src/incremental.rs +++ b/rust/src/tokenizer/src/incremental.rs @@ -72,21 +72,11 @@ const SAFE_SUFFIX_MIN: usize = 4; const SAFE_SUFFIX_MAX: usize = 6; impl DecodeStream<'_, T> { - fn record_token(&mut self, token_id: u32) { - if self.skip_special_tokens && self.tokenizer.is_special_id(token_id) { - self.decoded.record_zero_width_token(token_id); - } else { - self.decoded.record_pending_token(token_id); - } - } - /// Decode prompt-only context for prefix seeding. /// /// Prompt ids may come from the model vocabulary rather than the local /// tokenizer vocabulary. For this seeding path, ids that cannot be mapped - /// back to raw token text are dropped before retrying strict decode. This - /// tolerance is intentionally limited to prompt context; generated ids are - /// decoded later through the normal strict path. + /// back to raw token text are dropped before retrying strict decode. fn decode_prompt_context(&self, ids: &[u32]) -> Result<(String, Vec)> { match self.tokenizer.decode(ids, self.skip_special_tokens) { Ok(decoded) => Ok((decoded, ids.to_vec())), @@ -144,10 +134,22 @@ impl IncrementalDecoder for DecodeStream<'_, T> { } self.ids.push(token_id); - self.record_token(token_id); let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?; let prefix_len = self.prefix.len(); - if string.len() <= prefix_len || string.ends_with('\u{FFFD}') { + let produced_text = string.len() > prefix_len && !string.ends_with('\u{FFFD}'); + + // Skipped undefined IDs leave the decoded text unchanged. Check only + // non-emitting steps to avoid id_to_token's lookup and String allocation + // on the normal path. + let zero_width = (self.skip_special_tokens && self.tokenizer.is_special_id(token_id)) + || (!produced_text && self.tokenizer.id_to_token(token_id).is_none()); + if zero_width { + self.decoded.record_zero_width_token(token_id); + } else { + self.decoded.record_pending_token(token_id); + } + + if !produced_text { return Ok(0); } // Ensure we split at a utf-8 char boundary. @@ -224,8 +226,8 @@ mod tests { unreachable!() } - fn id_to_token(&self, _id: u32) -> Option { - unreachable!() + fn id_to_token(&self, id: u32) -> Option { + Some(id.to_string()) } } @@ -456,8 +458,8 @@ mod tests { unreachable!() } - fn id_to_token(&self, _id: u32) -> Option { - unreachable!() + fn id_to_token(&self, id: u32) -> Option { + Some(id.to_string()) } }