diff --git a/rust/src/tokenizer/src/byte_level_decode.rs b/rust/src/tokenizer/src/byte_level_decode.rs index ad71f6decb..ef7a3c1ef6 100644 --- a/rust/src/tokenizer/src/byte_level_decode.rs +++ b/rust/src/tokenizer/src/byte_level_decode.rs @@ -7,14 +7,16 @@ /// Reverse GPT-2 byte-to-unicode mapping: codepoint → original byte. The GPT-2 /// table only emits codepoints in U+0000..U+0143, so a flat array suffices. -const CHAR_TO_BYTE: [u8; 324] = build_char_to_byte(); +/// Codepoints in that range the table never emits, such as raw whitespace in +/// an added token's content, map to `None`. +const CHAR_TO_BYTE: [Option; 324] = build_char_to_byte(); const fn is_nice(b: u8) -> bool { (b >= b'!' && b <= b'~') || (b >= 0xA1 && b <= 0xAC) || b >= 0xAE } -const fn build_char_to_byte() -> [u8; 324] { - let mut table = [0u8; 324]; +const fn build_char_to_byte() -> [Option; 324] { + let mut table = [None; 324]; let mut b: u16 = 0; while b < 256 { let cp = if is_nice(b as u8) { @@ -22,7 +24,7 @@ const fn build_char_to_byte() -> [u8; 324] { } else { 256 + nice_offset(b as u8) }; - table[cp as usize] = b as u8; + table[cp as usize] = Some(b as u8); b += 1; } table @@ -41,20 +43,23 @@ const fn nice_offset(b: u8) -> u32 { } /// Decode byte-level encoded token strings into a single UTF-8 string, -/// matching `fastokens::decoders::ByteLevelDecoder`. +/// matching the HuggingFace `tokenizers` `ByteLevel` decoder. pub fn decode_byte_level<'a, I: IntoIterator>(tokens: I) -> String { let iter = tokens.into_iter(); let (lower, _) = iter.size_hint(); let mut bytes: Vec = Vec::with_capacity(lower.saturating_mul(4)); for token in iter { + let start = bytes.len(); for c in token.chars() { - let cp = c as usize; - if cp < CHAR_TO_BYTE.len() { - bytes.push(CHAR_TO_BYTE[cp]); + if let Some(&Some(b)) = CHAR_TO_BYTE.get(c as usize) { + bytes.push(b); } else { - // Non-GPT2 codepoints (e.g. DeepSeek's U+FF5C, U+2581) pass through. - let mut buf = [0u8; 4]; - bytes.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); + // As in HF, a token with any non-GPT2 codepoint is raw text and + // passes through whole: DeepSeek's U+FF5C and U+2581, or raw + // characters in added tokens (e.g. `\n`). + bytes.truncate(start); + bytes.extend_from_slice(token.as_bytes()); + break; } } } @@ -87,7 +92,7 @@ mod tests { for b in 0..=255u8 { let cp = byte_to_char[b as usize] as usize; assert!(cp < CHAR_TO_BYTE.len()); - assert_eq!(CHAR_TO_BYTE[cp], b, "mismatch for byte {b:#x}"); + assert_eq!(CHAR_TO_BYTE[cp], Some(b), "mismatch for byte {b:#x}"); } } @@ -119,4 +124,29 @@ mod tests { let tok = "<\u{FF5C}begin\u{2581}of\u{2581}sentence\u{FF5C}>"; assert_eq!(decode_byte_level([tok]), "<|begin▁of▁sentence|>"); } + + #[test] + fn decode_preserves_raw_chars_outside_gpt2_table() { + // Added tokens store raw text, which may contain codepoints below + // U+0144 that the GPT-2 table never emits. + let toks = [ + "", + "\n", + "\t\u{7F}\u{A0}\u{AD}", + ]; + assert_eq!( + decode_byte_level(toks), + "\n\t\u{7F}\u{A0}\u{AD}", + ); + } + + #[test] + fn decode_passes_through_whole_token_with_raw_chars() { + // Characters that the GPT-2 table does emit (`Ġ`, `é`) stay verbatim + // in a raw token instead of decoding to bytes 0x20 and 0xE9. + assert_eq!(decode_byte_level(["\u{120}x\n"]), "\u{120}x\n"); + assert_eq!(decode_byte_level(["\ncafé"]), "\ncafé"); + } } diff --git a/rust/src/tokenizer/src/hf.rs b/rust/src/tokenizer/src/hf.rs index 44ddc6cb27..3ae1779b42 100644 --- a/rust/src/tokenizer/src/hf.rs +++ b/rust/src/tokenizer/src/hf.rs @@ -554,6 +554,50 @@ mod tests { } } + /// Added tokens store raw text, including whitespace that the GPT-2 byte + /// table never emits, so a byte-level decode must pass it through verbatim, + /// along with characters the table does emit in the same token (`é`). + #[test] + fn added_tokens_with_raw_whitespace_decode_verbatim() { + const OPEN: &str = ">() + ); + let dir = tempdir().expect("create temp dir"); + let path = write_tokenizer_json(dir.path(), "tokenizer.json", &value); + let text = format!("{OPEN}city\">\nParis{CLOSE}{MIXED}"); + + let fastokens = HuggingFaceTokenizer::new_fastokens(&path).expect("load fastokens wrapper"); + assert!(matches!( + fastokens.backend, + super::Backend::FastokensByteLevel(_) + )); + let hf = HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper"); + for wrapper in [fastokens, hf] { + let ids = wrapper.encode(&text, false).expect("encode"); + assert!((256..=258).all(|id| ids.contains(&id)), "ids={ids:?}"); + assert_eq!(wrapper.decode(&ids, true).expect("decode"), text); + } + } + #[test] fn hf_vocab_size_counts_added_tokens() { let mut tokenizer = tiny_bpe_tokenizer();