[Bugfix][Rust Frontend] Pass through raw characters in byte-level decode (#59861)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-09 11:04:00 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9a585c62aa
commit ce4603cad6
2 changed files with 86 additions and 12 deletions
+42 -12
View File
@@ -7,14 +7,16 @@
/// Reverse GPT-2 byte-to-unicode mapping: codepoint → original byte. The GPT-2
/// table only emits codepoints in U+0000..U+0143, so a flat array suffices.
const CHAR_TO_BYTE: [u8; 324] = build_char_to_byte();
/// Codepoints in that range the table never emits, such as raw whitespace in
/// an added token's content, map to `None`.
const CHAR_TO_BYTE: [Option<u8>; 324] = build_char_to_byte();
const fn is_nice(b: u8) -> bool {
(b >= b'!' && b <= b'~') || (b >= 0xA1 && b <= 0xAC) || b >= 0xAE
}
const fn build_char_to_byte() -> [u8; 324] {
let mut table = [0u8; 324];
const fn build_char_to_byte() -> [Option<u8>; 324] {
let mut table = [None; 324];
let mut b: u16 = 0;
while b < 256 {
let cp = if is_nice(b as u8) {
@@ -22,7 +24,7 @@ const fn build_char_to_byte() -> [u8; 324] {
} else {
256 + nice_offset(b as u8)
};
table[cp as usize] = b as u8;
table[cp as usize] = Some(b as u8);
b += 1;
}
table
@@ -41,20 +43,23 @@ const fn nice_offset(b: u8) -> u32 {
}
/// Decode byte-level encoded token strings into a single UTF-8 string,
/// matching `fastokens::decoders::ByteLevelDecoder`.
/// matching the HuggingFace `tokenizers` `ByteLevel` decoder.
pub fn decode_byte_level<'a, I: IntoIterator<Item = &'a str>>(tokens: I) -> String {
let iter = tokens.into_iter();
let (lower, _) = iter.size_hint();
let mut bytes: Vec<u8> = Vec::with_capacity(lower.saturating_mul(4));
for token in iter {
let start = bytes.len();
for c in token.chars() {
let cp = c as usize;
if cp < CHAR_TO_BYTE.len() {
bytes.push(CHAR_TO_BYTE[cp]);
if let Some(&Some(b)) = CHAR_TO_BYTE.get(c as usize) {
bytes.push(b);
} else {
// Non-GPT2 codepoints (e.g. DeepSeek's U+FF5C, U+2581) pass through.
let mut buf = [0u8; 4];
bytes.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
// As in HF, a token with any non-GPT2 codepoint is raw text and
// passes through whole: DeepSeek's U+FF5C and U+2581, or raw
// characters in added tokens (e.g. `\n</parameter>`).
bytes.truncate(start);
bytes.extend_from_slice(token.as_bytes());
break;
}
}
}
@@ -87,7 +92,7 @@ mod tests {
for b in 0..=255u8 {
let cp = byte_to_char[b as usize] as usize;
assert!(cp < CHAR_TO_BYTE.len());
assert_eq!(CHAR_TO_BYTE[cp], b, "mismatch for byte {b:#x}");
assert_eq!(CHAR_TO_BYTE[cp], Some(b), "mismatch for byte {b:#x}");
}
}
@@ -119,4 +124,29 @@ mod tests {
let tok = "<\u{FF5C}begin\u{2581}of\u{2581}sentence\u{FF5C}>";
assert_eq!(decode_byte_level([tok]), "<|begin▁of▁sentence|>");
}
#[test]
fn decode_preserves_raw_chars_outside_gpt2_table() {
// Added tokens store raw text, which may contain codepoints below
// U+0144 that the GPT-2 table never emits.
let toks = [
"<parameter name=\"",
"city",
"\">",
"\n</parameter>",
"\t\u{7F}\u{A0}\u{AD}",
];
assert_eq!(
decode_byte_level(toks),
"<parameter name=\"city\">\n</parameter>\t\u{7F}\u{A0}\u{AD}",
);
}
#[test]
fn decode_passes_through_whole_token_with_raw_chars() {
// Characters that the GPT-2 table does emit (`Ġ`, `é`) stay verbatim
// in a raw token instead of decoding to bytes 0x20 and 0xE9.
assert_eq!(decode_byte_level(["\u{120}x\n"]), "\u{120}x\n");
assert_eq!(decode_byte_level(["\ncafé"]), "\ncafé");
}
}
+44
View File
@@ -554,6 +554,50 @@ mod tests {
}
}
/// Added tokens store raw text, including whitespace that the GPT-2 byte
/// table never emits, so a byte-level decode must pass it through verbatim,
/// along with characters the table does emit in the same token (`é`).
#[test]
fn added_tokens_with_raw_whitespace_decode_verbatim() {
const OPEN: &str = "<parameter name=\"";
const CLOSE: &str = "\n</parameter>";
const MIXED: &str = "\ncafé";
let mut value = ordinary_test_tokenizer_json(false, false);
value["added_tokens"] = json!(
[OPEN, CLOSE, MIXED]
.iter()
.enumerate()
.map(|(i, content)| {
json!({
"id": 256 + i,
"content": content,
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
})
})
.collect::<Vec<_>>()
);
let dir = tempdir().expect("create temp dir");
let path = write_tokenizer_json(dir.path(), "tokenizer.json", &value);
let text = format!("{OPEN}city\">\nParis{CLOSE}{MIXED}");
let fastokens = HuggingFaceTokenizer::new_fastokens(&path).expect("load fastokens wrapper");
assert!(matches!(
fastokens.backend,
super::Backend::FastokensByteLevel(_)
));
let hf = HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper");
for wrapper in [fastokens, hf] {
let ids = wrapper.encode(&text, false).expect("encode");
assert!((256..=258).all(|id| ids.contains(&id)), "ids={ids:?}");
assert_eq!(wrapper.decode(&ids, true).expect("decode"), text);
}
}
#[test]
fn hf_vocab_size_counts_added_tokens() {
let mut tokenizer = tiny_bpe_tokenizer();