[Bugfix][Rust Frontend] Pass through raw characters in byte-level decode (#59861)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9a585c62aa
commit
ce4603cad6
@@ -7,14 +7,16 @@
|
||||
|
||||
/// Reverse GPT-2 byte-to-unicode mapping: codepoint → original byte. The GPT-2
|
||||
/// table only emits codepoints in U+0000..U+0143, so a flat array suffices.
|
||||
const CHAR_TO_BYTE: [u8; 324] = build_char_to_byte();
|
||||
/// Codepoints in that range the table never emits, such as raw whitespace in
|
||||
/// an added token's content, map to `None`.
|
||||
const CHAR_TO_BYTE: [Option<u8>; 324] = build_char_to_byte();
|
||||
|
||||
const fn is_nice(b: u8) -> bool {
|
||||
(b >= b'!' && b <= b'~') || (b >= 0xA1 && b <= 0xAC) || b >= 0xAE
|
||||
}
|
||||
|
||||
const fn build_char_to_byte() -> [u8; 324] {
|
||||
let mut table = [0u8; 324];
|
||||
const fn build_char_to_byte() -> [Option<u8>; 324] {
|
||||
let mut table = [None; 324];
|
||||
let mut b: u16 = 0;
|
||||
while b < 256 {
|
||||
let cp = if is_nice(b as u8) {
|
||||
@@ -22,7 +24,7 @@ const fn build_char_to_byte() -> [u8; 324] {
|
||||
} else {
|
||||
256 + nice_offset(b as u8)
|
||||
};
|
||||
table[cp as usize] = b as u8;
|
||||
table[cp as usize] = Some(b as u8);
|
||||
b += 1;
|
||||
}
|
||||
table
|
||||
@@ -41,20 +43,23 @@ const fn nice_offset(b: u8) -> u32 {
|
||||
}
|
||||
|
||||
/// Decode byte-level encoded token strings into a single UTF-8 string,
|
||||
/// matching `fastokens::decoders::ByteLevelDecoder`.
|
||||
/// matching the HuggingFace `tokenizers` `ByteLevel` decoder.
|
||||
pub fn decode_byte_level<'a, I: IntoIterator<Item = &'a str>>(tokens: I) -> String {
|
||||
let iter = tokens.into_iter();
|
||||
let (lower, _) = iter.size_hint();
|
||||
let mut bytes: Vec<u8> = Vec::with_capacity(lower.saturating_mul(4));
|
||||
for token in iter {
|
||||
let start = bytes.len();
|
||||
for c in token.chars() {
|
||||
let cp = c as usize;
|
||||
if cp < CHAR_TO_BYTE.len() {
|
||||
bytes.push(CHAR_TO_BYTE[cp]);
|
||||
if let Some(&Some(b)) = CHAR_TO_BYTE.get(c as usize) {
|
||||
bytes.push(b);
|
||||
} else {
|
||||
// Non-GPT2 codepoints (e.g. DeepSeek's U+FF5C, U+2581) pass through.
|
||||
let mut buf = [0u8; 4];
|
||||
bytes.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
|
||||
// As in HF, a token with any non-GPT2 codepoint is raw text and
|
||||
// passes through whole: DeepSeek's U+FF5C and U+2581, or raw
|
||||
// characters in added tokens (e.g. `\n</parameter>`).
|
||||
bytes.truncate(start);
|
||||
bytes.extend_from_slice(token.as_bytes());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -87,7 +92,7 @@ mod tests {
|
||||
for b in 0..=255u8 {
|
||||
let cp = byte_to_char[b as usize] as usize;
|
||||
assert!(cp < CHAR_TO_BYTE.len());
|
||||
assert_eq!(CHAR_TO_BYTE[cp], b, "mismatch for byte {b:#x}");
|
||||
assert_eq!(CHAR_TO_BYTE[cp], Some(b), "mismatch for byte {b:#x}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,4 +124,29 @@ mod tests {
|
||||
let tok = "<\u{FF5C}begin\u{2581}of\u{2581}sentence\u{FF5C}>";
|
||||
assert_eq!(decode_byte_level([tok]), "<|begin▁of▁sentence|>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_preserves_raw_chars_outside_gpt2_table() {
|
||||
// Added tokens store raw text, which may contain codepoints below
|
||||
// U+0144 that the GPT-2 table never emits.
|
||||
let toks = [
|
||||
"<parameter name=\"",
|
||||
"city",
|
||||
"\">",
|
||||
"\n</parameter>",
|
||||
"\t\u{7F}\u{A0}\u{AD}",
|
||||
];
|
||||
assert_eq!(
|
||||
decode_byte_level(toks),
|
||||
"<parameter name=\"city\">\n</parameter>\t\u{7F}\u{A0}\u{AD}",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_passes_through_whole_token_with_raw_chars() {
|
||||
// Characters that the GPT-2 table does emit (`Ġ`, `é`) stay verbatim
|
||||
// in a raw token instead of decoding to bytes 0x20 and 0xE9.
|
||||
assert_eq!(decode_byte_level(["\u{120}x\n"]), "\u{120}x\n");
|
||||
assert_eq!(decode_byte_level(["\ncafé"]), "\ncafé");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -554,6 +554,50 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Added tokens store raw text, including whitespace that the GPT-2 byte
|
||||
/// table never emits, so a byte-level decode must pass it through verbatim,
|
||||
/// along with characters the table does emit in the same token (`é`).
|
||||
#[test]
|
||||
fn added_tokens_with_raw_whitespace_decode_verbatim() {
|
||||
const OPEN: &str = "<parameter name=\"";
|
||||
const CLOSE: &str = "\n</parameter>";
|
||||
const MIXED: &str = "\ncafé";
|
||||
|
||||
let mut value = ordinary_test_tokenizer_json(false, false);
|
||||
value["added_tokens"] = json!(
|
||||
[OPEN, CLOSE, MIXED]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, content)| {
|
||||
json!({
|
||||
"id": 256 + i,
|
||||
"content": content,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false,
|
||||
"special": false
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
let dir = tempdir().expect("create temp dir");
|
||||
let path = write_tokenizer_json(dir.path(), "tokenizer.json", &value);
|
||||
let text = format!("{OPEN}city\">\nParis{CLOSE}{MIXED}");
|
||||
|
||||
let fastokens = HuggingFaceTokenizer::new_fastokens(&path).expect("load fastokens wrapper");
|
||||
assert!(matches!(
|
||||
fastokens.backend,
|
||||
super::Backend::FastokensByteLevel(_)
|
||||
));
|
||||
let hf = HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper");
|
||||
for wrapper in [fastokens, hf] {
|
||||
let ids = wrapper.encode(&text, false).expect("encode");
|
||||
assert!((256..=258).all(|id| ids.contains(&id)), "ids={ids:?}");
|
||||
assert_eq!(wrapper.decode(&ids, true).expect("decode"), text);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_vocab_size_counts_added_tokens() {
|
||||
let mut tokenizer = tiny_bpe_tokenizer();
|
||||
|
||||
Reference in New Issue
Block a user