[Rust][Benchmark] Prevent invalid token IDs in random benchmarks (#50058)

Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
Reid
2026-08-01 03:20:29 +00:00
committed by GitHub
parent eb6453d95b
commit 62195e9784
+80 -18
View File
@@ -19,9 +19,9 @@ const NUM_RESERVED_SPECIAL_TOKENS: u32 = 256;
pub struct TiktokenTokenizer {
bpe: tiktoken_rs::CoreBPE,
vocab_size: u32,
#[allow(dead_code)]
num_base_tokens: u32,
/// IDs of all special tokens (for filtering from allowed_tokens)
/// Registered special token IDs. File-based tokenizers exclude them from
/// allowed tokens; built-in tokenizers retain the previously allowed IDs.
special_token_ids: Vec<u32>,
/// Reverse mapping: token_id -> byte sequence, for lossy UTF-8 decoding.
/// Empty for built-in encodings (use bpe.decode instead).
@@ -113,12 +113,17 @@ impl TiktokenTokenizer {
}
/// Create from a built-in tiktoken encoding (o200k_base, cl100k_base, etc.)
pub fn from_builtin_bpe(bpe: tiktoken_rs::CoreBPE, vocab_size: u32) -> Self {
pub fn from_builtin_bpe(
bpe: tiktoken_rs::CoreBPE,
vocab_size: u32,
num_base_tokens: u32,
special_token_ids: Vec<u32>,
) -> Self {
Self {
bpe,
vocab_size,
num_base_tokens: vocab_size,
special_token_ids: Vec::new(),
num_base_tokens,
special_token_ids,
decoder: Vec::new(),
is_builtin: true,
}
@@ -152,14 +157,15 @@ impl TiktokenTokenizer {
self.vocab_size
}
/// Get non-special token IDs whose byte representation is valid UTF-8.
/// Excludes ALL special tokens (like Python's `set(all_tokens) - set(prohibited_tokens)`).
/// Get token IDs that can be decoded safely.
///
/// File-based tokenizers exclude special tokens and tokens whose byte
/// representation is not valid UTF-8.
pub fn get_allowed_tokens(&self) -> Vec<u32> {
if self.is_builtin {
// For built-in encodings, return full token range.
// Note: when used with random dataset, these IDs are sent to vLLM as-is;
// ensure the model's tokenizer is compatible (e.g. GPT-4o for o200k_base).
return (0..self.vocab_size).collect();
return (0..self.num_base_tokens)
.chain(self.special_token_ids.iter().copied())
.collect();
}
let special_set: std::collections::HashSet<u32> =
self.special_token_ids.iter().copied().collect();
@@ -185,12 +191,22 @@ impl TiktokenTokenizer {
/// These encodings are bundled with tiktoken-rs — no network download required.
/// Useful for consistent cross-model token counting (e.g. Artificial Analysis methodology).
pub fn load_builtin_tiktoken(encoding: &str) -> Result<TiktokenTokenizer> {
let (bpe, vocab_size) = match encoding {
"o200k_base" => (tiktoken_rs::o200k_base(), 200_275u32),
"cl100k_base" => (tiktoken_rs::cl100k_base(), 100_277u32),
"p50k_base" => (tiktoken_rs::p50k_base(), 50_281u32),
"p50k_edit" => (tiktoken_rs::p50k_edit(), 50_281u32),
"r50k_base" | "gpt2" => (tiktoken_rs::r50k_base(), 50_257u32),
let (bpe, vocab_size, num_base_tokens, special_token_ids) = match encoding {
"o200k_base" => (
tiktoken_rs::o200k_base(),
200_275u32,
199_998u32,
vec![199_999, 200_018],
),
"cl100k_base" => (
tiktoken_rs::cl100k_base(),
100_277u32,
100_256u32,
vec![100_257, 100_258, 100_259, 100_260, 100_276],
),
"p50k_base" => (tiktoken_rs::p50k_base(), 50_281u32, 50_281u32, vec![]),
"p50k_edit" => (tiktoken_rs::p50k_edit(), 50_281u32, 50_281u32, vec![]),
"r50k_base" | "gpt2" => (tiktoken_rs::r50k_base(), 50_257u32, 50_256u32, vec![50_256]),
_ => {
return Err(BenchError::Tokenizer(format!(
"Unknown built-in tiktoken encoding: '{encoding}'. \
@@ -205,7 +221,12 @@ pub fn load_builtin_tiktoken(encoding: &str) -> Result<TiktokenTokenizer> {
vocab_size,
"loaded tokenizer"
);
Ok(TiktokenTokenizer::from_builtin_bpe(bpe, vocab_size))
Ok(TiktokenTokenizer::from_builtin_bpe(
bpe,
vocab_size,
num_base_tokens,
special_token_ids,
))
}
/// Try to load a tiktoken tokenizer from a local directory or HuggingFace model repo.
@@ -552,3 +573,44 @@ fn find_unescaped(s: &str, ch: char) -> Option<usize> {
}
None
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn builtin_allowed_tokens_are_decodable() {
let cases = [
("o200k_base", 200_000),
("cl100k_base", 100_261),
("p50k_base", 50_281),
("p50k_edit", 50_281),
("r50k_base", 50_257),
("gpt2", 50_257),
];
for (encoding, expected_count) in cases {
let tokenizer = load_builtin_tiktoken(encoding).unwrap();
let allowed_tokens = tokenizer.get_allowed_tokens();
assert_eq!(allowed_tokens.len(), expected_count, "{encoding}");
for token_id in allowed_tokens {
tokenizer
.decode(&[token_id])
.unwrap_or_else(|error| panic!("{encoding} token {token_id}: {error}"));
}
}
}
#[test]
fn sparse_builtin_allowed_tokens_skip_unassigned_ids() {
let o200k = load_builtin_tiktoken("o200k_base").unwrap().get_allowed_tokens();
assert_eq!(&o200k[199_997..], &[199_997, 199_999, 200_018]);
let cl100k = load_builtin_tiktoken("cl100k_base").unwrap().get_allowed_tokens();
assert_eq!(
&cl100k[100_255..],
&[100_255, 100_257, 100_258, 100_259, 100_260, 100_276]
);
}
}