[Rust Frontend] Add ordinary-text tokenizer encoding (#49992)

Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-28 15:46:51 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 61ac368021
commit 5ed3faa43d
10 changed files with 448 additions and 2 deletions
@@ -31,6 +31,10 @@ impl Tokenizer for FixtureTokenizer {
Ok(text.bytes().map(u32::from).collect())
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
token_ids: &[u32],
+4
View File
@@ -19,6 +19,10 @@ impl Tokenizer for BenchTokenizer {
Ok(text.chars().map(|_| u32::MAX).collect())
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
token_ids: &[u32],
+8
View File
@@ -414,6 +414,10 @@ mod tests {
Ok(text.chars().map(u32::from).collect())
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
token_ids: &[u32],
@@ -733,6 +737,10 @@ mod tests {
Ok(vec![])
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
_token_ids: &[u32],
+4
View File
@@ -177,6 +177,10 @@ mod tests {
Ok(vec![])
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
_token_ids: &[u32],
+288 -2
View File
@@ -1,13 +1,20 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::borrow::Cow;
use std::path::Path;
use std::sync::Arc;
use std::sync::{Arc, LazyLock};
use fastokens::Tokenizer as FastokensTokenizer;
use fastokens::decoders::Decoder as FastokensDecoder;
use fastokens::pre_tokenized::{
PreTokenizedString as FastokensPreTokenizedString, Split as FastokensSplit,
};
use fastokens::{PreTokenizer as FastokensPreTokenizer, Split as FastokensSplitPreTokenizer};
use thiserror_ext::AsReport as _;
use tokenizers::Tokenizer as HfTokenizer;
use tokenizers::{
AddedVocabulary, Model as _, OffsetType, PreTokenizer as _, Tokenizer as HfTokenizer,
};
use tracing::{info, warn};
use crate::byte_level_decode::decode_byte_level;
@@ -16,6 +23,8 @@ use crate::{Result, Tokenizer};
mod added_tokens;
static EMPTY_HF_ADDED_VOCABULARY: LazyLock<AddedVocabulary> = LazyLock::new(AddedVocabulary::new);
enum Backend {
Hf(Box<HfTokenizer>),
Fastokens(Box<FastokensTokenizer>),
@@ -53,6 +62,85 @@ fn decode_fastokens_byte_level(
Ok(decode_byte_level(tokens))
}
fn encode_hf_ordinary(tokenizer: &HfTokenizer, text: &str) -> tokenizers::Result<Vec<u32>> {
let mut pretokenized =
EMPTY_HF_ADDED_VOCABULARY.extract_and_normalize(tokenizer.get_normalizer(), text);
if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() {
pre_tokenizer.pre_tokenize(&mut pretokenized)?;
}
pretokenized.tokenize(|normalized| tokenizer.get_model().tokenize(normalized.get()))?;
let encoding = pretokenized.into_encoding(None, 0, OffsetType::Byte)?;
let encoding = tokenizer.post_process(encoding, None, false)?;
Ok(encoding.get_ids().to_vec())
}
fn fastokens_fused_split(tokenizer: &FastokensTokenizer) -> Option<&FastokensSplitPreTokenizer> {
// Keep this predicate aligned with fastokens::Tokenizer::detect_fused_byte_level.
let FastokensPreTokenizer::Sequence(steps) = tokenizer.pre_tokenizer()? else {
return None;
};
let [
FastokensPreTokenizer::Split(split),
FastokensPreTokenizer::ByteLevel(byte_level),
] = steps.as_slice()
else {
return None;
};
byte_level.is_bulk_only().then_some(split)
}
fn fastokens_pre_tokenized_ordinary(
tokenizer: &FastokensTokenizer,
text: &str,
) -> FastokensPreTokenizedString {
// This is fastokens::Tokenizer::build_pre_tokenized with added_tokens = None.
let normalized = tokenizer
.normalizer()
.map_or(Cow::Borrowed(text), |normalizer| normalizer.normalize(text));
match normalized {
Cow::Borrowed(_) => FastokensPreTokenizedString::from_text(text),
Cow::Owned(text) => {
let len = text.len();
FastokensPreTokenizedString::new(
text,
vec![FastokensSplit {
range: 0..len,
token_id: None,
}],
)
}
}
}
fn encode_fastokens_ordinary(
tokenizer: &FastokensTokenizer,
text: &str,
) -> std::result::Result<Vec<u32>, fastokens::Error> {
if text.is_empty() {
return Ok(Vec::new());
}
let mut pretokenized = fastokens_pre_tokenized_ordinary(tokenizer, text);
let ids = if let Some(split) = fastokens_fused_split(tokenizer) {
split.pre_tokenize(&mut pretokenized)?;
pretokenized
.tokenize_batched(|buffer, splits, output| {
tokenizer.model().tokenize_batch_fused(buffer, splits, output)
})
.map_err(fastokens::Error::Model)?
} else {
if let Some(pre_tokenizer) = tokenizer.pre_tokenizer() {
pre_tokenizer.pre_tokenize(&mut pretokenized)?;
}
pretokenized
.tokenize(|text, output| tokenizer.model().tokenize_into(text, output))
.map_err(fastokens::Error::Model)?
};
Ok(tokenizer.post_process(ids, false))
}
/// Tokenizer from `tokenizer.json` in HuggingFace format.
///
/// This tries to load with `fastokens` first for better performance, then falls
@@ -156,6 +244,17 @@ impl Tokenizer for HuggingFaceTokenizer {
}
}
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
match &self.backend {
Backend::Hf(tokenizer) => encode_hf_ordinary(tokenizer, text)
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report())),
Backend::Fastokens(tokenizer) | Backend::FastokensByteLevel(tokenizer) => {
encode_fastokens_ordinary(tokenizer, text)
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report()))
}
}
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
match &self.backend {
Backend::Hf(t) => t
@@ -200,12 +299,19 @@ impl Tokenizer for HuggingFaceTokenizer {
#[cfg(test)]
mod tests {
use std::path::{Path, PathBuf};
use serde_json::{Value, json};
use tempfile::tempdir;
use tokenizers::models::bpe::BPE;
use tokenizers::pre_tokenizers::byte_level::ByteLevel;
use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
use super::{HuggingFaceTokenizer, Tokenizer};
const REGULAR_TOKEN: &str = "<|regular|>";
const SPECIAL_TOKEN: &str = "<|special|>";
fn tiny_bpe_tokenizer() -> HfTokenizer {
let vocab = [
("<unk>".to_string(), 0),
@@ -232,6 +338,186 @@ mod tests {
HfTokenizer::new(model)
}
fn ordinary_test_tokenizer_json(fused: bool, with_added_tokens: bool) -> Value {
let mut alphabet: Vec<char> = ByteLevel::alphabet().into_iter().collect();
alphabet.sort_unstable();
let vocab = alphabet
.into_iter()
.enumerate()
.map(|(id, token)| (token.to_string(), json!(id)))
.collect::<serde_json::Map<_, _>>();
let pre_tokenizer = if fused {
json!({
"type": "Sequence",
"pretokenizers": [
{
"type": "Split",
"pattern": {"Regex": "\\S+|\\s+"},
"behavior": "Isolated",
"invert": false
},
{
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": false
}
]
})
} else {
json!({
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
})
};
let added_tokens = with_added_tokens.then(|| {
json!([
{
"id": 256,
"content": REGULAR_TOKEN,
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
},
{
"id": 257,
"content": SPECIAL_TOKEN,
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
])
});
json!({
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 24,
"strategy": "LongestFirst",
"stride": 0
},
"padding": null,
"added_tokens": added_tokens.unwrap_or_else(|| json!([])),
"normalizer": {"type": "NFC"},
"pre_tokenizer": pre_tokenizer,
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": vocab,
"merges": []
}
})
}
fn write_tokenizer_json(dir: &Path, name: &str, value: &Value) -> PathBuf {
let path = dir.join(name);
std::fs::write(
&path,
serde_json::to_vec(value).expect("serialize tokenizer"),
)
.expect("write tokenizer");
path
}
fn assert_ordinary_matches_added_empty(
constructor: fn(&Path) -> crate::Result<HuggingFaceTokenizer>,
fused: bool,
) {
let dir = tempdir().expect("create temp dir");
let added_path = write_tokenizer_json(
dir.path(),
"with-added.json",
&ordinary_test_tokenizer_json(fused, true),
);
let empty_path = write_tokenizer_json(
dir.path(),
"added-empty.json",
&ordinary_test_tokenizer_json(fused, false),
);
let tokenizer = constructor(&added_path).expect("load tokenizer with added tokens");
let added_empty = constructor(&empty_path).expect("load tokenizer with empty added tokens");
if let super::Backend::Fastokens(inner) | super::Backend::FastokensByteLevel(inner) =
&tokenizer.backend
{
assert_eq!(super::fastokens_fused_split(inner).is_some(), fused);
}
assert_eq!(
tokenizer.encode(REGULAR_TOKEN, false).unwrap(),
vec![tokenizer.token_to_id(REGULAR_TOKEN).unwrap()]
);
assert_eq!(
tokenizer.encode(SPECIAL_TOKEN, false).unwrap(),
vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()]
);
for text in [
"",
"hello",
"Cafe\u{301}",
REGULAR_TOKEN,
SPECIAL_TOKEN,
"hello <|regular|> Cafe\u{301} <|special|> tail",
] {
assert_eq!(
tokenizer.encode_ordinary(text).unwrap(),
added_empty.encode(text, false).unwrap(),
"fused={fused}, text={text:?}",
);
}
if matches!(&tokenizer.backend, super::Backend::Hf(_)) {
assert_eq!(
tokenizer
.encode_ordinary("hello <|regular|> Cafe\u{301} <|special|> tail")
.unwrap()
.len(),
24,
"HF post-processing must retain configured truncation",
);
}
}
#[test]
fn hf_ordinary_matches_original_encode_with_added_empty() {
for fused in [false, true] {
assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_hf, fused);
}
}
#[test]
fn fastokens_ordinary_matches_original_encode_with_added_empty() {
for fused in [false, true] {
assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_fastokens, fused);
}
}
#[test]
fn hf_constructor_resolves_added_token_ids() {
let mut tokenizer = tiny_bpe_tokenizer();
+12
View File
@@ -199,6 +199,10 @@ mod tests {
unreachable!()
}
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
Ok(String::from_utf8_lossy(&bytes).into_owned())
@@ -273,6 +277,10 @@ mod tests {
unreachable!()
}
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
let mut text = String::new();
for &token_id in token_ids {
@@ -410,6 +418,10 @@ mod tests {
unreachable!()
}
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
match token_ids {
[1] => Ok("abc".into()),
+4
View File
@@ -25,6 +25,10 @@ pub trait Tokenizer: Send + Sync {
/// Encode one prompt string into token IDs.
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>;
/// Equivalent to `encode(text, false)`, except that every added,
/// special, and control-token matcher is bypassed.
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>>;
/// Decode one token sequence into text.
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String>;
+54
View File
@@ -35,6 +35,12 @@ impl Tokenizer for TekkenTokenizer {
.map_err(|error| tokenizer_error!("encoding failed: {error}"))
}
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
self.inner
.encode(text, false, false)
.map_err(|error| tokenizer_error!("encoding failed: {error}"))
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
let policy = if skip_special_tokens {
tekken::SpecialTokenPolicy::Ignore
@@ -67,3 +73,51 @@ impl Tokenizer for TekkenTokenizer {
self.inner.is_special_token(token_id)
}
}
#[cfg(test)]
mod tests {
use base64::Engine as _;
use tekken::config::TokenizerVersion;
use tekken::{SpecialTokenInfo, TokenInfo};
use super::*;
fn test_tokenizer() -> TekkenTokenizer {
let vocab = (0_u8..=255)
.map(|byte| TokenInfo {
rank: byte as usize,
token_bytes: base64::engine::general_purpose::STANDARD.encode([byte]),
token_str: None,
})
.collect();
let special_tokens = vec![SpecialTokenInfo {
rank: 0,
token_str: "<control>".to_string(),
is_control: true,
}];
let inner = Tekkenizer::new(
vocab,
&special_tokens,
r"(?s).",
257,
1,
TokenizerVersion::V3,
None,
)
.expect("build Tekken tokenizer");
TekkenTokenizer { inner }
}
#[test]
fn ordinary_matches_tekkens_empty_special_encoding() {
let tokenizer = test_tokenizer();
let text = "user <control> text";
let control_id = tokenizer.token_to_id("<control>").unwrap();
let ordinary_ids = tokenizer.encode_ordinary(text).unwrap();
assert_eq!(control_id, 0);
assert_eq!(ordinary_ids, tokenizer.encode(text, false).unwrap());
assert!(!ordinary_ids.contains(&control_id));
assert_eq!(tokenizer.decode(&ordinary_ids, false).unwrap(), text);
}
}
+30
View File
@@ -208,6 +208,10 @@ impl Tokenizer for TestTokenizer {
Ok(ids)
}
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
Ok(text.as_bytes().iter().copied().map(u32::from).collect())
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
let mut output = String::new();
let mut pending_bytes = Vec::new();
@@ -374,6 +378,32 @@ mod tests {
assert!(!tokenizer.is_special_id(0xF002));
}
#[test]
fn ordinary_encoding_bypasses_all_configured_tokens() {
let tokenizer = TestTokenizer::new()
.with_bos_token("<bos>", 256)
.with_special_token("<control>", 257)
.with_regular_token("<visible>", 258);
let ordinary_text = "user <control> and <visible>";
assert_eq!(tokenizer.encode("<control>", false).unwrap(), vec![257]);
assert_eq!(tokenizer.encode("<visible>", false).unwrap(), vec![258]);
assert_eq!(
tokenizer.encode_ordinary(ordinary_text).unwrap(),
ordinary_text.as_bytes().iter().copied().map(u32::from).collect::<Vec<_>>()
);
let mut segmented = tokenizer.encode("<control>", false).unwrap();
segmented.extend(tokenizer.encode_ordinary(ordinary_text).unwrap());
segmented.extend(tokenizer.encode("<visible>", false).unwrap());
assert_eq!(segmented.first(), Some(&257));
assert_eq!(segmented.last(), Some(&258));
assert_eq!(
tokenizer.decode(&segmented, false).unwrap(),
format!("<control>{ordinary_text}<visible>")
);
}
#[test]
#[should_panic(expected = "configured test token id 255 overlaps byte fallback range 0..=255")]
fn configured_token_id_must_stay_outside_byte_range() {
+40
View File
@@ -462,6 +462,13 @@ impl Tokenizer for TiktokenTokenizer {
})
}
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
Ok(match &self.backend {
Backend::Riptoken(backend) => backend.inner.encode_ordinary(text),
Backend::TiktokenRs(backend) => backend.inner.encode_ordinary(text),
})
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
// Filter passes:
//
@@ -752,6 +759,39 @@ mod tests {
}
}
#[test]
fn tiktoken_ordinary_bypasses_every_registered_added_token() {
let dir = tempfile::tempdir().expect("create temp dir");
let bpe_path = write_synthetic_bpe_file(dir.path());
fs::write(
dir.path().join("tokenizer_config.json"),
r#"{
"added_tokens_decoder": {
"257": { "content": "<|im_end|>", "special": true },
"258": { "content": "<|tool_call_begin|>", "special": false }
}
}"#,
)
.expect("write tokenizer_config.json");
fs::write(dir.path().join("config.json"), r#"{"vocab_size": 260}"#)
.expect("write config.json");
let input = "<|im_end|><|tool_call_begin|><|reserved_token_259|>";
let expected: Vec<u32> = input.as_bytes().iter().copied().map(u32::from).collect();
for backend in explicit_backends(&bpe_path) {
assert_eq!(backend.encode("<|im_end|>", false).unwrap(), vec![257]);
assert_eq!(
backend.encode("<|tool_call_begin|>", false).unwrap(),
vec![258]
);
assert_eq!(
backend.encode("<|reserved_token_259|>", false).unwrap(),
vec![259]
);
assert_eq!(backend.encode_ordinary(input).unwrap(), expected);
}
}
/// `vocab_size` may live under `text_config` for composite (e.g.
/// multimodal) configs.
#[test]