[Rust Frontend] Add ordinary-text tokenizer encoding (#49992)
Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
61ac368021
commit
5ed3faa43d
@@ -31,6 +31,10 @@ impl Tokenizer for FixtureTokenizer {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
|
||||
@@ -19,6 +19,10 @@ impl Tokenizer for BenchTokenizer {
|
||||
Ok(text.chars().map(|_| u32::MAX).collect())
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
|
||||
@@ -414,6 +414,10 @@ mod tests {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
@@ -733,6 +737,10 @@ mod tests {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
|
||||
@@ -177,6 +177,10 @@ mod tests {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use fastokens::Tokenizer as FastokensTokenizer;
|
||||
use fastokens::decoders::Decoder as FastokensDecoder;
|
||||
use fastokens::pre_tokenized::{
|
||||
PreTokenizedString as FastokensPreTokenizedString, Split as FastokensSplit,
|
||||
};
|
||||
use fastokens::{PreTokenizer as FastokensPreTokenizer, Split as FastokensSplitPreTokenizer};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokenizers::Tokenizer as HfTokenizer;
|
||||
use tokenizers::{
|
||||
AddedVocabulary, Model as _, OffsetType, PreTokenizer as _, Tokenizer as HfTokenizer,
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::byte_level_decode::decode_byte_level;
|
||||
@@ -16,6 +23,8 @@ use crate::{Result, Tokenizer};
|
||||
|
||||
mod added_tokens;
|
||||
|
||||
static EMPTY_HF_ADDED_VOCABULARY: LazyLock<AddedVocabulary> = LazyLock::new(AddedVocabulary::new);
|
||||
|
||||
enum Backend {
|
||||
Hf(Box<HfTokenizer>),
|
||||
Fastokens(Box<FastokensTokenizer>),
|
||||
@@ -53,6 +62,85 @@ fn decode_fastokens_byte_level(
|
||||
Ok(decode_byte_level(tokens))
|
||||
}
|
||||
|
||||
fn encode_hf_ordinary(tokenizer: &HfTokenizer, text: &str) -> tokenizers::Result<Vec<u32>> {
|
||||
let mut pretokenized =
|
||||
EMPTY_HF_ADDED_VOCABULARY.extract_and_normalize(tokenizer.get_normalizer(), text);
|
||||
|
||||
if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() {
|
||||
pre_tokenizer.pre_tokenize(&mut pretokenized)?;
|
||||
}
|
||||
pretokenized.tokenize(|normalized| tokenizer.get_model().tokenize(normalized.get()))?;
|
||||
let encoding = pretokenized.into_encoding(None, 0, OffsetType::Byte)?;
|
||||
let encoding = tokenizer.post_process(encoding, None, false)?;
|
||||
Ok(encoding.get_ids().to_vec())
|
||||
}
|
||||
|
||||
fn fastokens_fused_split(tokenizer: &FastokensTokenizer) -> Option<&FastokensSplitPreTokenizer> {
|
||||
// Keep this predicate aligned with fastokens::Tokenizer::detect_fused_byte_level.
|
||||
let FastokensPreTokenizer::Sequence(steps) = tokenizer.pre_tokenizer()? else {
|
||||
return None;
|
||||
};
|
||||
let [
|
||||
FastokensPreTokenizer::Split(split),
|
||||
FastokensPreTokenizer::ByteLevel(byte_level),
|
||||
] = steps.as_slice()
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
byte_level.is_bulk_only().then_some(split)
|
||||
}
|
||||
|
||||
fn fastokens_pre_tokenized_ordinary(
|
||||
tokenizer: &FastokensTokenizer,
|
||||
text: &str,
|
||||
) -> FastokensPreTokenizedString {
|
||||
// This is fastokens::Tokenizer::build_pre_tokenized with added_tokens = None.
|
||||
let normalized = tokenizer
|
||||
.normalizer()
|
||||
.map_or(Cow::Borrowed(text), |normalizer| normalizer.normalize(text));
|
||||
match normalized {
|
||||
Cow::Borrowed(_) => FastokensPreTokenizedString::from_text(text),
|
||||
Cow::Owned(text) => {
|
||||
let len = text.len();
|
||||
FastokensPreTokenizedString::new(
|
||||
text,
|
||||
vec![FastokensSplit {
|
||||
range: 0..len,
|
||||
token_id: None,
|
||||
}],
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_fastokens_ordinary(
|
||||
tokenizer: &FastokensTokenizer,
|
||||
text: &str,
|
||||
) -> std::result::Result<Vec<u32>, fastokens::Error> {
|
||||
if text.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut pretokenized = fastokens_pre_tokenized_ordinary(tokenizer, text);
|
||||
let ids = if let Some(split) = fastokens_fused_split(tokenizer) {
|
||||
split.pre_tokenize(&mut pretokenized)?;
|
||||
pretokenized
|
||||
.tokenize_batched(|buffer, splits, output| {
|
||||
tokenizer.model().tokenize_batch_fused(buffer, splits, output)
|
||||
})
|
||||
.map_err(fastokens::Error::Model)?
|
||||
} else {
|
||||
if let Some(pre_tokenizer) = tokenizer.pre_tokenizer() {
|
||||
pre_tokenizer.pre_tokenize(&mut pretokenized)?;
|
||||
}
|
||||
pretokenized
|
||||
.tokenize(|text, output| tokenizer.model().tokenize_into(text, output))
|
||||
.map_err(fastokens::Error::Model)?
|
||||
};
|
||||
|
||||
Ok(tokenizer.post_process(ids, false))
|
||||
}
|
||||
|
||||
/// Tokenizer from `tokenizer.json` in HuggingFace format.
|
||||
///
|
||||
/// This tries to load with `fastokens` first for better performance, then falls
|
||||
@@ -156,6 +244,17 @@ impl Tokenizer for HuggingFaceTokenizer {
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
|
||||
match &self.backend {
|
||||
Backend::Hf(tokenizer) => encode_hf_ordinary(tokenizer, text)
|
||||
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report())),
|
||||
Backend::Fastokens(tokenizer) | Backend::FastokensByteLevel(tokenizer) => {
|
||||
encode_fastokens_ordinary(tokenizer, text)
|
||||
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
match &self.backend {
|
||||
Backend::Hf(t) => t
|
||||
@@ -200,12 +299,19 @@ impl Tokenizer for HuggingFaceTokenizer {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use tempfile::tempdir;
|
||||
use tokenizers::models::bpe::BPE;
|
||||
use tokenizers::pre_tokenizers::byte_level::ByteLevel;
|
||||
use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
|
||||
|
||||
use super::{HuggingFaceTokenizer, Tokenizer};
|
||||
|
||||
const REGULAR_TOKEN: &str = "<|regular|>";
|
||||
const SPECIAL_TOKEN: &str = "<|special|>";
|
||||
|
||||
fn tiny_bpe_tokenizer() -> HfTokenizer {
|
||||
let vocab = [
|
||||
("<unk>".to_string(), 0),
|
||||
@@ -232,6 +338,186 @@ mod tests {
|
||||
HfTokenizer::new(model)
|
||||
}
|
||||
|
||||
fn ordinary_test_tokenizer_json(fused: bool, with_added_tokens: bool) -> Value {
|
||||
let mut alphabet: Vec<char> = ByteLevel::alphabet().into_iter().collect();
|
||||
alphabet.sort_unstable();
|
||||
let vocab = alphabet
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(id, token)| (token.to_string(), json!(id)))
|
||||
.collect::<serde_json::Map<_, _>>();
|
||||
|
||||
let pre_tokenizer = if fused {
|
||||
json!({
|
||||
"type": "Sequence",
|
||||
"pretokenizers": [
|
||||
{
|
||||
"type": "Split",
|
||||
"pattern": {"Regex": "\\S+|\\s+"},
|
||||
"behavior": "Isolated",
|
||||
"invert": false
|
||||
},
|
||||
{
|
||||
"type": "ByteLevel",
|
||||
"add_prefix_space": false,
|
||||
"trim_offsets": true,
|
||||
"use_regex": false
|
||||
}
|
||||
]
|
||||
})
|
||||
} else {
|
||||
json!({
|
||||
"type": "ByteLevel",
|
||||
"add_prefix_space": false,
|
||||
"trim_offsets": true,
|
||||
"use_regex": true
|
||||
})
|
||||
};
|
||||
let added_tokens = with_added_tokens.then(|| {
|
||||
json!([
|
||||
{
|
||||
"id": 256,
|
||||
"content": REGULAR_TOKEN,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": true,
|
||||
"special": false
|
||||
},
|
||||
{
|
||||
"id": 257,
|
||||
"content": SPECIAL_TOKEN,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false,
|
||||
"special": true
|
||||
}
|
||||
])
|
||||
});
|
||||
|
||||
json!({
|
||||
"version": "1.0",
|
||||
"truncation": {
|
||||
"direction": "Right",
|
||||
"max_length": 24,
|
||||
"strategy": "LongestFirst",
|
||||
"stride": 0
|
||||
},
|
||||
"padding": null,
|
||||
"added_tokens": added_tokens.unwrap_or_else(|| json!([])),
|
||||
"normalizer": {"type": "NFC"},
|
||||
"pre_tokenizer": pre_tokenizer,
|
||||
"post_processor": {
|
||||
"type": "ByteLevel",
|
||||
"add_prefix_space": false,
|
||||
"trim_offsets": true,
|
||||
"use_regex": true
|
||||
},
|
||||
"decoder": {
|
||||
"type": "ByteLevel",
|
||||
"add_prefix_space": false,
|
||||
"trim_offsets": true,
|
||||
"use_regex": true
|
||||
},
|
||||
"model": {
|
||||
"type": "BPE",
|
||||
"dropout": null,
|
||||
"unk_token": null,
|
||||
"continuing_subword_prefix": null,
|
||||
"end_of_word_suffix": null,
|
||||
"fuse_unk": false,
|
||||
"byte_fallback": false,
|
||||
"ignore_merges": false,
|
||||
"vocab": vocab,
|
||||
"merges": []
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn write_tokenizer_json(dir: &Path, name: &str, value: &Value) -> PathBuf {
|
||||
let path = dir.join(name);
|
||||
std::fs::write(
|
||||
&path,
|
||||
serde_json::to_vec(value).expect("serialize tokenizer"),
|
||||
)
|
||||
.expect("write tokenizer");
|
||||
path
|
||||
}
|
||||
|
||||
fn assert_ordinary_matches_added_empty(
|
||||
constructor: fn(&Path) -> crate::Result<HuggingFaceTokenizer>,
|
||||
fused: bool,
|
||||
) {
|
||||
let dir = tempdir().expect("create temp dir");
|
||||
let added_path = write_tokenizer_json(
|
||||
dir.path(),
|
||||
"with-added.json",
|
||||
&ordinary_test_tokenizer_json(fused, true),
|
||||
);
|
||||
let empty_path = write_tokenizer_json(
|
||||
dir.path(),
|
||||
"added-empty.json",
|
||||
&ordinary_test_tokenizer_json(fused, false),
|
||||
);
|
||||
let tokenizer = constructor(&added_path).expect("load tokenizer with added tokens");
|
||||
let added_empty = constructor(&empty_path).expect("load tokenizer with empty added tokens");
|
||||
|
||||
if let super::Backend::Fastokens(inner) | super::Backend::FastokensByteLevel(inner) =
|
||||
&tokenizer.backend
|
||||
{
|
||||
assert_eq!(super::fastokens_fused_split(inner).is_some(), fused);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
tokenizer.encode(REGULAR_TOKEN, false).unwrap(),
|
||||
vec![tokenizer.token_to_id(REGULAR_TOKEN).unwrap()]
|
||||
);
|
||||
assert_eq!(
|
||||
tokenizer.encode(SPECIAL_TOKEN, false).unwrap(),
|
||||
vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()]
|
||||
);
|
||||
|
||||
for text in [
|
||||
"",
|
||||
"hello",
|
||||
"Cafe\u{301}",
|
||||
REGULAR_TOKEN,
|
||||
SPECIAL_TOKEN,
|
||||
"hello <|regular|> Cafe\u{301} <|special|> tail",
|
||||
] {
|
||||
assert_eq!(
|
||||
tokenizer.encode_ordinary(text).unwrap(),
|
||||
added_empty.encode(text, false).unwrap(),
|
||||
"fused={fused}, text={text:?}",
|
||||
);
|
||||
}
|
||||
if matches!(&tokenizer.backend, super::Backend::Hf(_)) {
|
||||
assert_eq!(
|
||||
tokenizer
|
||||
.encode_ordinary("hello <|regular|> Cafe\u{301} <|special|> tail")
|
||||
.unwrap()
|
||||
.len(),
|
||||
24,
|
||||
"HF post-processing must retain configured truncation",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_ordinary_matches_original_encode_with_added_empty() {
|
||||
for fused in [false, true] {
|
||||
assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_hf, fused);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fastokens_ordinary_matches_original_encode_with_added_empty() {
|
||||
for fused in [false, true] {
|
||||
assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_fastokens, fused);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_constructor_resolves_added_token_ids() {
|
||||
let mut tokenizer = tiny_bpe_tokenizer();
|
||||
|
||||
@@ -199,6 +199,10 @@ mod tests {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
|
||||
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
@@ -273,6 +277,10 @@ mod tests {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
let mut text = String::new();
|
||||
for &token_id in token_ids {
|
||||
@@ -410,6 +418,10 @@ mod tests {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
|
||||
match token_ids {
|
||||
[1] => Ok("abc".into()),
|
||||
|
||||
@@ -25,6 +25,10 @@ pub trait Tokenizer: Send + Sync {
|
||||
/// Encode one prompt string into token IDs.
|
||||
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>;
|
||||
|
||||
/// Equivalent to `encode(text, false)`, except that every added,
|
||||
/// special, and control-token matcher is bypassed.
|
||||
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>>;
|
||||
|
||||
/// Decode one token sequence into text.
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String>;
|
||||
|
||||
|
||||
@@ -35,6 +35,12 @@ impl Tokenizer for TekkenTokenizer {
|
||||
.map_err(|error| tokenizer_error!("encoding failed: {error}"))
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
|
||||
self.inner
|
||||
.encode(text, false, false)
|
||||
.map_err(|error| tokenizer_error!("encoding failed: {error}"))
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
let policy = if skip_special_tokens {
|
||||
tekken::SpecialTokenPolicy::Ignore
|
||||
@@ -67,3 +73,51 @@ impl Tokenizer for TekkenTokenizer {
|
||||
self.inner.is_special_token(token_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use base64::Engine as _;
|
||||
use tekken::config::TokenizerVersion;
|
||||
use tekken::{SpecialTokenInfo, TokenInfo};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn test_tokenizer() -> TekkenTokenizer {
|
||||
let vocab = (0_u8..=255)
|
||||
.map(|byte| TokenInfo {
|
||||
rank: byte as usize,
|
||||
token_bytes: base64::engine::general_purpose::STANDARD.encode([byte]),
|
||||
token_str: None,
|
||||
})
|
||||
.collect();
|
||||
let special_tokens = vec![SpecialTokenInfo {
|
||||
rank: 0,
|
||||
token_str: "<control>".to_string(),
|
||||
is_control: true,
|
||||
}];
|
||||
let inner = Tekkenizer::new(
|
||||
vocab,
|
||||
&special_tokens,
|
||||
r"(?s).",
|
||||
257,
|
||||
1,
|
||||
TokenizerVersion::V3,
|
||||
None,
|
||||
)
|
||||
.expect("build Tekken tokenizer");
|
||||
TekkenTokenizer { inner }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_matches_tekkens_empty_special_encoding() {
|
||||
let tokenizer = test_tokenizer();
|
||||
let text = "user <control> text";
|
||||
let control_id = tokenizer.token_to_id("<control>").unwrap();
|
||||
let ordinary_ids = tokenizer.encode_ordinary(text).unwrap();
|
||||
|
||||
assert_eq!(control_id, 0);
|
||||
assert_eq!(ordinary_ids, tokenizer.encode(text, false).unwrap());
|
||||
assert!(!ordinary_ids.contains(&control_id));
|
||||
assert_eq!(tokenizer.decode(&ordinary_ids, false).unwrap(), text);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,6 +208,10 @@ impl Tokenizer for TestTokenizer {
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
|
||||
Ok(text.as_bytes().iter().copied().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
let mut output = String::new();
|
||||
let mut pending_bytes = Vec::new();
|
||||
@@ -374,6 +378,32 @@ mod tests {
|
||||
assert!(!tokenizer.is_special_id(0xF002));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_encoding_bypasses_all_configured_tokens() {
|
||||
let tokenizer = TestTokenizer::new()
|
||||
.with_bos_token("<bos>", 256)
|
||||
.with_special_token("<control>", 257)
|
||||
.with_regular_token("<visible>", 258);
|
||||
let ordinary_text = "user <control> and <visible>";
|
||||
|
||||
assert_eq!(tokenizer.encode("<control>", false).unwrap(), vec![257]);
|
||||
assert_eq!(tokenizer.encode("<visible>", false).unwrap(), vec![258]);
|
||||
assert_eq!(
|
||||
tokenizer.encode_ordinary(ordinary_text).unwrap(),
|
||||
ordinary_text.as_bytes().iter().copied().map(u32::from).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
let mut segmented = tokenizer.encode("<control>", false).unwrap();
|
||||
segmented.extend(tokenizer.encode_ordinary(ordinary_text).unwrap());
|
||||
segmented.extend(tokenizer.encode("<visible>", false).unwrap());
|
||||
assert_eq!(segmented.first(), Some(&257));
|
||||
assert_eq!(segmented.last(), Some(&258));
|
||||
assert_eq!(
|
||||
tokenizer.decode(&segmented, false).unwrap(),
|
||||
format!("<control>{ordinary_text}<visible>")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "configured test token id 255 overlaps byte fallback range 0..=255")]
|
||||
fn configured_token_id_must_stay_outside_byte_range() {
|
||||
|
||||
@@ -462,6 +462,13 @@ impl Tokenizer for TiktokenTokenizer {
|
||||
})
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
|
||||
Ok(match &self.backend {
|
||||
Backend::Riptoken(backend) => backend.inner.encode_ordinary(text),
|
||||
Backend::TiktokenRs(backend) => backend.inner.encode_ordinary(text),
|
||||
})
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
// Filter passes:
|
||||
//
|
||||
@@ -752,6 +759,39 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiktoken_ordinary_bypasses_every_registered_added_token() {
|
||||
let dir = tempfile::tempdir().expect("create temp dir");
|
||||
let bpe_path = write_synthetic_bpe_file(dir.path());
|
||||
fs::write(
|
||||
dir.path().join("tokenizer_config.json"),
|
||||
r#"{
|
||||
"added_tokens_decoder": {
|
||||
"257": { "content": "<|im_end|>", "special": true },
|
||||
"258": { "content": "<|tool_call_begin|>", "special": false }
|
||||
}
|
||||
}"#,
|
||||
)
|
||||
.expect("write tokenizer_config.json");
|
||||
fs::write(dir.path().join("config.json"), r#"{"vocab_size": 260}"#)
|
||||
.expect("write config.json");
|
||||
|
||||
let input = "<|im_end|><|tool_call_begin|><|reserved_token_259|>";
|
||||
let expected: Vec<u32> = input.as_bytes().iter().copied().map(u32::from).collect();
|
||||
for backend in explicit_backends(&bpe_path) {
|
||||
assert_eq!(backend.encode("<|im_end|>", false).unwrap(), vec![257]);
|
||||
assert_eq!(
|
||||
backend.encode("<|tool_call_begin|>", false).unwrap(),
|
||||
vec![258]
|
||||
);
|
||||
assert_eq!(
|
||||
backend.encode("<|reserved_token_259|>", false).unwrap(),
|
||||
vec![259]
|
||||
);
|
||||
assert_eq!(backend.encode_ordinary(input).unwrap(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
/// `vocab_size` may live under `text_config` for composite (e.g.
|
||||
/// multimodal) configs.
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user