[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,35 @@
|
||||
[package]
|
||||
name = "vllm-tokenizer"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
fastokens.workspace = true
|
||||
riptoken.workspace = true
|
||||
rustc-hash.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tekken.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
tiktoken-rs.workspace = true
|
||||
tokenizers.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
hf-hub.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[[bench]]
|
||||
name = "hf"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "tiktoken"
|
||||
harness = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,118 @@
|
||||
use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use hf_hub::api::sync::ApiBuilder;
|
||||
use vllm_tokenizer::{HuggingFaceTokenizer, Tokenizer};
|
||||
|
||||
const MODEL_ID: &str = "Qwen/Qwen3.5-0.8B";
|
||||
const SAMPLE_TEXT: &str = "\
|
||||
<|im_start|>system
|
||||
You are Qwen3.5, a helpful assistant.
|
||||
<|im_end|>
|
||||
<|im_start|>user
|
||||
请用中英混合总结以下需求,并给出一个简短的 JSON 示例。
|
||||
The service should stop cleanly at EOS, avoid leaking the next template turn, and keep decode latency low.
|
||||
Input: 4 concurrent requests, 10240 prompt tokens, 16 generated tokens.
|
||||
<|im_end|>
|
||||
<|im_start|>assistant
|
||||
";
|
||||
|
||||
struct BenchFixture {
|
||||
fastokens: HuggingFaceTokenizer,
|
||||
hf: HuggingFaceTokenizer,
|
||||
text: String,
|
||||
token_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
impl BenchFixture {
|
||||
fn load() -> Self {
|
||||
let path = tokenizer_json();
|
||||
let fastokens =
|
||||
HuggingFaceTokenizer::new_fastokens(&path).expect("load fastokens tokenizer");
|
||||
let hf = HuggingFaceTokenizer::new_hf(&path).expect("load huggingface tokenizer");
|
||||
|
||||
let text = SAMPLE_TEXT.repeat(32);
|
||||
let hf_token_ids =
|
||||
hf.encode(text.as_str(), false).expect("encode sample text with hf tokenizer");
|
||||
let fastokens_token_ids = fastokens
|
||||
.encode(text.as_str(), false)
|
||||
.expect("encode sample text with fastokens");
|
||||
assert_eq!(fastokens_token_ids, hf_token_ids);
|
||||
|
||||
let hf_decoded = hf
|
||||
.decode(hf_token_ids.as_slice(), false)
|
||||
.expect("decode sample token ids with hf tokenizer");
|
||||
let fastokens_decoded = fastokens
|
||||
.decode(hf_token_ids.as_slice(), false)
|
||||
.expect("decode sample token ids with fastokens");
|
||||
assert_eq!(fastokens_decoded, hf_decoded);
|
||||
|
||||
Self {
|
||||
fastokens,
|
||||
hf,
|
||||
text,
|
||||
token_ids: hf_token_ids,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenizer_json() -> std::path::PathBuf {
|
||||
ApiBuilder::from_env()
|
||||
.with_progress(false)
|
||||
.build()
|
||||
.expect("build hf-hub api")
|
||||
.model(MODEL_ID.to_string())
|
||||
.get("tokenizer.json")
|
||||
.expect("fetch tokenizer.json from hf-hub")
|
||||
}
|
||||
|
||||
fn bench_encode(c: &mut Criterion) {
|
||||
let fixture = BenchFixture::load();
|
||||
let mut group = c.benchmark_group("tokenizer_encode");
|
||||
group.throughput(Throughput::Bytes(fixture.text.len() as u64));
|
||||
|
||||
group.bench_function("fastokens", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.fastokens
|
||||
.encode(black_box(fixture.text.as_str()), black_box(false))
|
||||
.expect("encode sample text with fastokens")
|
||||
})
|
||||
});
|
||||
group.bench_function("hf_tokenizers", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.hf
|
||||
.encode(black_box(fixture.text.as_str()), black_box(false))
|
||||
.expect("encode sample text with hf tokenizer")
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_decode(c: &mut Criterion) {
|
||||
let fixture = BenchFixture::load();
|
||||
let mut group = c.benchmark_group("tokenizer_decode");
|
||||
group.throughput(Throughput::Elements(fixture.token_ids.len() as u64));
|
||||
|
||||
group.bench_function("fastokens", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.fastokens
|
||||
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
|
||||
.expect("decode sample token ids with fastokens")
|
||||
})
|
||||
});
|
||||
group.bench_function("hf_tokenizers", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.hf
|
||||
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
|
||||
.expect("decode sample token ids with hf tokenizer")
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_encode, bench_decode);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,117 @@
|
||||
use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use hf_hub::api::sync::ApiBuilder;
|
||||
use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
|
||||
|
||||
const MODEL_ID: &str = "moonshotai/Kimi-K2.5";
|
||||
const SAMPLE_TEXT: &str = "\
|
||||
<think>
|
||||
I'm sure it's fine, but I can't say I'd trust that it's what we'd ship.
|
||||
</think>
|
||||
请用中英混合总结以下需求,并保留 tool-call marker:
|
||||
<|tool_calls_section_begin|>{\"name\":\"summarize\",\"arguments\":{\"style\":\"brief\"}}<|tool_calls_section_end|>
|
||||
The service should stop cleanly at EOS, avoid leaking the next template turn, and keep decode latency low.
|
||||
";
|
||||
|
||||
struct BenchFixture {
|
||||
riptoken: TiktokenTokenizer,
|
||||
tiktoken_rs: TiktokenTokenizer,
|
||||
text: String,
|
||||
token_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
impl BenchFixture {
|
||||
fn load() -> Self {
|
||||
let path = tiktoken_model();
|
||||
let riptoken = TiktokenTokenizer::new_riptoken(&path).expect("load riptoken tokenizer");
|
||||
let tiktoken_rs =
|
||||
TiktokenTokenizer::new_tiktoken_rs(&path).expect("load tiktoken-rs tokenizer");
|
||||
|
||||
let text = SAMPLE_TEXT.repeat(32);
|
||||
let riptoken_token_ids =
|
||||
riptoken.encode(text.as_str(), false).expect("encode sample text with riptoken");
|
||||
let tiktoken_rs_token_ids = tiktoken_rs
|
||||
.encode(text.as_str(), false)
|
||||
.expect("encode sample text with tiktoken-rs");
|
||||
assert_eq!(riptoken_token_ids, tiktoken_rs_token_ids);
|
||||
|
||||
let riptoken_decoded = riptoken
|
||||
.decode(riptoken_token_ids.as_slice(), false)
|
||||
.expect("decode sample token ids with riptoken");
|
||||
let tiktoken_rs_decoded = tiktoken_rs
|
||||
.decode(riptoken_token_ids.as_slice(), false)
|
||||
.expect("decode sample token ids with tiktoken-rs");
|
||||
assert_eq!(riptoken_decoded, tiktoken_rs_decoded);
|
||||
|
||||
Self {
|
||||
riptoken,
|
||||
tiktoken_rs,
|
||||
text,
|
||||
token_ids: riptoken_token_ids,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tiktoken_model() -> std::path::PathBuf {
|
||||
let repo = ApiBuilder::from_env()
|
||||
.with_progress(false)
|
||||
.build()
|
||||
.expect("build hf-hub api")
|
||||
.model(MODEL_ID.to_string());
|
||||
repo.get("config.json").expect("fetch config.json from hf-hub");
|
||||
repo.get("tokenizer_config.json")
|
||||
.expect("fetch tokenizer_config.json from hf-hub");
|
||||
repo.get("tiktoken.model").expect("fetch tiktoken.model from hf-hub")
|
||||
}
|
||||
|
||||
fn bench_encode(c: &mut Criterion) {
|
||||
let fixture = BenchFixture::load();
|
||||
let mut group = c.benchmark_group("tiktoken_encode");
|
||||
group.throughput(Throughput::Bytes(fixture.text.len() as u64));
|
||||
|
||||
group.bench_function("riptoken", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.riptoken
|
||||
.encode(black_box(fixture.text.as_str()), black_box(false))
|
||||
.expect("encode sample text with riptoken")
|
||||
})
|
||||
});
|
||||
group.bench_function("tiktoken_rs", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.tiktoken_rs
|
||||
.encode(black_box(fixture.text.as_str()), black_box(false))
|
||||
.expect("encode sample text with tiktoken-rs")
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_decode(c: &mut Criterion) {
|
||||
let fixture = BenchFixture::load();
|
||||
let mut group = c.benchmark_group("tiktoken_decode");
|
||||
group.throughput(Throughput::Elements(fixture.token_ids.len() as u64));
|
||||
|
||||
group.bench_function("riptoken", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.riptoken
|
||||
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
|
||||
.expect("decode sample token ids with riptoken")
|
||||
})
|
||||
});
|
||||
group.bench_function("tiktoken_rs", |b| {
|
||||
b.iter(|| {
|
||||
fixture
|
||||
.tiktoken_rs
|
||||
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
|
||||
.expect("decode sample token ids with tiktoken-rs")
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_encode, bench_decode);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Fast GPT-2 byte-level detokenization that writes into a single `Vec<u8>`,
|
||||
//! avoiding the `Vec<String>` / `String::join` assembly in fastokens' generic
|
||||
//! `Decoder::decode` pipeline.
|
||||
|
||||
/// Reverse GPT-2 byte-to-unicode mapping: codepoint → original byte. The GPT-2
|
||||
/// table only emits codepoints in U+0000..U+0143, so a flat array suffices.
|
||||
const CHAR_TO_BYTE: [u8; 324] = build_char_to_byte();
|
||||
|
||||
const fn is_nice(b: u8) -> bool {
|
||||
(b >= b'!' && b <= b'~') || (b >= 0xA1 && b <= 0xAC) || b >= 0xAE
|
||||
}
|
||||
|
||||
const fn build_char_to_byte() -> [u8; 324] {
|
||||
let mut table = [0u8; 324];
|
||||
let mut b: u16 = 0;
|
||||
while b < 256 {
|
||||
let cp = if is_nice(b as u8) {
|
||||
b as u32
|
||||
} else {
|
||||
256 + nice_offset(b as u8)
|
||||
};
|
||||
table[cp as usize] = b as u8;
|
||||
b += 1;
|
||||
}
|
||||
table
|
||||
}
|
||||
|
||||
const fn nice_offset(b: u8) -> u32 {
|
||||
let mut i: u16 = 0;
|
||||
let mut n: u32 = 0;
|
||||
while i < b as u16 {
|
||||
if !is_nice(i as u8) {
|
||||
n += 1;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
/// Decode byte-level encoded token strings into a single UTF-8 string,
|
||||
/// matching `fastokens::decoders::ByteLevelDecoder`.
|
||||
pub fn decode_byte_level<'a, I: IntoIterator<Item = &'a str>>(tokens: I) -> String {
|
||||
let iter = tokens.into_iter();
|
||||
let (lower, _) = iter.size_hint();
|
||||
let mut bytes: Vec<u8> = Vec::with_capacity(lower.saturating_mul(4));
|
||||
for token in iter {
|
||||
for c in token.chars() {
|
||||
let cp = c as usize;
|
||||
if cp < CHAR_TO_BYTE.len() {
|
||||
bytes.push(CHAR_TO_BYTE[cp]);
|
||||
} else {
|
||||
// Non-GPT2 codepoints (e.g. DeepSeek's U+FF5C, U+2581) pass through.
|
||||
let mut buf = [0u8; 4];
|
||||
bytes.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn build_byte_to_char_ref() -> [char; 256] {
|
||||
let mut table = ['\0'; 256];
|
||||
let mut next: u32 = 256;
|
||||
for b in 0..=255u8 {
|
||||
let cp = if is_nice(b) {
|
||||
b as u32
|
||||
} else {
|
||||
let cp = next;
|
||||
next += 1;
|
||||
cp
|
||||
};
|
||||
table[b as usize] = char::from_u32(cp).unwrap();
|
||||
}
|
||||
table
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_to_byte_roundtrips_every_byte() {
|
||||
let byte_to_char = build_byte_to_char_ref();
|
||||
for b in 0..=255u8 {
|
||||
let cp = byte_to_char[b as usize] as usize;
|
||||
assert!(cp < CHAR_TO_BYTE.len());
|
||||
assert_eq!(CHAR_TO_BYTE[cp], b, "mismatch for byte {b:#x}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_ascii() {
|
||||
assert_eq!(decode_byte_level(["Hello"]), "Hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_space_marker() {
|
||||
// GPT-2 maps 0x20 → Ġ (U+0120).
|
||||
assert_eq!(
|
||||
decode_byte_level(["\u{120}Hello", "\u{120}world"]),
|
||||
" Hello world",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_multibyte_euro() {
|
||||
// € → 0xE2 0x82 0xAC, each mapped to a specific GPT-2 char.
|
||||
let byte_to_char = build_byte_to_char_ref();
|
||||
let encoded: String =
|
||||
[0xE2u8, 0x82, 0xAC].iter().map(|&b| byte_to_char[b as usize]).collect();
|
||||
assert_eq!(decode_byte_level([encoded.as_str()]), "€");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_preserves_non_gpt2_chars() {
|
||||
let tok = "<\u{FF5C}begin\u{2581}of\u{2581}sentence\u{FF5C}>";
|
||||
assert_eq!(decode_byte_level([tok]), "<|begin▁of▁sentence|>");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
|
||||
pub type Result<T> = std::result::Result<T, TokenizerError>;
|
||||
|
||||
#[derive(Debug, Error, Macro)]
|
||||
#[thiserror_ext(macro(path = "crate::error"))]
|
||||
#[error("tokenizer error: {0}")]
|
||||
pub struct TokenizerError(#[message] pub String);
|
||||
@@ -0,0 +1,344 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use fastokens::Tokenizer as FastokensTokenizer;
|
||||
use fastokens::decoders::Decoder as FastokensDecoder;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokenizers::Tokenizer as HfTokenizer;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::byte_level_decode::decode_byte_level;
|
||||
use crate::{Result, Tokenizer};
|
||||
|
||||
enum Backend {
|
||||
Hf(Box<HfTokenizer>),
|
||||
Fastokens(Box<FastokensTokenizer>),
|
||||
/// Fastokens tokenizer whose decoder is pure GPT-2 byte-level, so we can
|
||||
/// bypass `Decoder::decode`'s `Vec<String>`/`join("")` assembly.
|
||||
FastokensByteLevel(Box<FastokensTokenizer>),
|
||||
}
|
||||
|
||||
/// True if `dec` is effectively a single `ByteLevel` stage — one `ByteLevel`
|
||||
/// leaf in a tree of `Sequence`s (fastokens represents `Fuse` as an empty
|
||||
/// `Sequence`, which is a no-op for our purposes).
|
||||
fn is_byte_level_only(dec: &FastokensDecoder) -> bool {
|
||||
fn count_byte_level(dec: &FastokensDecoder) -> usize {
|
||||
match dec {
|
||||
FastokensDecoder::ByteLevel(_) => 1,
|
||||
FastokensDecoder::Sequence(steps) => steps.iter().map(count_byte_level).sum(),
|
||||
}
|
||||
}
|
||||
count_byte_level(dec) == 1
|
||||
}
|
||||
|
||||
fn decode_fastokens_byte_level(
|
||||
t: &FastokensTokenizer,
|
||||
token_ids: &[u32],
|
||||
skip_special_tokens: bool,
|
||||
) -> Result<String> {
|
||||
let tokens: Vec<&str> = token_ids
|
||||
.iter()
|
||||
.filter(|&&id| !(skip_special_tokens && t.is_special_token(id)))
|
||||
.map(|&id| {
|
||||
t.id_to_token(id)
|
||||
.ok_or_else(|| tokenizer_error!("decoding failed: unknown token ID: {id}"))
|
||||
})
|
||||
.collect::<Result<_>>()?;
|
||||
Ok(decode_byte_level(tokens))
|
||||
}
|
||||
|
||||
/// Tokenizer from `tokenizer.json` in HuggingFace format.
|
||||
///
|
||||
/// This tries to load with `fastokens` first for better performance, then falls
|
||||
/// back to HuggingFace's `tokenizers` if the former fails (e.g. due to
|
||||
/// unsupported tokenizer features or file formats).
|
||||
pub struct HuggingFaceTokenizer {
|
||||
backend: Backend,
|
||||
special_token_ids: Arc<[u32]>,
|
||||
}
|
||||
|
||||
impl HuggingFaceTokenizer {
|
||||
fn from_hf_backend(tokenizer: HfTokenizer) -> Self {
|
||||
let special_token_ids = {
|
||||
let mut ids: Vec<u32> = tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.iter()
|
||||
.filter(|(_id, token)| token.special)
|
||||
.map(|(id, _token)| *id)
|
||||
.collect();
|
||||
ids.sort_unstable();
|
||||
ids.dedup();
|
||||
Arc::from(ids)
|
||||
};
|
||||
Self {
|
||||
backend: Backend::Hf(Box::new(tokenizer)),
|
||||
special_token_ids,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_fastokens_backend(tokenizer: FastokensTokenizer) -> Self {
|
||||
let special_token_ids = {
|
||||
let mut ids: Vec<u32> = tokenizer
|
||||
.added_tokens()
|
||||
.into_iter()
|
||||
.flat_map(|added_tokens| added_tokens.iter())
|
||||
.filter(|token| token.special)
|
||||
.map(|token| token.id)
|
||||
.collect();
|
||||
ids.sort_unstable();
|
||||
ids.dedup();
|
||||
Arc::from(ids)
|
||||
};
|
||||
let byte_level = tokenizer.decoder().is_some_and(is_byte_level_only);
|
||||
let backend = if byte_level {
|
||||
Backend::FastokensByteLevel(Box::new(tokenizer))
|
||||
} else {
|
||||
Backend::Fastokens(Box::new(tokenizer))
|
||||
};
|
||||
Self {
|
||||
backend,
|
||||
special_token_ids,
|
||||
}
|
||||
}
|
||||
|
||||
/// Load from `tokenizer.json` with `fastokens`.
|
||||
pub fn new_fastokens(path: &Path) -> Result<Self> {
|
||||
info!(path = %path.display(), "loading tokenizer with fastokens");
|
||||
let t = FastokensTokenizer::from_file(path)
|
||||
.map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
|
||||
Ok(Self::from_fastokens_backend(t))
|
||||
}
|
||||
|
||||
/// Load from `tokenizer.json` with Hugging Face `tokenizers`.
|
||||
pub fn new_hf(path: &Path) -> Result<Self> {
|
||||
info!(path = %path.display(), "loading tokenizer with huggingface tokenizers");
|
||||
let t = HfTokenizer::from_file(path)
|
||||
.map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
|
||||
Ok(Self::from_hf_backend(t))
|
||||
}
|
||||
|
||||
/// Load from `tokenizer.json` via fastokens or HuggingFace tokenizers.
|
||||
pub fn new(path: &Path) -> Result<Self> {
|
||||
match Self::new_fastokens(path) {
|
||||
Ok(tokenizer) => Ok(tokenizer),
|
||||
Err(error) => {
|
||||
warn!(
|
||||
path = %path.display(),
|
||||
error = %error.as_report(),
|
||||
"failed to load tokenizer with fastokens; falling back to HuggingFace tokenizers"
|
||||
);
|
||||
Self::new_hf(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Tokenizer for HuggingFaceTokenizer {
|
||||
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
match &self.backend {
|
||||
Backend::Hf(t) => {
|
||||
let encoding = t
|
||||
.encode(text, add_special_tokens)
|
||||
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report()))?;
|
||||
Ok(encoding.get_ids().to_vec())
|
||||
}
|
||||
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => t
|
||||
.encode_with_special_tokens(text, add_special_tokens)
|
||||
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report())),
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
match &self.backend {
|
||||
Backend::Hf(t) => t
|
||||
.decode(token_ids, skip_special_tokens)
|
||||
.map_err(|error| tokenizer_error!("decoding failed: {}", error.as_report())),
|
||||
Backend::Fastokens(t) => t
|
||||
.decode(token_ids, skip_special_tokens)
|
||||
.map_err(|error| tokenizer_error!("decoding failed: {}", error.as_report())),
|
||||
Backend::FastokensByteLevel(t) => {
|
||||
decode_fastokens_byte_level(t, token_ids, skip_special_tokens)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match &self.backend {
|
||||
Backend::Hf(t) => t.token_to_id(token),
|
||||
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => t.token_to_id(token),
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
match &self.backend {
|
||||
Backend::Hf(t) => t.id_to_token(id),
|
||||
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => {
|
||||
t.id_to_token(id).map(ToOwned::to_owned)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
self.special_token_ids.binary_search(&token_id).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use tempfile::tempdir;
|
||||
use tokenizers::models::bpe::BPE;
|
||||
use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
|
||||
|
||||
use super::{HuggingFaceTokenizer, Tokenizer};
|
||||
|
||||
fn tiny_bpe_tokenizer() -> HfTokenizer {
|
||||
let vocab = [
|
||||
("<unk>".to_string(), 0),
|
||||
("h".to_string(), 1),
|
||||
("e".to_string(), 2),
|
||||
("l".to_string(), 3),
|
||||
("o".to_string(), 4),
|
||||
("he".to_string(), 5),
|
||||
("ll".to_string(), 6),
|
||||
("hell".to_string(), 7),
|
||||
("hello".to_string(), 8),
|
||||
];
|
||||
let merges = vec![
|
||||
("h".to_string(), "e".to_string()),
|
||||
("l".to_string(), "l".to_string()),
|
||||
("he".to_string(), "ll".to_string()),
|
||||
("hell".to_string(), "o".to_string()),
|
||||
];
|
||||
let model = BPE::builder()
|
||||
.vocab_and_merges(vocab, merges)
|
||||
.unk_token("<unk>".to_string())
|
||||
.build()
|
||||
.expect("build bpe tokenizer");
|
||||
HfTokenizer::new(model)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_constructor_resolves_added_token_ids() {
|
||||
let mut tokenizer = tiny_bpe_tokenizer();
|
||||
tokenizer.add_special_tokens(&[AddedToken::from("<|im_end|>", true)]);
|
||||
|
||||
let dir = tempdir().expect("create temp dir");
|
||||
let path = dir.path().join("tokenizer.json");
|
||||
tokenizer.save(&path, false).expect("save tokenizer json");
|
||||
|
||||
let wrapper = HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper");
|
||||
let special_id = wrapper.token_to_id("<|im_end|>").expect("resolve added special token id");
|
||||
assert!(wrapper.is_special_id(special_id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_fastokens_preserves_special_ids_from_fastokens_metadata() {
|
||||
let mut tokenizer = tiny_bpe_tokenizer();
|
||||
tokenizer.add_special_tokens(&[AddedToken::from("<|im_end|>", true)]);
|
||||
|
||||
let dir = tempdir().expect("create temp dir");
|
||||
let path = dir.path().join("tokenizer.json");
|
||||
tokenizer.save(&path, false).expect("save tokenizer json");
|
||||
|
||||
let wrapper = HuggingFaceTokenizer::new_fastokens(&path)
|
||||
.expect("load wrapper with fastokens backend");
|
||||
assert!(matches!(
|
||||
wrapper.backend,
|
||||
super::Backend::Fastokens(_) | super::Backend::FastokensByteLevel(_),
|
||||
));
|
||||
let special_id = wrapper.token_to_id("<|im_end|>").expect("resolve added special token id");
|
||||
assert!(wrapper.is_special_id(special_id));
|
||||
}
|
||||
|
||||
/// BPE tokenizer that round-trips through fastokens with a genuine
|
||||
/// `ByteLevel` decoder; vocab covers both GPT-2 (Ġ U+0120) and non-GPT-2
|
||||
/// (| U+FF5C) codepoints.
|
||||
fn tiny_byte_level_bpe() -> fastokens::Tokenizer {
|
||||
let raw = r#"{
|
||||
"version": "1.0",
|
||||
"truncation": null,
|
||||
"padding": null,
|
||||
"added_tokens": [
|
||||
{"id": 0, "content": "<|endoftext|>", "single_word": false,
|
||||
"lstrip": false, "rstrip": false, "normalized": false, "special": true}
|
||||
],
|
||||
"normalizer": null,
|
||||
"pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": false,
|
||||
"trim_offsets": true, "use_regex": true},
|
||||
"post_processor": null,
|
||||
"decoder": {"type": "ByteLevel", "add_prefix_space": false,
|
||||
"trim_offsets": true, "use_regex": true},
|
||||
"model": {
|
||||
"type": "BPE",
|
||||
"dropout": null,
|
||||
"unk_token": null,
|
||||
"continuing_subword_prefix": null,
|
||||
"end_of_word_suffix": null,
|
||||
"fuse_unk": false,
|
||||
"byte_fallback": false,
|
||||
"ignore_merges": false,
|
||||
"vocab": {
|
||||
"<|endoftext|>": 0,
|
||||
"H": 1, "e": 2, "l": 3, "o": 4, "w": 5, "r": 6, "d": 7,
|
||||
"Ġ": 8, "!": 9,
|
||||
"|": 10
|
||||
},
|
||||
"merges": []
|
||||
}
|
||||
}"#;
|
||||
let value: serde_json::Value = serde_json::from_str(raw).expect("parse tokenizer json");
|
||||
fastokens::Tokenizer::from_json(value).expect("build fastokens tokenizer")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_level_detected_direct() {
|
||||
let t = tiny_byte_level_bpe();
|
||||
assert!(super::is_byte_level_only(t.decoder().expect("decoder")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_level_detected_inside_sequence() {
|
||||
let raw = r#"{
|
||||
"type": "Sequence",
|
||||
"decoders": [
|
||||
{"type": "ByteLevel", "add_prefix_space": false,
|
||||
"trim_offsets": true, "use_regex": true},
|
||||
{"type": "Fuse"}
|
||||
]
|
||||
}"#;
|
||||
let config: fastokens::DecoderConfig =
|
||||
serde_json::from_str(raw).expect("parse decoder config");
|
||||
let dec =
|
||||
fastokens::decoders::Decoder::from_config(config).expect("build decoder from config");
|
||||
assert!(super::is_byte_level_only(&dec));
|
||||
}
|
||||
|
||||
/// Fast path must produce byte-identical output to fastokens' own decode.
|
||||
#[test]
|
||||
fn fast_byte_level_matches_fastokens_decode() {
|
||||
let t = tiny_byte_level_bpe();
|
||||
let cases: &[&[u32]] = &[
|
||||
&[],
|
||||
&[1, 2, 3, 3, 4], // "Hello"
|
||||
&[1, 2, 3, 3, 4, 8, 5, 4, 6, 3, 7], // "Hello world"
|
||||
&[0, 1, 2, 3, 3, 4, 0, 9, 0], // specials interleaved
|
||||
&[10, 1, 2, 3, 3, 4, 10], // |Hello| (non-GPT2 chars)
|
||||
];
|
||||
for ids in cases {
|
||||
for &skip in &[false, true] {
|
||||
let expected = t.decode(ids, skip).expect("fastokens decode");
|
||||
let got =
|
||||
super::decode_fastokens_byte_level(&t, ids, skip).expect("fast-path decode");
|
||||
assert_eq!(got, expected, "ids={ids:?} skip={skip}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fast_byte_level_errors_on_unknown_id() {
|
||||
let t = tiny_byte_level_bpe();
|
||||
let err = super::decode_fastokens_byte_level(&t, &[999], false)
|
||||
.expect_err("unknown id must error");
|
||||
assert!(format!("{err:?}").contains("999"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
use std::mem::take;
|
||||
|
||||
use crate::{Result, Tokenizer};
|
||||
|
||||
/// Stateful incremental decoder that emits text chunks one token at a time.
|
||||
pub trait IncrementalDecoder: Send {
|
||||
/// Push one generated token and return how many new string bytes were
|
||||
/// added.
|
||||
fn push_token(&mut self, token_id: u32) -> Result<usize>;
|
||||
|
||||
/// Consume any text which is currently ready.
|
||||
fn next_chunk(&mut self) -> Option<String>;
|
||||
|
||||
/// Flush any remaining buffered text that has not yet been emitted.
|
||||
///
|
||||
/// Called after the final generated token to force out buffered/incomplete
|
||||
/// fragments.
|
||||
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)>;
|
||||
|
||||
/// Return cumulative decoded text so far.
|
||||
fn output(&self) -> &str;
|
||||
}
|
||||
|
||||
/// [`IncrementalDecoder`] built on [`Tokenizer::decode()`] with prefix-diffing.
|
||||
///
|
||||
/// This is the same sliding-window algorithm used by `tokenizers::DecodeStream`
|
||||
pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> {
|
||||
tokenizer: &'a T,
|
||||
skip_special_tokens: bool,
|
||||
min_bytes_to_buffer: usize,
|
||||
// mutated state
|
||||
ids: Vec<u32>,
|
||||
prefix: String,
|
||||
prefix_index: usize,
|
||||
cumulative_output: String,
|
||||
output_index: usize,
|
||||
}
|
||||
|
||||
impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
|
||||
pub(crate) fn new(
|
||||
tokenizer: &'a T,
|
||||
prompt_token_ids: &[u32],
|
||||
skip_special_tokens: bool,
|
||||
min_bytes_to_buffer: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
tokenizer,
|
||||
skip_special_tokens,
|
||||
min_bytes_to_buffer,
|
||||
ids: prompt_token_ids.to_vec(),
|
||||
prefix: String::new(),
|
||||
prefix_index: 0,
|
||||
cumulative_output: String::new(),
|
||||
output_index: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try a short tail suffix first (covers a CJK glyph straddling 1-2 token
|
||||
/// boundaries); beyond 6 tokens the fallback full-prompt decode is no worse
|
||||
/// than baseline so widening the sweep just adds overhead.
|
||||
const SAFE_SUFFIX_MIN: usize = 4;
|
||||
const SAFE_SUFFIX_MAX: usize = 6;
|
||||
|
||||
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
|
||||
/// Seed `self.prefix` from the shortest trailing suffix whose decoded text
|
||||
/// has no U+FFFD — a clean decode means the suffix starts and ends at
|
||||
/// valid UTF-8/token boundaries, so priming from it is equivalent to
|
||||
/// priming from the full prompt.
|
||||
fn seed_prefix(&mut self) -> Result<()> {
|
||||
let prompt_len = self.ids.len();
|
||||
if prompt_len > SAFE_SUFFIX_MIN {
|
||||
let max_try = SAFE_SUFFIX_MAX.min(prompt_len - 1);
|
||||
for suffix_len in SAFE_SUFFIX_MIN..=max_try {
|
||||
let start = prompt_len - suffix_len;
|
||||
let decoded =
|
||||
self.tokenizer.decode(&self.ids[start..], self.skip_special_tokens)?;
|
||||
if !decoded.contains('\u{FFFD}') {
|
||||
self.prefix = decoded;
|
||||
self.ids.drain(..start);
|
||||
self.prefix_index = self.ids.len();
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
let decoded = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
if !decoded.ends_with('\u{FFFD}') {
|
||||
self.prefix = decoded;
|
||||
self.prefix_index = self.ids.len();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
|
||||
fn push_token(&mut self, token_id: u32) -> Result<usize> {
|
||||
if self.prefix.is_empty() && !self.ids.is_empty() {
|
||||
self.seed_prefix()?;
|
||||
}
|
||||
|
||||
self.ids.push(token_id);
|
||||
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
let prefix_len = self.prefix.len();
|
||||
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
|
||||
return Ok(0);
|
||||
}
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
|
||||
self.cumulative_output.push_str(new_chunk);
|
||||
self.ids.drain(..self.prefix_index);
|
||||
self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
self.prefix_index = self.ids.len();
|
||||
Ok(new_chunk.len())
|
||||
}
|
||||
|
||||
fn next_chunk(&mut self) -> Option<String> {
|
||||
let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer);
|
||||
(cutoff > self.output_index).then(|| {
|
||||
let chunk = self.cumulative_output[self.output_index..cutoff].to_string();
|
||||
self.output_index = cutoff;
|
||||
chunk
|
||||
})
|
||||
}
|
||||
|
||||
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)> {
|
||||
if !self.ids.is_empty() {
|
||||
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
|
||||
let prefix_len = self.prefix.len();
|
||||
self.ids.clear();
|
||||
self.prefix.clear();
|
||||
self.prefix_index = 0;
|
||||
// Ensure we split at a utf-8 char boundary.
|
||||
self.cumulative_output
|
||||
.push_str(&string[string.floor_char_boundary(prefix_len)..]);
|
||||
}
|
||||
if let Some(truncate_output_to) = truncate_output_to {
|
||||
self.cumulative_output.truncate(truncate_output_to);
|
||||
}
|
||||
let last_chunk = (self.output_index < self.cumulative_output.len())
|
||||
.then(|| self.cumulative_output[self.output_index..].to_string());
|
||||
self.output_index = 0;
|
||||
Ok((last_chunk, take(&mut self.cumulative_output)))
|
||||
}
|
||||
|
||||
fn output(&self) -> &str {
|
||||
&self.cumulative_output
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
|
||||
#[derive(Debug)]
|
||||
struct Utf8Backend;
|
||||
|
||||
impl Tokenizer for Utf8Backend {
|
||||
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
|
||||
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn holds_incomplete_utf8_until_complete() {
|
||||
let backend = Utf8Backend;
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
// 你 = U+4F60 = 0xE4 0xBD 0xA0
|
||||
assert_eq!(decoder.push_token(0xe4).unwrap(), 0);
|
||||
assert_eq!(decoder.push_token(0xbd).unwrap(), 0);
|
||||
assert_eq!(decoder.push_token(0xa0).unwrap(), 3); // "你" is 3 bytes
|
||||
assert_eq!(decoder.output(), "你");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_ascii_immediately() {
|
||||
let backend = Utf8Backend;
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.output(), "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_returns_none_when_fully_consumed() {
|
||||
let backend = Utf8Backend;
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.next_chunk().as_deref(), Some("o"));
|
||||
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
|
||||
assert_eq!(decoder.next_chunk().as_deref(), Some("k"));
|
||||
// All text already consumed via next_chunk
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None);
|
||||
assert_eq!(full_text, "ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_emits_buffered_incomplete_utf8() {
|
||||
let backend = Utf8Backend;
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
// Push incomplete multi-byte sequence — step returns 0 bytes.
|
||||
assert_eq!(decoder.push_token(0xe4).unwrap(), 0);
|
||||
assert_eq!(decoder.push_token(0xbd).unwrap(), 0);
|
||||
|
||||
// Flush forces out whatever the decoder can produce (lossy replacement).
|
||||
let (last_chunk, _full_text) = decoder.flush(None).unwrap();
|
||||
assert!(last_chunk.is_some());
|
||||
}
|
||||
|
||||
/// Backend where token 0 is a special token.
|
||||
#[derive(Debug)]
|
||||
struct SpecialTokenBackend;
|
||||
|
||||
impl Tokenizer for SpecialTokenBackend {
|
||||
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
let mut text = String::new();
|
||||
for &token_id in token_ids {
|
||||
match token_id {
|
||||
0 if !skip_special_tokens => text.push_str("<special>"),
|
||||
0 => {}
|
||||
1 => text.push('a'),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn respects_skip_special_tokens() {
|
||||
let backend = SpecialTokenBackend;
|
||||
let mut skip_decoder = backend.create_decode_stream(&[], true, 0);
|
||||
let mut keep_decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
assert_eq!(skip_decoder.push_token(0).unwrap(), 0);
|
||||
assert_eq!(keep_decoder.push_token(0).unwrap(), 9); // "<special>" is 9 bytes
|
||||
assert_eq!(keep_decoder.output(), "<special>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_tokens_provide_context_without_re_emission() {
|
||||
let backend = Utf8Backend;
|
||||
let prompt = &[b'H' as u32, b'i' as u32];
|
||||
let mut decoder = backend.create_decode_stream(prompt, false, 0);
|
||||
|
||||
// First generated token should not re-emit "Hi".
|
||||
let added = decoder.push_token(b'!' as u32).unwrap();
|
||||
assert_eq!(added, 1);
|
||||
assert_eq!(decoder.output(), "!");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunks_concatenate_to_full_text() {
|
||||
let backend = Utf8Backend;
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
let input = b"Hello, world!";
|
||||
let mut full = String::new();
|
||||
for &byte in input {
|
||||
decoder.push_token(byte as u32).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
full.push_str(&chunk);
|
||||
}
|
||||
}
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk, None); // all consumed via next_chunk
|
||||
assert_eq!(full, "Hello, world!");
|
||||
assert_eq!(full_text, "Hello, world!");
|
||||
}
|
||||
|
||||
/// Backend simulating non-monotonic decode where adding a token changes how
|
||||
/// earlier tokens decode (context-dependent normalization), causing
|
||||
/// prefix_len to land mid-UTF-8. Reproduces the class of bug from
|
||||
/// vllm-project/vllm#17448.
|
||||
#[derive(Debug)]
|
||||
struct NonMonotonicBackend;
|
||||
|
||||
impl Tokenizer for NonMonotonicBackend {
|
||||
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
|
||||
match token_ids {
|
||||
[1] => Ok("abc".into()),
|
||||
[1, 2] => Ok("ab".into()),
|
||||
// Token 3 triggers a normalization change: "ab" becomes emoji + "d".
|
||||
// prefix_len=3 ("abc") lands inside the 4-byte emoji 🎉.
|
||||
[1, 2, 3] => Ok("🎉d".into()), // 🎉 is 4 bytes + d = 5 bytes
|
||||
[2, 3] => Ok("🎉d".into()), // prefix recompute after drain
|
||||
[3] => Ok("d".into()), // after drain
|
||||
_ => panic!("unexpected decode: {:?}", token_ids),
|
||||
}
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
/// Without the char-boundary fix, this panics slicing mid-emoji.
|
||||
#[test]
|
||||
fn non_monotonic_decode_does_not_panic() {
|
||||
let backend = NonMonotonicBackend;
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 0);
|
||||
|
||||
// Token 1: "abc", prefix="abc"
|
||||
assert_eq!(decoder.push_token(1).unwrap(), 3);
|
||||
// Token 2: "ab" (shorter), no emit
|
||||
assert_eq!(decoder.push_token(2).unwrap(), 0);
|
||||
// Token 3: "🎉d" — prefix_len=3 is mid-emoji. Without fix this panics.
|
||||
let added = decoder.push_token(3).unwrap();
|
||||
assert!(added > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_chunk_with_hold_back() {
|
||||
let backend = Utf8Backend;
|
||||
// hold_back_bytes: 3 means we buffer the last 3 bytes
|
||||
let mut decoder = backend.create_decode_stream(&[], false, 3);
|
||||
|
||||
let input = b"Hello!";
|
||||
let mut chunks = String::new();
|
||||
for &byte in input {
|
||||
decoder.push_token(byte as u32).unwrap();
|
||||
if let Some(chunk) = decoder.next_chunk() {
|
||||
chunks.push_str(&chunk);
|
||||
}
|
||||
}
|
||||
// With hold_back_bytes=3, last 3 bytes ("lo!") are held back
|
||||
assert_eq!(chunks, "Hel");
|
||||
// Flush returns the rest
|
||||
let (last_chunk, full_text) = decoder.flush(None).unwrap();
|
||||
assert_eq!(last_chunk.as_deref(), Some("lo!"));
|
||||
assert_eq!(full_text, "Hello!");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::incremental::DecodeStream;
|
||||
|
||||
mod byte_level_decode;
|
||||
#[macro_use]
|
||||
mod error;
|
||||
mod hf;
|
||||
mod incremental;
|
||||
mod tekken;
|
||||
mod tiktoken;
|
||||
|
||||
pub use error::{Result, TokenizerError};
|
||||
pub use hf::HuggingFaceTokenizer;
|
||||
pub use incremental::IncrementalDecoder;
|
||||
pub use tekken::TekkenTokenizer;
|
||||
pub use tiktoken::TiktokenTokenizer;
|
||||
|
||||
pub trait Tokenizer: Send + Sync {
|
||||
/// Encode one prompt string into token IDs.
|
||||
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>;
|
||||
|
||||
/// Decode one token sequence into text.
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String>;
|
||||
|
||||
/// Convert one token string into a token ID, returning `None` if the token
|
||||
/// is not in the tokenizer vocabulary.
|
||||
fn token_to_id(&self, token: &str) -> Option<u32>;
|
||||
|
||||
/// Convert one token ID into the tokenizer's raw token string.
|
||||
fn id_to_token(&self, _id: u32) -> Option<String> {
|
||||
// TODO: remove default impl and require this to be implemented by all
|
||||
// tokenizers
|
||||
None
|
||||
}
|
||||
|
||||
/// Return whether the given token ID is special.
|
||||
fn is_special_id(&self, _token_id: u32) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Create a stateful incremental decoder primed with the given prompt
|
||||
/// tokens.
|
||||
///
|
||||
/// The prompt tokens provide left context for the first generated token;
|
||||
/// the decoder does not re-emit prompt text.
|
||||
fn create_decode_stream(
|
||||
&self,
|
||||
prompt_token_ids: &[u32],
|
||||
skip_special_tokens: bool,
|
||||
min_bytes_to_buffer: usize,
|
||||
) -> Box<dyn IncrementalDecoder + '_> {
|
||||
Box::new(DecodeStream::new(
|
||||
self,
|
||||
prompt_token_ids,
|
||||
skip_special_tokens,
|
||||
min_bytes_to_buffer,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub type DynTokenizer = Arc<dyn Tokenizer>;
|
||||
@@ -0,0 +1,62 @@
|
||||
use std::path::Path;
|
||||
|
||||
use tekken::Tekkenizer;
|
||||
use tracing::info;
|
||||
|
||||
use crate::{Result, Tokenizer};
|
||||
|
||||
/// Mistral Tekken tokenizer from a `tekken.json` file.
|
||||
pub struct TekkenTokenizer {
|
||||
inner: Tekkenizer,
|
||||
}
|
||||
|
||||
impl TekkenTokenizer {
|
||||
/// Load a Mistral Tekken tokenizer from a `tekken.json` file.
|
||||
pub fn new(path: &Path) -> Result<Self> {
|
||||
info!(path = %path.display(), "loading tokenizer with Mistral Tekken");
|
||||
|
||||
let inner = Tekkenizer::from_file(path).map_err(|error| {
|
||||
tokenizer_error!(
|
||||
"failed to load tekken tokenizer from {}: {error}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
}
|
||||
|
||||
impl Tokenizer for TekkenTokenizer {
|
||||
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>> {
|
||||
self.inner
|
||||
.encode(text, add_special_tokens, false)
|
||||
.map_err(|error| tokenizer_error!("encoding failed: {error}"))
|
||||
}
|
||||
|
||||
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
|
||||
let policy = if skip_special_tokens {
|
||||
tekken::SpecialTokenPolicy::Ignore
|
||||
} else {
|
||||
tekken::SpecialTokenPolicy::Keep
|
||||
};
|
||||
self.inner
|
||||
.decode(token_ids, policy)
|
||||
.map_err(|error| tokenizer_error!("decoding failed: {error}"))
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
// tekken-rs exposes `get_control_token` for special tokens. Try that first,
|
||||
// then fall back to encoding.
|
||||
self.inner.get_control_token(token).ok().or_else(|| {
|
||||
let ids = self.inner.encode(token, false, false).ok()?;
|
||||
if ids.len() == 1 { Some(ids[0]) } else { None }
|
||||
})
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
self.inner.id_to_piece(id).ok()
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
self.inner.is_special_token(token_id)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user