[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+35
View File
@@ -0,0 +1,35 @@
[package]
name = "vllm-tokenizer"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
base64.workspace = true
fastokens.workspace = true
riptoken.workspace = true
rustc-hash.workspace = true
serde.workspace = true
serde_json.workspace = true
tekken.workspace = true
thiserror.workspace = true
thiserror-ext.workspace = true
tiktoken-rs.workspace = true
tokenizers.workspace = true
tracing.workspace = true
[dev-dependencies]
criterion.workspace = true
hf-hub.workspace = true
tempfile.workspace = true
[[bench]]
name = "hf"
harness = false
[[bench]]
name = "tiktoken"
harness = false
[lints]
workspace = true
+118
View File
@@ -0,0 +1,118 @@
use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main};
use hf_hub::api::sync::ApiBuilder;
use vllm_tokenizer::{HuggingFaceTokenizer, Tokenizer};
const MODEL_ID: &str = "Qwen/Qwen3.5-0.8B";
const SAMPLE_TEXT: &str = "\
<|im_start|>system
You are Qwen3.5, a helpful assistant.
<|im_end|>
<|im_start|>user
请用中英混合总结以下需求,并给出一个简短的 JSON 示例。
The service should stop cleanly at EOS, avoid leaking the next template turn, and keep decode latency low.
Input: 4 concurrent requests, 10240 prompt tokens, 16 generated tokens.
<|im_end|>
<|im_start|>assistant
";
struct BenchFixture {
fastokens: HuggingFaceTokenizer,
hf: HuggingFaceTokenizer,
text: String,
token_ids: Vec<u32>,
}
impl BenchFixture {
fn load() -> Self {
let path = tokenizer_json();
let fastokens =
HuggingFaceTokenizer::new_fastokens(&path).expect("load fastokens tokenizer");
let hf = HuggingFaceTokenizer::new_hf(&path).expect("load huggingface tokenizer");
let text = SAMPLE_TEXT.repeat(32);
let hf_token_ids =
hf.encode(text.as_str(), false).expect("encode sample text with hf tokenizer");
let fastokens_token_ids = fastokens
.encode(text.as_str(), false)
.expect("encode sample text with fastokens");
assert_eq!(fastokens_token_ids, hf_token_ids);
let hf_decoded = hf
.decode(hf_token_ids.as_slice(), false)
.expect("decode sample token ids with hf tokenizer");
let fastokens_decoded = fastokens
.decode(hf_token_ids.as_slice(), false)
.expect("decode sample token ids with fastokens");
assert_eq!(fastokens_decoded, hf_decoded);
Self {
fastokens,
hf,
text,
token_ids: hf_token_ids,
}
}
}
fn tokenizer_json() -> std::path::PathBuf {
ApiBuilder::from_env()
.with_progress(false)
.build()
.expect("build hf-hub api")
.model(MODEL_ID.to_string())
.get("tokenizer.json")
.expect("fetch tokenizer.json from hf-hub")
}
fn bench_encode(c: &mut Criterion) {
let fixture = BenchFixture::load();
let mut group = c.benchmark_group("tokenizer_encode");
group.throughput(Throughput::Bytes(fixture.text.len() as u64));
group.bench_function("fastokens", |b| {
b.iter(|| {
fixture
.fastokens
.encode(black_box(fixture.text.as_str()), black_box(false))
.expect("encode sample text with fastokens")
})
});
group.bench_function("hf_tokenizers", |b| {
b.iter(|| {
fixture
.hf
.encode(black_box(fixture.text.as_str()), black_box(false))
.expect("encode sample text with hf tokenizer")
})
});
group.finish();
}
fn bench_decode(c: &mut Criterion) {
let fixture = BenchFixture::load();
let mut group = c.benchmark_group("tokenizer_decode");
group.throughput(Throughput::Elements(fixture.token_ids.len() as u64));
group.bench_function("fastokens", |b| {
b.iter(|| {
fixture
.fastokens
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
.expect("decode sample token ids with fastokens")
})
});
group.bench_function("hf_tokenizers", |b| {
b.iter(|| {
fixture
.hf
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
.expect("decode sample token ids with hf tokenizer")
})
});
group.finish();
}
criterion_group!(benches, bench_encode, bench_decode);
criterion_main!(benches);
+117
View File
@@ -0,0 +1,117 @@
use criterion::{Criterion, Throughput, black_box, criterion_group, criterion_main};
use hf_hub::api::sync::ApiBuilder;
use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
const MODEL_ID: &str = "moonshotai/Kimi-K2.5";
const SAMPLE_TEXT: &str = "\
<think>
I'm sure it's fine, but I can't say I'd trust that it's what we'd ship.
</think>
请用中英混合总结以下需求,并保留 tool-call marker:
<|tool_calls_section_begin|>{\"name\":\"summarize\",\"arguments\":{\"style\":\"brief\"}}<|tool_calls_section_end|>
The service should stop cleanly at EOS, avoid leaking the next template turn, and keep decode latency low.
";
struct BenchFixture {
riptoken: TiktokenTokenizer,
tiktoken_rs: TiktokenTokenizer,
text: String,
token_ids: Vec<u32>,
}
impl BenchFixture {
fn load() -> Self {
let path = tiktoken_model();
let riptoken = TiktokenTokenizer::new_riptoken(&path).expect("load riptoken tokenizer");
let tiktoken_rs =
TiktokenTokenizer::new_tiktoken_rs(&path).expect("load tiktoken-rs tokenizer");
let text = SAMPLE_TEXT.repeat(32);
let riptoken_token_ids =
riptoken.encode(text.as_str(), false).expect("encode sample text with riptoken");
let tiktoken_rs_token_ids = tiktoken_rs
.encode(text.as_str(), false)
.expect("encode sample text with tiktoken-rs");
assert_eq!(riptoken_token_ids, tiktoken_rs_token_ids);
let riptoken_decoded = riptoken
.decode(riptoken_token_ids.as_slice(), false)
.expect("decode sample token ids with riptoken");
let tiktoken_rs_decoded = tiktoken_rs
.decode(riptoken_token_ids.as_slice(), false)
.expect("decode sample token ids with tiktoken-rs");
assert_eq!(riptoken_decoded, tiktoken_rs_decoded);
Self {
riptoken,
tiktoken_rs,
text,
token_ids: riptoken_token_ids,
}
}
}
fn tiktoken_model() -> std::path::PathBuf {
let repo = ApiBuilder::from_env()
.with_progress(false)
.build()
.expect("build hf-hub api")
.model(MODEL_ID.to_string());
repo.get("config.json").expect("fetch config.json from hf-hub");
repo.get("tokenizer_config.json")
.expect("fetch tokenizer_config.json from hf-hub");
repo.get("tiktoken.model").expect("fetch tiktoken.model from hf-hub")
}
fn bench_encode(c: &mut Criterion) {
let fixture = BenchFixture::load();
let mut group = c.benchmark_group("tiktoken_encode");
group.throughput(Throughput::Bytes(fixture.text.len() as u64));
group.bench_function("riptoken", |b| {
b.iter(|| {
fixture
.riptoken
.encode(black_box(fixture.text.as_str()), black_box(false))
.expect("encode sample text with riptoken")
})
});
group.bench_function("tiktoken_rs", |b| {
b.iter(|| {
fixture
.tiktoken_rs
.encode(black_box(fixture.text.as_str()), black_box(false))
.expect("encode sample text with tiktoken-rs")
})
});
group.finish();
}
fn bench_decode(c: &mut Criterion) {
let fixture = BenchFixture::load();
let mut group = c.benchmark_group("tiktoken_decode");
group.throughput(Throughput::Elements(fixture.token_ids.len() as u64));
group.bench_function("riptoken", |b| {
b.iter(|| {
fixture
.riptoken
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
.expect("decode sample token ids with riptoken")
})
});
group.bench_function("tiktoken_rs", |b| {
b.iter(|| {
fixture
.tiktoken_rs
.decode(black_box(fixture.token_ids.as_slice()), black_box(false))
.expect("decode sample token ids with tiktoken-rs")
})
});
group.finish();
}
criterion_group!(benches, bench_encode, bench_decode);
criterion_main!(benches);
+119
View File
@@ -0,0 +1,119 @@
//! Fast GPT-2 byte-level detokenization that writes into a single `Vec<u8>`,
//! avoiding the `Vec<String>` / `String::join` assembly in fastokens' generic
//! `Decoder::decode` pipeline.
/// Reverse GPT-2 byte-to-unicode mapping: codepoint → original byte. The GPT-2
/// table only emits codepoints in U+0000..U+0143, so a flat array suffices.
const CHAR_TO_BYTE: [u8; 324] = build_char_to_byte();
const fn is_nice(b: u8) -> bool {
(b >= b'!' && b <= b'~') || (b >= 0xA1 && b <= 0xAC) || b >= 0xAE
}
const fn build_char_to_byte() -> [u8; 324] {
let mut table = [0u8; 324];
let mut b: u16 = 0;
while b < 256 {
let cp = if is_nice(b as u8) {
b as u32
} else {
256 + nice_offset(b as u8)
};
table[cp as usize] = b as u8;
b += 1;
}
table
}
const fn nice_offset(b: u8) -> u32 {
let mut i: u16 = 0;
let mut n: u32 = 0;
while i < b as u16 {
if !is_nice(i as u8) {
n += 1;
}
i += 1;
}
n
}
/// Decode byte-level encoded token strings into a single UTF-8 string,
/// matching `fastokens::decoders::ByteLevelDecoder`.
pub fn decode_byte_level<'a, I: IntoIterator<Item = &'a str>>(tokens: I) -> String {
let iter = tokens.into_iter();
let (lower, _) = iter.size_hint();
let mut bytes: Vec<u8> = Vec::with_capacity(lower.saturating_mul(4));
for token in iter {
for c in token.chars() {
let cp = c as usize;
if cp < CHAR_TO_BYTE.len() {
bytes.push(CHAR_TO_BYTE[cp]);
} else {
// Non-GPT2 codepoints (e.g. DeepSeek's U+FF5C, U+2581) pass through.
let mut buf = [0u8; 4];
bytes.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
}
}
}
String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
}
#[cfg(test)]
mod tests {
use super::*;
fn build_byte_to_char_ref() -> [char; 256] {
let mut table = ['\0'; 256];
let mut next: u32 = 256;
for b in 0..=255u8 {
let cp = if is_nice(b) {
b as u32
} else {
let cp = next;
next += 1;
cp
};
table[b as usize] = char::from_u32(cp).unwrap();
}
table
}
#[test]
fn char_to_byte_roundtrips_every_byte() {
let byte_to_char = build_byte_to_char_ref();
for b in 0..=255u8 {
let cp = byte_to_char[b as usize] as usize;
assert!(cp < CHAR_TO_BYTE.len());
assert_eq!(CHAR_TO_BYTE[cp], b, "mismatch for byte {b:#x}");
}
}
#[test]
fn decode_ascii() {
assert_eq!(decode_byte_level(["Hello"]), "Hello");
}
#[test]
fn decode_space_marker() {
// GPT-2 maps 0x20 → Ġ (U+0120).
assert_eq!(
decode_byte_level(["\u{120}Hello", "\u{120}world"]),
" Hello world",
);
}
#[test]
fn decode_multibyte_euro() {
// € → 0xE2 0x82 0xAC, each mapped to a specific GPT-2 char.
let byte_to_char = build_byte_to_char_ref();
let encoded: String =
[0xE2u8, 0x82, 0xAC].iter().map(|&b| byte_to_char[b as usize]).collect();
assert_eq!(decode_byte_level([encoded.as_str()]), "€");
}
#[test]
fn decode_preserves_non_gpt2_chars() {
let tok = "<\u{FF5C}begin\u{2581}of\u{2581}sentence\u{FF5C}>";
assert_eq!(decode_byte_level([tok]), "<|begin▁of▁sentence|>");
}
}
+9
View File
@@ -0,0 +1,9 @@
use thiserror::Error;
use thiserror_ext::Macro;
pub type Result<T> = std::result::Result<T, TokenizerError>;
#[derive(Debug, Error, Macro)]
#[thiserror_ext(macro(path = "crate::error"))]
#[error("tokenizer error: {0}")]
pub struct TokenizerError(#[message] pub String);
+344
View File
@@ -0,0 +1,344 @@
use std::path::Path;
use std::sync::Arc;
use fastokens::Tokenizer as FastokensTokenizer;
use fastokens::decoders::Decoder as FastokensDecoder;
use thiserror_ext::AsReport as _;
use tokenizers::Tokenizer as HfTokenizer;
use tracing::{info, warn};
use crate::byte_level_decode::decode_byte_level;
use crate::{Result, Tokenizer};
enum Backend {
Hf(Box<HfTokenizer>),
Fastokens(Box<FastokensTokenizer>),
/// Fastokens tokenizer whose decoder is pure GPT-2 byte-level, so we can
/// bypass `Decoder::decode`'s `Vec<String>`/`join("")` assembly.
FastokensByteLevel(Box<FastokensTokenizer>),
}
/// True if `dec` is effectively a single `ByteLevel` stage — one `ByteLevel`
/// leaf in a tree of `Sequence`s (fastokens represents `Fuse` as an empty
/// `Sequence`, which is a no-op for our purposes).
fn is_byte_level_only(dec: &FastokensDecoder) -> bool {
fn count_byte_level(dec: &FastokensDecoder) -> usize {
match dec {
FastokensDecoder::ByteLevel(_) => 1,
FastokensDecoder::Sequence(steps) => steps.iter().map(count_byte_level).sum(),
}
}
count_byte_level(dec) == 1
}
fn decode_fastokens_byte_level(
t: &FastokensTokenizer,
token_ids: &[u32],
skip_special_tokens: bool,
) -> Result<String> {
let tokens: Vec<&str> = token_ids
.iter()
.filter(|&&id| !(skip_special_tokens && t.is_special_token(id)))
.map(|&id| {
t.id_to_token(id)
.ok_or_else(|| tokenizer_error!("decoding failed: unknown token ID: {id}"))
})
.collect::<Result<_>>()?;
Ok(decode_byte_level(tokens))
}
/// Tokenizer from `tokenizer.json` in HuggingFace format.
///
/// This tries to load with `fastokens` first for better performance, then falls
/// back to HuggingFace's `tokenizers` if the former fails (e.g. due to
/// unsupported tokenizer features or file formats).
pub struct HuggingFaceTokenizer {
backend: Backend,
special_token_ids: Arc<[u32]>,
}
impl HuggingFaceTokenizer {
fn from_hf_backend(tokenizer: HfTokenizer) -> Self {
let special_token_ids = {
let mut ids: Vec<u32> = tokenizer
.get_added_tokens_decoder()
.iter()
.filter(|(_id, token)| token.special)
.map(|(id, _token)| *id)
.collect();
ids.sort_unstable();
ids.dedup();
Arc::from(ids)
};
Self {
backend: Backend::Hf(Box::new(tokenizer)),
special_token_ids,
}
}
fn from_fastokens_backend(tokenizer: FastokensTokenizer) -> Self {
let special_token_ids = {
let mut ids: Vec<u32> = tokenizer
.added_tokens()
.into_iter()
.flat_map(|added_tokens| added_tokens.iter())
.filter(|token| token.special)
.map(|token| token.id)
.collect();
ids.sort_unstable();
ids.dedup();
Arc::from(ids)
};
let byte_level = tokenizer.decoder().is_some_and(is_byte_level_only);
let backend = if byte_level {
Backend::FastokensByteLevel(Box::new(tokenizer))
} else {
Backend::Fastokens(Box::new(tokenizer))
};
Self {
backend,
special_token_ids,
}
}
/// Load from `tokenizer.json` with `fastokens`.
pub fn new_fastokens(path: &Path) -> Result<Self> {
info!(path = %path.display(), "loading tokenizer with fastokens");
let t = FastokensTokenizer::from_file(path)
.map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
Ok(Self::from_fastokens_backend(t))
}
/// Load from `tokenizer.json` with Hugging Face `tokenizers`.
pub fn new_hf(path: &Path) -> Result<Self> {
info!(path = %path.display(), "loading tokenizer with huggingface tokenizers");
let t = HfTokenizer::from_file(path)
.map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
Ok(Self::from_hf_backend(t))
}
/// Load from `tokenizer.json` via fastokens or HuggingFace tokenizers.
pub fn new(path: &Path) -> Result<Self> {
match Self::new_fastokens(path) {
Ok(tokenizer) => Ok(tokenizer),
Err(error) => {
warn!(
path = %path.display(),
error = %error.as_report(),
"failed to load tokenizer with fastokens; falling back to HuggingFace tokenizers"
);
Self::new_hf(path)
}
}
}
}
impl Tokenizer for HuggingFaceTokenizer {
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>> {
match &self.backend {
Backend::Hf(t) => {
let encoding = t
.encode(text, add_special_tokens)
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report()))?;
Ok(encoding.get_ids().to_vec())
}
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => t
.encode_with_special_tokens(text, add_special_tokens)
.map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report())),
}
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
match &self.backend {
Backend::Hf(t) => t
.decode(token_ids, skip_special_tokens)
.map_err(|error| tokenizer_error!("decoding failed: {}", error.as_report())),
Backend::Fastokens(t) => t
.decode(token_ids, skip_special_tokens)
.map_err(|error| tokenizer_error!("decoding failed: {}", error.as_report())),
Backend::FastokensByteLevel(t) => {
decode_fastokens_byte_level(t, token_ids, skip_special_tokens)
}
}
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match &self.backend {
Backend::Hf(t) => t.token_to_id(token),
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => t.token_to_id(token),
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
match &self.backend {
Backend::Hf(t) => t.id_to_token(id),
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => {
t.id_to_token(id).map(ToOwned::to_owned)
}
}
}
fn is_special_id(&self, token_id: u32) -> bool {
self.special_token_ids.binary_search(&token_id).is_ok()
}
}
#[cfg(test)]
mod tests {
use tempfile::tempdir;
use tokenizers::models::bpe::BPE;
use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
use super::{HuggingFaceTokenizer, Tokenizer};
fn tiny_bpe_tokenizer() -> HfTokenizer {
let vocab = [
("<unk>".to_string(), 0),
("h".to_string(), 1),
("e".to_string(), 2),
("l".to_string(), 3),
("o".to_string(), 4),
("he".to_string(), 5),
("ll".to_string(), 6),
("hell".to_string(), 7),
("hello".to_string(), 8),
];
let merges = vec![
("h".to_string(), "e".to_string()),
("l".to_string(), "l".to_string()),
("he".to_string(), "ll".to_string()),
("hell".to_string(), "o".to_string()),
];
let model = BPE::builder()
.vocab_and_merges(vocab, merges)
.unk_token("<unk>".to_string())
.build()
.expect("build bpe tokenizer");
HfTokenizer::new(model)
}
#[test]
fn hf_constructor_resolves_added_token_ids() {
let mut tokenizer = tiny_bpe_tokenizer();
tokenizer.add_special_tokens(&[AddedToken::from("<|im_end|>", true)]);
let dir = tempdir().expect("create temp dir");
let path = dir.path().join("tokenizer.json");
tokenizer.save(&path, false).expect("save tokenizer json");
let wrapper = HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper");
let special_id = wrapper.token_to_id("<|im_end|>").expect("resolve added special token id");
assert!(wrapper.is_special_id(special_id));
}
#[test]
fn new_fastokens_preserves_special_ids_from_fastokens_metadata() {
let mut tokenizer = tiny_bpe_tokenizer();
tokenizer.add_special_tokens(&[AddedToken::from("<|im_end|>", true)]);
let dir = tempdir().expect("create temp dir");
let path = dir.path().join("tokenizer.json");
tokenizer.save(&path, false).expect("save tokenizer json");
let wrapper = HuggingFaceTokenizer::new_fastokens(&path)
.expect("load wrapper with fastokens backend");
assert!(matches!(
wrapper.backend,
super::Backend::Fastokens(_) | super::Backend::FastokensByteLevel(_),
));
let special_id = wrapper.token_to_id("<|im_end|>").expect("resolve added special token id");
assert!(wrapper.is_special_id(special_id));
}
/// BPE tokenizer that round-trips through fastokens with a genuine
/// `ByteLevel` decoder; vocab covers both GPT-2 (Ġ U+0120) and non-GPT-2
/// (| U+FF5C) codepoints.
fn tiny_byte_level_bpe() -> fastokens::Tokenizer {
let raw = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{"id": 0, "content": "<|endoftext|>", "single_word": false,
"lstrip": false, "rstrip": false, "normalized": false, "special": true}
],
"normalizer": null,
"pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": false,
"trim_offsets": true, "use_regex": true},
"post_processor": null,
"decoder": {"type": "ByteLevel", "add_prefix_space": false,
"trim_offsets": true, "use_regex": true},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|endoftext|>": 0,
"H": 1, "e": 2, "l": 3, "o": 4, "w": 5, "r": 6, "d": 7,
"Ġ": 8, "!": 9,
"|": 10
},
"merges": []
}
}"#;
let value: serde_json::Value = serde_json::from_str(raw).expect("parse tokenizer json");
fastokens::Tokenizer::from_json(value).expect("build fastokens tokenizer")
}
#[test]
fn byte_level_detected_direct() {
let t = tiny_byte_level_bpe();
assert!(super::is_byte_level_only(t.decoder().expect("decoder")));
}
#[test]
fn byte_level_detected_inside_sequence() {
let raw = r#"{
"type": "Sequence",
"decoders": [
{"type": "ByteLevel", "add_prefix_space": false,
"trim_offsets": true, "use_regex": true},
{"type": "Fuse"}
]
}"#;
let config: fastokens::DecoderConfig =
serde_json::from_str(raw).expect("parse decoder config");
let dec =
fastokens::decoders::Decoder::from_config(config).expect("build decoder from config");
assert!(super::is_byte_level_only(&dec));
}
/// Fast path must produce byte-identical output to fastokens' own decode.
#[test]
fn fast_byte_level_matches_fastokens_decode() {
let t = tiny_byte_level_bpe();
let cases: &[&[u32]] = &[
&[],
&[1, 2, 3, 3, 4], // "Hello"
&[1, 2, 3, 3, 4, 8, 5, 4, 6, 3, 7], // "Hello world"
&[0, 1, 2, 3, 3, 4, 0, 9, 0], // specials interleaved
&[10, 1, 2, 3, 3, 4, 10], // |Hello| (non-GPT2 chars)
];
for ids in cases {
for &skip in &[false, true] {
let expected = t.decode(ids, skip).expect("fastokens decode");
let got =
super::decode_fastokens_byte_level(&t, ids, skip).expect("fast-path decode");
assert_eq!(got, expected, "ids={ids:?} skip={skip}");
}
}
}
#[test]
fn fast_byte_level_errors_on_unknown_id() {
let t = tiny_byte_level_bpe();
let err = super::decode_fastokens_byte_level(&t, &[999], false)
.expect_err("unknown id must error");
assert!(format!("{err:?}").contains("999"));
}
}
+359
View File
@@ -0,0 +1,359 @@
use std::mem::take;
use crate::{Result, Tokenizer};
/// Stateful incremental decoder that emits text chunks one token at a time.
pub trait IncrementalDecoder: Send {
/// Push one generated token and return how many new string bytes were
/// added.
fn push_token(&mut self, token_id: u32) -> Result<usize>;
/// Consume any text which is currently ready.
fn next_chunk(&mut self) -> Option<String>;
/// Flush any remaining buffered text that has not yet been emitted.
///
/// Called after the final generated token to force out buffered/incomplete
/// fragments.
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)>;
/// Return cumulative decoded text so far.
fn output(&self) -> &str;
}
/// [`IncrementalDecoder`] built on [`Tokenizer::decode()`] with prefix-diffing.
///
/// This is the same sliding-window algorithm used by `tokenizers::DecodeStream`
pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> {
tokenizer: &'a T,
skip_special_tokens: bool,
min_bytes_to_buffer: usize,
// mutated state
ids: Vec<u32>,
prefix: String,
prefix_index: usize,
cumulative_output: String,
output_index: usize,
}
impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
pub(crate) fn new(
tokenizer: &'a T,
prompt_token_ids: &[u32],
skip_special_tokens: bool,
min_bytes_to_buffer: usize,
) -> Self {
Self {
tokenizer,
skip_special_tokens,
min_bytes_to_buffer,
ids: prompt_token_ids.to_vec(),
prefix: String::new(),
prefix_index: 0,
cumulative_output: String::new(),
output_index: 0,
}
}
}
/// Try a short tail suffix first (covers a CJK glyph straddling 1-2 token
/// boundaries); beyond 6 tokens the fallback full-prompt decode is no worse
/// than baseline so widening the sweep just adds overhead.
const SAFE_SUFFIX_MIN: usize = 4;
const SAFE_SUFFIX_MAX: usize = 6;
impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
/// Seed `self.prefix` from the shortest trailing suffix whose decoded text
/// has no U+FFFD — a clean decode means the suffix starts and ends at
/// valid UTF-8/token boundaries, so priming from it is equivalent to
/// priming from the full prompt.
fn seed_prefix(&mut self) -> Result<()> {
let prompt_len = self.ids.len();
if prompt_len > SAFE_SUFFIX_MIN {
let max_try = SAFE_SUFFIX_MAX.min(prompt_len - 1);
for suffix_len in SAFE_SUFFIX_MIN..=max_try {
let start = prompt_len - suffix_len;
let decoded =
self.tokenizer.decode(&self.ids[start..], self.skip_special_tokens)?;
if !decoded.contains('\u{FFFD}') {
self.prefix = decoded;
self.ids.drain(..start);
self.prefix_index = self.ids.len();
return Ok(());
}
}
}
let decoded = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
if !decoded.ends_with('\u{FFFD}') {
self.prefix = decoded;
self.prefix_index = self.ids.len();
}
Ok(())
}
}
impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
fn push_token(&mut self, token_id: u32) -> Result<usize> {
if self.prefix.is_empty() && !self.ids.is_empty() {
self.seed_prefix()?;
}
self.ids.push(token_id);
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
if string.len() <= prefix_len || string.ends_with('\u{FFFD}') {
return Ok(0);
}
// Ensure we split at a utf-8 char boundary.
let new_chunk = &string[string.floor_char_boundary(prefix_len)..];
self.cumulative_output.push_str(new_chunk);
self.ids.drain(..self.prefix_index);
self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
self.prefix_index = self.ids.len();
Ok(new_chunk.len())
}
fn next_chunk(&mut self) -> Option<String> {
let cutoff = self.cumulative_output.len().saturating_sub(self.min_bytes_to_buffer);
(cutoff > self.output_index).then(|| {
let chunk = self.cumulative_output[self.output_index..cutoff].to_string();
self.output_index = cutoff;
chunk
})
}
fn flush(&mut self, truncate_output_to: Option<usize>) -> Result<(Option<String>, String)> {
if !self.ids.is_empty() {
let string = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
let prefix_len = self.prefix.len();
self.ids.clear();
self.prefix.clear();
self.prefix_index = 0;
// Ensure we split at a utf-8 char boundary.
self.cumulative_output
.push_str(&string[string.floor_char_boundary(prefix_len)..]);
}
if let Some(truncate_output_to) = truncate_output_to {
self.cumulative_output.truncate(truncate_output_to);
}
let last_chunk = (self.output_index < self.cumulative_output.len())
.then(|| self.cumulative_output[self.output_index..].to_string());
self.output_index = 0;
Ok((last_chunk, take(&mut self.cumulative_output)))
}
fn output(&self) -> &str {
&self.cumulative_output
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
#[derive(Debug)]
struct Utf8Backend;
impl Tokenizer for Utf8Backend {
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
#[test]
fn holds_incomplete_utf8_until_complete() {
let backend = Utf8Backend;
let mut decoder = backend.create_decode_stream(&[], false, 0);
// 你 = U+4F60 = 0xE4 0xBD 0xA0
assert_eq!(decoder.push_token(0xe4).unwrap(), 0);
assert_eq!(decoder.push_token(0xbd).unwrap(), 0);
assert_eq!(decoder.push_token(0xa0).unwrap(), 3); // "你" is 3 bytes
assert_eq!(decoder.output(), "你");
}
#[test]
fn emits_ascii_immediately() {
let backend = Utf8Backend;
let mut decoder = backend.create_decode_stream(&[], false, 0);
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
assert_eq!(decoder.output(), "ok");
}
#[test]
fn flush_returns_none_when_fully_consumed() {
let backend = Utf8Backend;
let mut decoder = backend.create_decode_stream(&[], false, 0);
assert_eq!(decoder.push_token(b'o' as u32).unwrap(), 1);
assert_eq!(decoder.next_chunk().as_deref(), Some("o"));
assert_eq!(decoder.push_token(b'k' as u32).unwrap(), 1);
assert_eq!(decoder.next_chunk().as_deref(), Some("k"));
// All text already consumed via next_chunk
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None);
assert_eq!(full_text, "ok");
}
#[test]
fn flush_emits_buffered_incomplete_utf8() {
let backend = Utf8Backend;
let mut decoder = backend.create_decode_stream(&[], false, 0);
// Push incomplete multi-byte sequence — step returns 0 bytes.
assert_eq!(decoder.push_token(0xe4).unwrap(), 0);
assert_eq!(decoder.push_token(0xbd).unwrap(), 0);
// Flush forces out whatever the decoder can produce (lossy replacement).
let (last_chunk, _full_text) = decoder.flush(None).unwrap();
assert!(last_chunk.is_some());
}
/// Backend where token 0 is a special token.
#[derive(Debug)]
struct SpecialTokenBackend;
impl Tokenizer for SpecialTokenBackend {
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
let mut text = String::new();
for &token_id in token_ids {
match token_id {
0 if !skip_special_tokens => text.push_str("<special>"),
0 => {}
1 => text.push('a'),
_ => {}
}
}
Ok(text)
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
#[test]
fn respects_skip_special_tokens() {
let backend = SpecialTokenBackend;
let mut skip_decoder = backend.create_decode_stream(&[], true, 0);
let mut keep_decoder = backend.create_decode_stream(&[], false, 0);
assert_eq!(skip_decoder.push_token(0).unwrap(), 0);
assert_eq!(keep_decoder.push_token(0).unwrap(), 9); // "<special>" is 9 bytes
assert_eq!(keep_decoder.output(), "<special>");
}
#[test]
fn prompt_tokens_provide_context_without_re_emission() {
let backend = Utf8Backend;
let prompt = &[b'H' as u32, b'i' as u32];
let mut decoder = backend.create_decode_stream(prompt, false, 0);
// First generated token should not re-emit "Hi".
let added = decoder.push_token(b'!' as u32).unwrap();
assert_eq!(added, 1);
assert_eq!(decoder.output(), "!");
}
#[test]
fn chunks_concatenate_to_full_text() {
let backend = Utf8Backend;
let mut decoder = backend.create_decode_stream(&[], false, 0);
let input = b"Hello, world!";
let mut full = String::new();
for &byte in input {
decoder.push_token(byte as u32).unwrap();
if let Some(chunk) = decoder.next_chunk() {
full.push_str(&chunk);
}
}
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk, None); // all consumed via next_chunk
assert_eq!(full, "Hello, world!");
assert_eq!(full_text, "Hello, world!");
}
/// Backend simulating non-monotonic decode where adding a token changes how
/// earlier tokens decode (context-dependent normalization), causing
/// prefix_len to land mid-UTF-8. Reproduces the class of bug from
/// vllm-project/vllm#17448.
#[derive(Debug)]
struct NonMonotonicBackend;
impl Tokenizer for NonMonotonicBackend {
fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result<Vec<u32>> {
unreachable!()
}
fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
match token_ids {
[1] => Ok("abc".into()),
[1, 2] => Ok("ab".into()),
// Token 3 triggers a normalization change: "ab" becomes emoji + "d".
// prefix_len=3 ("abc") lands inside the 4-byte emoji 🎉.
[1, 2, 3] => Ok("🎉d".into()), // 🎉 is 4 bytes + d = 5 bytes
[2, 3] => Ok("🎉d".into()), // prefix recompute after drain
[3] => Ok("d".into()), // after drain
_ => panic!("unexpected decode: {:?}", token_ids),
}
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
/// Without the char-boundary fix, this panics slicing mid-emoji.
#[test]
fn non_monotonic_decode_does_not_panic() {
let backend = NonMonotonicBackend;
let mut decoder = backend.create_decode_stream(&[], false, 0);
// Token 1: "abc", prefix="abc"
assert_eq!(decoder.push_token(1).unwrap(), 3);
// Token 2: "ab" (shorter), no emit
assert_eq!(decoder.push_token(2).unwrap(), 0);
// Token 3: "🎉d" — prefix_len=3 is mid-emoji. Without fix this panics.
let added = decoder.push_token(3).unwrap();
assert!(added > 0);
}
#[test]
fn next_chunk_with_hold_back() {
let backend = Utf8Backend;
// hold_back_bytes: 3 means we buffer the last 3 bytes
let mut decoder = backend.create_decode_stream(&[], false, 3);
let input = b"Hello!";
let mut chunks = String::new();
for &byte in input {
decoder.push_token(byte as u32).unwrap();
if let Some(chunk) = decoder.next_chunk() {
chunks.push_str(&chunk);
}
}
// With hold_back_bytes=3, last 3 bytes ("lo!") are held back
assert_eq!(chunks, "Hel");
// Flush returns the rest
let (last_chunk, full_text) = decoder.flush(None).unwrap();
assert_eq!(last_chunk.as_deref(), Some("lo!"));
assert_eq!(full_text, "Hello!");
}
}
+62
View File
@@ -0,0 +1,62 @@
use std::sync::Arc;
use crate::incremental::DecodeStream;
mod byte_level_decode;
#[macro_use]
mod error;
mod hf;
mod incremental;
mod tekken;
mod tiktoken;
pub use error::{Result, TokenizerError};
pub use hf::HuggingFaceTokenizer;
pub use incremental::IncrementalDecoder;
pub use tekken::TekkenTokenizer;
pub use tiktoken::TiktokenTokenizer;
pub trait Tokenizer: Send + Sync {
/// Encode one prompt string into token IDs.
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>;
/// Decode one token sequence into text.
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String>;
/// Convert one token string into a token ID, returning `None` if the token
/// is not in the tokenizer vocabulary.
fn token_to_id(&self, token: &str) -> Option<u32>;
/// Convert one token ID into the tokenizer's raw token string.
fn id_to_token(&self, _id: u32) -> Option<String> {
// TODO: remove default impl and require this to be implemented by all
// tokenizers
None
}
/// Return whether the given token ID is special.
fn is_special_id(&self, _token_id: u32) -> bool {
false
}
/// Create a stateful incremental decoder primed with the given prompt
/// tokens.
///
/// The prompt tokens provide left context for the first generated token;
/// the decoder does not re-emit prompt text.
fn create_decode_stream(
&self,
prompt_token_ids: &[u32],
skip_special_tokens: bool,
min_bytes_to_buffer: usize,
) -> Box<dyn IncrementalDecoder + '_> {
Box::new(DecodeStream::new(
self,
prompt_token_ids,
skip_special_tokens,
min_bytes_to_buffer,
))
}
}
pub type DynTokenizer = Arc<dyn Tokenizer>;
+62
View File
@@ -0,0 +1,62 @@
use std::path::Path;
use tekken::Tekkenizer;
use tracing::info;
use crate::{Result, Tokenizer};
/// Mistral Tekken tokenizer from a `tekken.json` file.
pub struct TekkenTokenizer {
inner: Tekkenizer,
}
impl TekkenTokenizer {
/// Load a Mistral Tekken tokenizer from a `tekken.json` file.
pub fn new(path: &Path) -> Result<Self> {
info!(path = %path.display(), "loading tokenizer with Mistral Tekken");
let inner = Tekkenizer::from_file(path).map_err(|error| {
tokenizer_error!(
"failed to load tekken tokenizer from {}: {error}",
path.display()
)
})?;
Ok(Self { inner })
}
}
impl Tokenizer for TekkenTokenizer {
fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>> {
self.inner
.encode(text, add_special_tokens, false)
.map_err(|error| tokenizer_error!("encoding failed: {error}"))
}
fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
let policy = if skip_special_tokens {
tekken::SpecialTokenPolicy::Ignore
} else {
tekken::SpecialTokenPolicy::Keep
};
self.inner
.decode(token_ids, policy)
.map_err(|error| tokenizer_error!("decoding failed: {error}"))
}
fn token_to_id(&self, token: &str) -> Option<u32> {
// tekken-rs exposes `get_control_token` for special tokens. Try that first,
// then fall back to encoding.
self.inner.get_control_token(token).ok().or_else(|| {
let ids = self.inner.encode(token, false, false).ok()?;
if ids.len() == 1 { Some(ids[0]) } else { None }
})
}
fn id_to_token(&self, id: u32) -> Option<String> {
self.inner.id_to_piece(id).ok()
}
fn is_special_id(&self, token_id: u32) -> bool {
self.inner.is_special_token(token_id)
}
}
File diff suppressed because it is too large Load Diff