diff --git a/rust/src/chat/src/renderer/inkling/tests.rs b/rust/src/chat/src/renderer/inkling/tests.rs index d0d55e1be2..57e08ac015 100644 --- a/rust/src/chat/src/renderer/inkling/tests.rs +++ b/rust/src/chat/src/renderer/inkling/tests.rs @@ -31,6 +31,10 @@ impl Tokenizer for FixtureTokenizer { Ok(text.bytes().map(u32::from).collect()) } + fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result> { + self.encode(text, false) + } + fn decode( &self, token_ids: &[u32], diff --git a/rust/src/parser/benches/utils/adapter.rs b/rust/src/parser/benches/utils/adapter.rs index 243f19e3ce..9cba325116 100644 --- a/rust/src/parser/benches/utils/adapter.rs +++ b/rust/src/parser/benches/utils/adapter.rs @@ -19,6 +19,10 @@ impl Tokenizer for BenchTokenizer { Ok(text.chars().map(|_| u32::MAX).collect()) } + fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result> { + self.encode(text, false) + } + fn decode( &self, token_ids: &[u32], diff --git a/rust/src/parser/src/unified/inkling.rs b/rust/src/parser/src/unified/inkling.rs index eb78321d12..1ce69d8bfb 100644 --- a/rust/src/parser/src/unified/inkling.rs +++ b/rust/src/parser/src/unified/inkling.rs @@ -414,6 +414,10 @@ mod tests { Ok(text.chars().map(u32::from).collect()) } + fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result> { + self.encode(text, false) + } + fn decode( &self, token_ids: &[u32], @@ -733,6 +737,10 @@ mod tests { Ok(vec![]) } + fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result> { + self.encode(text, false) + } + fn decode( &self, _token_ids: &[u32], diff --git a/rust/src/text/src/backend/hf/mod.rs b/rust/src/text/src/backend/hf/mod.rs index 49ae5dbd6b..4fc7a18753 100644 --- a/rust/src/text/src/backend/hf/mod.rs +++ b/rust/src/text/src/backend/hf/mod.rs @@ -177,6 +177,10 @@ mod tests { Ok(vec![]) } + fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result> { + self.encode(text, false) + } + fn decode( &self, _token_ids: &[u32], diff --git a/rust/src/tokenizer/src/hf.rs b/rust/src/tokenizer/src/hf.rs index 08ec5a22d6..eb527294a5 100644 --- a/rust/src/tokenizer/src/hf.rs +++ b/rust/src/tokenizer/src/hf.rs @@ -1,13 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::borrow::Cow; use std::path::Path; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use fastokens::Tokenizer as FastokensTokenizer; use fastokens::decoders::Decoder as FastokensDecoder; +use fastokens::pre_tokenized::{ + PreTokenizedString as FastokensPreTokenizedString, Split as FastokensSplit, +}; +use fastokens::{PreTokenizer as FastokensPreTokenizer, Split as FastokensSplitPreTokenizer}; use thiserror_ext::AsReport as _; -use tokenizers::Tokenizer as HfTokenizer; +use tokenizers::{ + AddedVocabulary, Model as _, OffsetType, PreTokenizer as _, Tokenizer as HfTokenizer, +}; use tracing::{info, warn}; use crate::byte_level_decode::decode_byte_level; @@ -16,6 +23,8 @@ use crate::{Result, Tokenizer}; mod added_tokens; +static EMPTY_HF_ADDED_VOCABULARY: LazyLock = LazyLock::new(AddedVocabulary::new); + enum Backend { Hf(Box), Fastokens(Box), @@ -53,6 +62,85 @@ fn decode_fastokens_byte_level( Ok(decode_byte_level(tokens)) } +fn encode_hf_ordinary(tokenizer: &HfTokenizer, text: &str) -> tokenizers::Result> { + let mut pretokenized = + EMPTY_HF_ADDED_VOCABULARY.extract_and_normalize(tokenizer.get_normalizer(), text); + + if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() { + pre_tokenizer.pre_tokenize(&mut pretokenized)?; + } + pretokenized.tokenize(|normalized| tokenizer.get_model().tokenize(normalized.get()))?; + let encoding = pretokenized.into_encoding(None, 0, OffsetType::Byte)?; + let encoding = tokenizer.post_process(encoding, None, false)?; + Ok(encoding.get_ids().to_vec()) +} + +fn fastokens_fused_split(tokenizer: &FastokensTokenizer) -> Option<&FastokensSplitPreTokenizer> { + // Keep this predicate aligned with fastokens::Tokenizer::detect_fused_byte_level. + let FastokensPreTokenizer::Sequence(steps) = tokenizer.pre_tokenizer()? else { + return None; + }; + let [ + FastokensPreTokenizer::Split(split), + FastokensPreTokenizer::ByteLevel(byte_level), + ] = steps.as_slice() + else { + return None; + }; + byte_level.is_bulk_only().then_some(split) +} + +fn fastokens_pre_tokenized_ordinary( + tokenizer: &FastokensTokenizer, + text: &str, +) -> FastokensPreTokenizedString { + // This is fastokens::Tokenizer::build_pre_tokenized with added_tokens = None. + let normalized = tokenizer + .normalizer() + .map_or(Cow::Borrowed(text), |normalizer| normalizer.normalize(text)); + match normalized { + Cow::Borrowed(_) => FastokensPreTokenizedString::from_text(text), + Cow::Owned(text) => { + let len = text.len(); + FastokensPreTokenizedString::new( + text, + vec![FastokensSplit { + range: 0..len, + token_id: None, + }], + ) + } + } +} + +fn encode_fastokens_ordinary( + tokenizer: &FastokensTokenizer, + text: &str, +) -> std::result::Result, fastokens::Error> { + if text.is_empty() { + return Ok(Vec::new()); + } + + let mut pretokenized = fastokens_pre_tokenized_ordinary(tokenizer, text); + let ids = if let Some(split) = fastokens_fused_split(tokenizer) { + split.pre_tokenize(&mut pretokenized)?; + pretokenized + .tokenize_batched(|buffer, splits, output| { + tokenizer.model().tokenize_batch_fused(buffer, splits, output) + }) + .map_err(fastokens::Error::Model)? + } else { + if let Some(pre_tokenizer) = tokenizer.pre_tokenizer() { + pre_tokenizer.pre_tokenize(&mut pretokenized)?; + } + pretokenized + .tokenize(|text, output| tokenizer.model().tokenize_into(text, output)) + .map_err(fastokens::Error::Model)? + }; + + Ok(tokenizer.post_process(ids, false)) +} + /// Tokenizer from `tokenizer.json` in HuggingFace format. /// /// This tries to load with `fastokens` first for better performance, then falls @@ -156,6 +244,17 @@ impl Tokenizer for HuggingFaceTokenizer { } } + fn encode_ordinary(&self, text: &str) -> Result> { + match &self.backend { + Backend::Hf(tokenizer) => encode_hf_ordinary(tokenizer, text) + .map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report())), + Backend::Fastokens(tokenizer) | Backend::FastokensByteLevel(tokenizer) => { + encode_fastokens_ordinary(tokenizer, text) + .map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report())) + } + } + } + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { match &self.backend { Backend::Hf(t) => t @@ -200,12 +299,19 @@ impl Tokenizer for HuggingFaceTokenizer { #[cfg(test)] mod tests { + use std::path::{Path, PathBuf}; + + use serde_json::{Value, json}; use tempfile::tempdir; use tokenizers::models::bpe::BPE; + use tokenizers::pre_tokenizers::byte_level::ByteLevel; use tokenizers::{AddedToken, Tokenizer as HfTokenizer}; use super::{HuggingFaceTokenizer, Tokenizer}; + const REGULAR_TOKEN: &str = "<|regular|>"; + const SPECIAL_TOKEN: &str = "<|special|>"; + fn tiny_bpe_tokenizer() -> HfTokenizer { let vocab = [ ("".to_string(), 0), @@ -232,6 +338,186 @@ mod tests { HfTokenizer::new(model) } + fn ordinary_test_tokenizer_json(fused: bool, with_added_tokens: bool) -> Value { + let mut alphabet: Vec = ByteLevel::alphabet().into_iter().collect(); + alphabet.sort_unstable(); + let vocab = alphabet + .into_iter() + .enumerate() + .map(|(id, token)| (token.to_string(), json!(id))) + .collect::>(); + + let pre_tokenizer = if fused { + json!({ + "type": "Sequence", + "pretokenizers": [ + { + "type": "Split", + "pattern": {"Regex": "\\S+|\\s+"}, + "behavior": "Isolated", + "invert": false + }, + { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": false + } + ] + }) + } else { + json!({ + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }) + }; + let added_tokens = with_added_tokens.then(|| { + json!([ + { + "id": 256, + "content": REGULAR_TOKEN, + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": false + }, + { + "id": 257, + "content": SPECIAL_TOKEN, + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ]) + }); + + json!({ + "version": "1.0", + "truncation": { + "direction": "Right", + "max_length": 24, + "strategy": "LongestFirst", + "stride": 0 + }, + "padding": null, + "added_tokens": added_tokens.unwrap_or_else(|| json!([])), + "normalizer": {"type": "NFC"}, + "pre_tokenizer": pre_tokenizer, + "post_processor": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": vocab, + "merges": [] + } + }) + } + + fn write_tokenizer_json(dir: &Path, name: &str, value: &Value) -> PathBuf { + let path = dir.join(name); + std::fs::write( + &path, + serde_json::to_vec(value).expect("serialize tokenizer"), + ) + .expect("write tokenizer"); + path + } + + fn assert_ordinary_matches_added_empty( + constructor: fn(&Path) -> crate::Result, + fused: bool, + ) { + let dir = tempdir().expect("create temp dir"); + let added_path = write_tokenizer_json( + dir.path(), + "with-added.json", + &ordinary_test_tokenizer_json(fused, true), + ); + let empty_path = write_tokenizer_json( + dir.path(), + "added-empty.json", + &ordinary_test_tokenizer_json(fused, false), + ); + let tokenizer = constructor(&added_path).expect("load tokenizer with added tokens"); + let added_empty = constructor(&empty_path).expect("load tokenizer with empty added tokens"); + + if let super::Backend::Fastokens(inner) | super::Backend::FastokensByteLevel(inner) = + &tokenizer.backend + { + assert_eq!(super::fastokens_fused_split(inner).is_some(), fused); + } + + assert_eq!( + tokenizer.encode(REGULAR_TOKEN, false).unwrap(), + vec![tokenizer.token_to_id(REGULAR_TOKEN).unwrap()] + ); + assert_eq!( + tokenizer.encode(SPECIAL_TOKEN, false).unwrap(), + vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()] + ); + + for text in [ + "", + "hello", + "Cafe\u{301}", + REGULAR_TOKEN, + SPECIAL_TOKEN, + "hello <|regular|> Cafe\u{301} <|special|> tail", + ] { + assert_eq!( + tokenizer.encode_ordinary(text).unwrap(), + added_empty.encode(text, false).unwrap(), + "fused={fused}, text={text:?}", + ); + } + if matches!(&tokenizer.backend, super::Backend::Hf(_)) { + assert_eq!( + tokenizer + .encode_ordinary("hello <|regular|> Cafe\u{301} <|special|> tail") + .unwrap() + .len(), + 24, + "HF post-processing must retain configured truncation", + ); + } + } + + #[test] + fn hf_ordinary_matches_original_encode_with_added_empty() { + for fused in [false, true] { + assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_hf, fused); + } + } + + #[test] + fn fastokens_ordinary_matches_original_encode_with_added_empty() { + for fused in [false, true] { + assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_fastokens, fused); + } + } + #[test] fn hf_constructor_resolves_added_token_ids() { let mut tokenizer = tiny_bpe_tokenizer(); diff --git a/rust/src/tokenizer/src/incremental.rs b/rust/src/tokenizer/src/incremental.rs index 5e470ae4b0..f608fa874b 100644 --- a/rust/src/tokenizer/src/incremental.rs +++ b/rust/src/tokenizer/src/incremental.rs @@ -199,6 +199,10 @@ mod tests { unreachable!() } + fn encode_ordinary(&self, _text: &str) -> Result> { + unreachable!() + } + fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result { let bytes = token_ids.iter().map(|id| *id as u8).collect::>(); Ok(String::from_utf8_lossy(&bytes).into_owned()) @@ -273,6 +277,10 @@ mod tests { unreachable!() } + fn encode_ordinary(&self, _text: &str) -> Result> { + unreachable!() + } + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { let mut text = String::new(); for &token_id in token_ids { @@ -410,6 +418,10 @@ mod tests { unreachable!() } + fn encode_ordinary(&self, _text: &str) -> Result> { + unreachable!() + } + fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result { match token_ids { [1] => Ok("abc".into()), diff --git a/rust/src/tokenizer/src/lib.rs b/rust/src/tokenizer/src/lib.rs index 6c9fcd3fde..0f8c7dc16e 100644 --- a/rust/src/tokenizer/src/lib.rs +++ b/rust/src/tokenizer/src/lib.rs @@ -25,6 +25,10 @@ pub trait Tokenizer: Send + Sync { /// Encode one prompt string into token IDs. fn encode(&self, text: &str, add_special_tokens: bool) -> Result>; + /// Equivalent to `encode(text, false)`, except that every added, + /// special, and control-token matcher is bypassed. + fn encode_ordinary(&self, text: &str) -> Result>; + /// Decode one token sequence into text. fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result; diff --git a/rust/src/tokenizer/src/tekken.rs b/rust/src/tokenizer/src/tekken.rs index 20b6c26ffc..5f9342e4a5 100644 --- a/rust/src/tokenizer/src/tekken.rs +++ b/rust/src/tokenizer/src/tekken.rs @@ -35,6 +35,12 @@ impl Tokenizer for TekkenTokenizer { .map_err(|error| tokenizer_error!("encoding failed: {error}")) } + fn encode_ordinary(&self, text: &str) -> Result> { + self.inner + .encode(text, false, false) + .map_err(|error| tokenizer_error!("encoding failed: {error}")) + } + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { let policy = if skip_special_tokens { tekken::SpecialTokenPolicy::Ignore @@ -67,3 +73,51 @@ impl Tokenizer for TekkenTokenizer { self.inner.is_special_token(token_id) } } + +#[cfg(test)] +mod tests { + use base64::Engine as _; + use tekken::config::TokenizerVersion; + use tekken::{SpecialTokenInfo, TokenInfo}; + + use super::*; + + fn test_tokenizer() -> TekkenTokenizer { + let vocab = (0_u8..=255) + .map(|byte| TokenInfo { + rank: byte as usize, + token_bytes: base64::engine::general_purpose::STANDARD.encode([byte]), + token_str: None, + }) + .collect(); + let special_tokens = vec![SpecialTokenInfo { + rank: 0, + token_str: "".to_string(), + is_control: true, + }]; + let inner = Tekkenizer::new( + vocab, + &special_tokens, + r"(?s).", + 257, + 1, + TokenizerVersion::V3, + None, + ) + .expect("build Tekken tokenizer"); + TekkenTokenizer { inner } + } + + #[test] + fn ordinary_matches_tekkens_empty_special_encoding() { + let tokenizer = test_tokenizer(); + let text = "user text"; + let control_id = tokenizer.token_to_id("").unwrap(); + let ordinary_ids = tokenizer.encode_ordinary(text).unwrap(); + + assert_eq!(control_id, 0); + assert_eq!(ordinary_ids, tokenizer.encode(text, false).unwrap()); + assert!(!ordinary_ids.contains(&control_id)); + assert_eq!(tokenizer.decode(&ordinary_ids, false).unwrap(), text); + } +} diff --git a/rust/src/tokenizer/src/test_utils.rs b/rust/src/tokenizer/src/test_utils.rs index 36d1f3d18a..6fdb6e0272 100644 --- a/rust/src/tokenizer/src/test_utils.rs +++ b/rust/src/tokenizer/src/test_utils.rs @@ -208,6 +208,10 @@ impl Tokenizer for TestTokenizer { Ok(ids) } + fn encode_ordinary(&self, text: &str) -> Result> { + Ok(text.as_bytes().iter().copied().map(u32::from).collect()) + } + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { let mut output = String::new(); let mut pending_bytes = Vec::new(); @@ -374,6 +378,32 @@ mod tests { assert!(!tokenizer.is_special_id(0xF002)); } + #[test] + fn ordinary_encoding_bypasses_all_configured_tokens() { + let tokenizer = TestTokenizer::new() + .with_bos_token("", 256) + .with_special_token("", 257) + .with_regular_token("", 258); + let ordinary_text = "user and "; + + assert_eq!(tokenizer.encode("", false).unwrap(), vec![257]); + assert_eq!(tokenizer.encode("", false).unwrap(), vec![258]); + assert_eq!( + tokenizer.encode_ordinary(ordinary_text).unwrap(), + ordinary_text.as_bytes().iter().copied().map(u32::from).collect::>() + ); + + let mut segmented = tokenizer.encode("", false).unwrap(); + segmented.extend(tokenizer.encode_ordinary(ordinary_text).unwrap()); + segmented.extend(tokenizer.encode("", false).unwrap()); + assert_eq!(segmented.first(), Some(&257)); + assert_eq!(segmented.last(), Some(&258)); + assert_eq!( + tokenizer.decode(&segmented, false).unwrap(), + format!("{ordinary_text}") + ); + } + #[test] #[should_panic(expected = "configured test token id 255 overlaps byte fallback range 0..=255")] fn configured_token_id_must_stay_outside_byte_range() { diff --git a/rust/src/tokenizer/src/tiktoken.rs b/rust/src/tokenizer/src/tiktoken.rs index d5ca119b80..0f355566b9 100644 --- a/rust/src/tokenizer/src/tiktoken.rs +++ b/rust/src/tokenizer/src/tiktoken.rs @@ -462,6 +462,13 @@ impl Tokenizer for TiktokenTokenizer { }) } + fn encode_ordinary(&self, text: &str) -> Result> { + Ok(match &self.backend { + Backend::Riptoken(backend) => backend.inner.encode_ordinary(text), + Backend::TiktokenRs(backend) => backend.inner.encode_ordinary(text), + }) + } + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { // Filter passes: // @@ -752,6 +759,39 @@ mod tests { } } + #[test] + fn tiktoken_ordinary_bypasses_every_registered_added_token() { + let dir = tempfile::tempdir().expect("create temp dir"); + let bpe_path = write_synthetic_bpe_file(dir.path()); + fs::write( + dir.path().join("tokenizer_config.json"), + r#"{ + "added_tokens_decoder": { + "257": { "content": "<|im_end|>", "special": true }, + "258": { "content": "<|tool_call_begin|>", "special": false } + } + }"#, + ) + .expect("write tokenizer_config.json"); + fs::write(dir.path().join("config.json"), r#"{"vocab_size": 260}"#) + .expect("write config.json"); + + let input = "<|im_end|><|tool_call_begin|><|reserved_token_259|>"; + let expected: Vec = input.as_bytes().iter().copied().map(u32::from).collect(); + for backend in explicit_backends(&bpe_path) { + assert_eq!(backend.encode("<|im_end|>", false).unwrap(), vec![257]); + assert_eq!( + backend.encode("<|tool_call_begin|>", false).unwrap(), + vec![258] + ); + assert_eq!( + backend.encode("<|reserved_token_259|>", false).unwrap(), + vec![259] + ); + assert_eq!(backend.encode_ordinary(input).unwrap(), expected); + } + } + /// `vocab_size` may live under `text_config` for composite (e.g. /// multimodal) configs. #[test]