From 22df3a34e098738b9e6fbd09f63967d60acc24c9 Mon Sep 17 00:00:00 2001 From: Salt Sato Date: Tue, 1 Sep 2026 02:25:06 +0100 Subject: [PATCH] [Perf][Rust Frontend] Count the tokenizer vocabulary once at construction (#54449) Co-authored-by: Bugen Zhao Co-authored-by: OpenAI Codex Signed-off-by: Feathbow Signed-off-by: Bugen Zhao --- rust/src/tokenizer/src/hf.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/rust/src/tokenizer/src/hf.rs b/rust/src/tokenizer/src/hf.rs index d936ac80c6..0d7769ce66 100644 --- a/rust/src/tokenizer/src/hf.rs +++ b/rust/src/tokenizer/src/hf.rs @@ -150,6 +150,7 @@ pub struct HuggingFaceTokenizer { backend: Backend, special_token_ids: Arc<[u32]>, added_vocab: Box<[(String, u32)]>, + vocab_size: usize, } impl HuggingFaceTokenizer { @@ -174,10 +175,13 @@ impl HuggingFaceTokenizer { ids.dedup(); Arc::from(ids) }; + // HF materializes the merged vocabulary to count it, so cache the result. + let vocab_size = tokenizer.get_vocab_size(true); Self { backend: Backend::Hf(Box::new(tokenizer)), special_token_ids, added_vocab, + vocab_size, } } @@ -205,6 +209,7 @@ impl HuggingFaceTokenizer { Arc::from(ids) }; let byte_level = tokenizer.decoder().is_some_and(is_byte_level_only); + let vocab_size = tokenizer.vocab_size(); let backend = if byte_level { Backend::FastokensByteLevel(Box::new(tokenizer)) } else { @@ -214,6 +219,7 @@ impl HuggingFaceTokenizer { backend, special_token_ids, added_vocab, + vocab_size, } } @@ -299,10 +305,7 @@ impl Tokenizer for HuggingFaceTokenizer { } fn vocab_size(&self) -> usize { - match &self.backend { - Backend::Hf(t) => t.get_vocab_size(true), - Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => t.vocab_size(), - } + self.vocab_size } fn id_to_token(&self, id: u32) -> Option { @@ -551,6 +554,20 @@ mod tests { } } + #[test] + fn hf_vocab_size_counts_added_tokens() { + let mut tokenizer = tiny_bpe_tokenizer(); + tokenizer.add_special_tokens(&[AddedToken::from("<|im_end|>", true)]); + let expected = tokenizer.get_vocab_size(true); + + let dir = tempdir().expect("create temp dir"); + let path = dir.path().join("tokenizer.json"); + tokenizer.save(&path, false).expect("save tokenizer json"); + + let wrapper = HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper"); + assert_eq!(wrapper.vocab_size(), expected); + } + #[test] fn hf_constructor_resolves_added_token_ids() { let mut tokenizer = tiny_bpe_tokenizer();