From 7a327f0b4f8f6886df77996d0e35ce547c9c2bd1 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 30 Jun 2026 22:34:43 +0800 Subject: [PATCH] [Rust Frontend] Simplify unit tests with shared `TestTokenizer` (#47125) Signed-off-by: Bugen Zhao --- rust/Cargo.lock | 1 + rust/src/chat/Cargo.toml | 1 + rust/src/chat/src/backend/hf.rs | 29 +- rust/src/chat/src/multimodal.rs | 66 +-- rust/src/chat/src/output/default/mod.rs | 41 +- rust/src/chat/src/parser/reasoning/tests.rs | 27 +- rust/src/chat/src/parser/unified.rs | 37 +- rust/src/chat/tests/chat.rs | 137 ++---- rust/src/parser/Cargo.toml | 1 + rust/src/parser/benches/utils/adapter.rs | 4 + rust/src/parser/src/reasoning/seed_oss.rs | 27 +- rust/src/parser/src/reasoning/step3p5.rs | 37 +- rust/src/parser/src/reasoning/tests.rs | 108 ++--- rust/src/parser/src/unified/combined.rs | 37 +- rust/src/parser/src/unified/gemma4.rs | 81 +--- rust/src/server/Cargo.toml | 1 + rust/src/server/src/grpc/tests.rs | 33 +- .../server/src/routes/http_client_tests.rs | 33 +- .../src/routes/openai/completions/convert.rs | 54 +-- rust/src/server/src/routes/tests.rs | 167 +------ rust/src/text/Cargo.toml | 1 + rust/src/text/src/lower.rs | 61 +-- rust/src/text/src/output/decoded.rs | 32 +- rust/src/text/src/output/logprobs.rs | 33 +- rust/src/tokenizer/Cargo.toml | 3 + rust/src/tokenizer/src/incremental.rs | 12 + rust/src/tokenizer/src/lib.rs | 8 +- rust/src/tokenizer/src/test_utils.rs | 434 ++++++++++++++++++ 28 files changed, 680 insertions(+), 826 deletions(-) create mode 100644 rust/src/tokenizer/src/test_utils.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index da782e41e6..7b6cb928e1 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5310,6 +5310,7 @@ dependencies = [ "vllm-llm", "vllm-metrics", "vllm-text", + "vllm-tokenizer", "zeromq", ] diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 40498bac3f..95ce5ff2e4 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -50,6 +50,7 @@ tokio.workspace = true tracing-subscriber.workspace = true uuid.workspace = true vllm-engine-core-client = { workspace = true, features = ["test-util"] } +vllm-tokenizer = { workspace = true, features = ["test-utils"] } zeromq.workspace = true [lints] diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 9dff25ea49..9997bc2473 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -154,7 +154,8 @@ mod tests { use thiserror_ext::AsReport as _; use vllm_text::Prompt; use vllm_text::backend::hf::TokenizerSource; - use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; + use vllm_text::tokenizer::DynTokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::HfChatBackend; use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions}; @@ -196,32 +197,8 @@ mod tests { } } - struct TestTokenizer; - - impl Tokenizer for TestTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(Vec::new()) - } - - fn decode( - &self, - _token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok(String::new()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } - } - fn test_tokenizer() -> DynTokenizer { - Arc::new(TestTokenizer) + Arc::new(TestTokenizer::new()) } fn backend_for_selection( diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 024e4b63ea..8fd44376f9 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -563,7 +563,7 @@ mod tests { use llm_multimodal::TokenId; use vllm_engine_core_client::protocol::tensor::WireArrayData; - use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError}; + use vllm_tokenizer::test_utils::TestTokenizer; use super::*; @@ -574,60 +574,14 @@ mod tests { const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093; const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094; - struct TestTokenizer; - - impl Tokenizer for TestTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> std::result::Result, TokenizerError> { - Ok(match text { - "<|image|>" => vec![LLAMA4_IMAGE_ID], - text => text.bytes().map(u32::from).collect(), - }) - } - - fn decode( - &self, - _token_ids: &[u32], - _skip_special_tokens: bool, - ) -> std::result::Result { - Ok(String::new()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "<|image_start|>" => Some(LLAMA4_IMAGE_START_ID), - "<|image_end|>" => Some(LLAMA4_IMAGE_END_ID), - "<|image|>" => Some(LLAMA4_IMAGE_ID), - "<|patch|>" => Some(LLAMA4_PATCH_ID), - "<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID), - "<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID), - _ => None, - } - } - - fn id_to_token(&self, id: u32) -> Option { - match id { - LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()), - LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()), - LLAMA4_IMAGE_ID => Some("<|image|>".to_string()), - LLAMA4_PATCH_ID => Some("<|patch|>".to_string()), - LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()), - LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()), - _ => None, - } - } - - fn create_decode_stream( - &self, - _prompt_token_ids: &[u32], - _skip_special_tokens: bool, - _min_bytes_to_buffer: usize, - ) -> Box { - unreachable!("not used") - } + fn llama4_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID) + .with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID) + .with_regular_token("<|image|>", LLAMA4_IMAGE_ID) + .with_regular_token("<|patch|>", LLAMA4_PATCH_ID) + .with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID) + .with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID) } fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo { @@ -635,7 +589,7 @@ mod tests { model_id: format!("{model_type}-test"), model_type: Some(model_type.to_string()), config, - tokenizer: TokenizerResolver(Arc::new(TestTokenizer)), + tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())), }; let spec = context .resolve_model_spec() diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index c494df600f..b42d24dcaa 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { mod tests { use std::sync::Arc; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::DefaultChatOutputProcessor; use crate::Error; use crate::parser::ParserSelection; use crate::request::ChatRequest; - struct FakeTokenizer; - - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "<|channel>" => Some(1), - "" => Some(2), - _ => None, - } - } - } - - fn tokenizer() -> Arc { - Arc::new(FakeTokenizer) + fn tokenizer() -> Arc { + Arc::new( + TestTokenizer::new() + .with_regular_token("<|channel>", 256) + .with_regular_token("", 257), + ) } #[test] diff --git a/rust/src/chat/src/parser/reasoning/tests.rs b/rust/src/chat/src/parser/reasoning/tests.rs index e6255d14a0..b6ae5ba9c3 100644 --- a/rust/src/chat/src/parser/reasoning/tests.rs +++ b/rust/src/chat/src/parser/reasoning/tests.rs @@ -1,32 +1,9 @@ use std::sync::Arc; -use vllm_tokenizer::Tokenizer; +use vllm_tokenizer::test_utils::TestTokenizer; use super::{ReasoningParserFactory, names}; -struct FakeTokenizer; - -impl Tokenizer for FakeTokenizer { - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } -} - #[test] fn factory_contains_and_lists_registered_parsers() { let factory = ReasoningParserFactory::new(); @@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() { #[test] fn factory_rejects_unknown_parser_names() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(TestTokenizer::new()); let factory = ReasoningParserFactory::new(); let error = match factory.create("missing", tokenizer) { Ok(_) => panic!("expected parser lookup to fail"), diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index 6456cfda75..50f2104d61 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -75,39 +75,14 @@ impl UnifiedParserFactory { mod tests { use std::sync::Arc; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::{UnifiedParserFactory, names}; - struct FakeTokenizer; - - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "<|channel>" => Some(1), - "" => Some(2), - _ => None, - } - } + fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("<|channel>", 256) + .with_regular_token("", 257) } #[test] @@ -119,6 +94,6 @@ mod tests { factory.resolve_name_for_model("google/gemma-4-27b-it"), Some(names::GEMMA4) ); - factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap(); + factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap(); } } diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index 5c4a2c29b7..884398a16c 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -21,15 +21,17 @@ use vllm_engine_core_client::protocol::{ use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig}; use vllm_llm::Llm; -use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; +use vllm_text::tokenizer::DynTokenizer; use vllm_text::{ DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, Prompt, TextBackend, }; +use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; const SPECIAL_STOP_TOKEN_ID: u32 = 256; +const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000; fn request_output( request_id: &str, @@ -158,45 +160,18 @@ async fn connect_chat_llm_with_ipc( struct FakeChatBackend { has_template: bool, model_id: String, + tokenizer: DynTokenizer, } -#[derive(Debug)] -struct FakeChatTokenizer; - -impl Tokenizer for FakeChatTokenizer { - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result> { - Ok(text.bytes().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - let bytes = token_ids - .iter() - .filter_map(|id| { - if skip_special_tokens && *id == SPECIAL_STOP_TOKEN_ID { - None - } else { - Some(*id as u8) - } - }) - .collect::>(); - Ok(String::from_utf8_lossy(&bytes).into_owned()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(0xF001), - "" => Some(0xF002), - "<|START_THINKING|>" => Some(0xF003), - "<|END_THINKING|>" => Some(0xF004), - "◁think▷" => Some(0xF005), - "◁/think▷" => Some(0xF006), - _ => None, - } - } +fn fake_chat_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_special_token("", SPECIAL_STOP_TOKEN_ID) + .with_regular_token("", 0xF001) + .with_regular_token("", 0xF002) + .with_regular_token("<|START_THINKING|>", 0xF003) + .with_regular_token("<|END_THINKING|>", 0xF004) + .with_regular_token("◁think▷", 0xF005) + .with_regular_token("◁/think▷", 0xF006) } impl fmt::Debug for FakeChatBackend { @@ -210,6 +185,7 @@ impl FakeChatBackend { Self { has_template: true, model_id: "test-model".to_string(), + tokenizer: Arc::new(fake_chat_tokenizer()), } } @@ -217,6 +193,7 @@ impl FakeChatBackend { Self { has_template: false, model_id: "test-model".to_string(), + tokenizer: Arc::new(fake_chat_tokenizer()), } } @@ -224,13 +201,19 @@ impl FakeChatBackend { Self { has_template: true, model_id: model_id.into(), + tokenizer: Arc::new(fake_chat_tokenizer()), } } + + fn with_tokenizer(mut self, tokenizer: DynTokenizer) -> Self { + self.tokenizer = tokenizer; + self + } } impl TextBackend for FakeChatBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(FakeChatTokenizer) + Arc::clone(&self.tokenizer) } fn model_id(&self) -> &str { @@ -282,65 +265,6 @@ impl ChatRenderer for FakeChatBackend { } } -#[derive(Clone, Debug)] -struct FailingDecodeBackend { - inner: FakeChatBackend, -} - -#[derive(Debug)] -struct FailingDecodeTokenizer; - -impl Tokenizer for FailingDecodeTokenizer { - fn encode(&self, text: &str, add_special_tokens: bool) -> vllm_tokenizer::Result> { - FakeChatTokenizer.encode(text, add_special_tokens) - } - - fn decode( - &self, - token_ids: &[u32], - skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - if token_ids.contains(&(b'i' as u32)) { - return Err(vllm_tokenizer::TokenizerError("decode failed".to_string())); - } - FakeChatTokenizer.decode(token_ids, skip_special_tokens) - } - - fn token_to_id(&self, token: &str) -> Option { - FakeChatTokenizer.token_to_id(token) - } -} - -impl TextBackend for FailingDecodeBackend { - fn tokenizer(&self) -> DynTokenizer { - Arc::new(FailingDecodeTokenizer) - } - - fn model_id(&self) -> &str { - self.inner.model_id() - } -} - -impl ChatBackend for FailingDecodeBackend { - fn chat_renderer(&self) -> DynChatRenderer { - Arc::new(self.clone()) - } - - fn new_chat_output_processor( - &self, - _request: &mut ChatRequest, - _options: NewChatOutputProcessorOptions<'_>, - ) -> vllm_chat::Result { - Ok(Box::new(DefaultChatOutputProcessor::plain_text_only())) - } -} - -impl ChatRenderer for FailingDecodeBackend { - fn render(&self, request: &ChatRequest) -> vllm_chat::Result { - self.inner.render(request) - } -} - /// Skip `LogprobsDelta` events that carry only token_ids (no logprobs), /// returning the next semantically interesting event. async fn next_semantic(stream: &mut S) -> Option> @@ -738,7 +662,12 @@ async fn chat_stream_reports_decode_failure_as_error_event() { send_outputs( push, EngineCoreOutputs { - outputs: vec![request_output("chat-4", vec![b'i' as u32], None, None)], + outputs: vec![request_output( + "chat-4", + vec![UNKNOWN_DECODE_TOKEN_ID], + None, + None, + )], ..Default::default() }, ) @@ -747,9 +676,8 @@ async fn chat_stream_reports_decode_failure_as_error_event() { }, ); - let backend: Arc = Arc::new(FailingDecodeBackend { - inner: FakeChatBackend::new(), - }); + let backend: Arc = + Arc::new(FakeChatBackend::new().with_tokenizer(Arc::new(TestTokenizer::new()))); let chat = connect_chat_llm_with_ipc( EngineCoreClientConfig::new_single(handshake_address), &ipc, @@ -769,7 +697,10 @@ async fn chat_stream_reports_decode_failure_as_error_event() { match timeout(Duration::from_secs(2), stream.next()).await.unwrap() { Some(Err(vllm_chat::Error::Text(vllm_text::Error::Tokenizer(message)))) => { - assert_eq!(message, "decode failed"); + assert_eq!( + message, + format!("test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}") + ); } other => panic!("unexpected event after close: {other:?}"), } diff --git a/rust/src/parser/Cargo.toml b/rust/src/parser/Cargo.toml index 09c3d6b5dc..3c49362c5d 100644 --- a/rust/src/parser/Cargo.toml +++ b/rust/src/parser/Cargo.toml @@ -23,6 +23,7 @@ expect-test.workspace = true futures.workspace = true openai-protocol.workspace = true tool-parser.workspace = true +vllm-tokenizer = { workspace = true, features = ["test-utils"] } [[bench]] name = "deepseek_v3" diff --git a/rust/src/parser/benches/utils/adapter.rs b/rust/src/parser/benches/utils/adapter.rs index 103ed18d09..20f8977441 100644 --- a/rust/src/parser/benches/utils/adapter.rs +++ b/rust/src/parser/benches/utils/adapter.rs @@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer { fn token_to_id(&self, _token: &str) -> Option { Some(u32::MAX) } + + fn id_to_token(&self, _id: u32) -> Option { + Some("\u{FFFD}".to_string()) + } } /// Bench-only adapter that exposes a unified parser through the tool-parser diff --git a/rust/src/parser/src/reasoning/seed_oss.rs b/rust/src/parser/src/reasoning/seed_oss.rs index eb996f8477..580e95e795 100644 --- a/rust/src/parser/src/reasoning/seed_oss.rs +++ b/rust/src/parser/src/reasoning/seed_oss.rs @@ -49,11 +49,14 @@ mod tests { use std::sync::Arc; use super::SeedOssReasoningParser; - use crate::reasoning::{ReasoningParser, tests::FakeTokenizer}; + use crate::reasoning::{ + ReasoningParser, + tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer}, + }; #[test] fn without_prompt_markers_expects_start_token() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("implicit reasoninganswer").unwrap(); @@ -66,10 +69,10 @@ mod tests { #[test] fn picks_up_prompt_start_boundary() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); - // Prompt prefills `` (id 10), opening reasoning before the stream. - parser.initialize(&[10]).unwrap(); + // Prompt prefills ``, opening reasoning before the stream. + parser.initialize(&[SEED_THINK_START_ID]).unwrap(); let delta = parser.push("reasonanswer").unwrap(); assert_eq!(delta.reasoning.as_deref(), Some("reason")); @@ -78,10 +81,10 @@ mod tests { #[test] fn respects_prompt_end_boundary() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); - // Prompt already closed reasoning with `` (id 11). - parser.initialize(&[11]).unwrap(); + // Prompt already closed reasoning with ``. + parser.initialize(&[SEED_THINK_END_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); @@ -91,7 +94,7 @@ mod tests { #[test] fn handles_explicit_start_token() { // An explicit start delimiter must not leak into reasoning text. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasonanswer").unwrap(); @@ -103,7 +106,7 @@ mod tests { fn streams_explicit_start_token_across_pushes() { // Start token, reasoning body, end token, and content arrive in separate // streaming deltas. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); let mut reasoning = String::new(); @@ -131,9 +134,9 @@ mod tests { #[test] fn handles_partial_delimiters_across_pushes() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[10]).unwrap(); + parser.initialize(&[SEED_THINK_START_ID]).unwrap(); // Closing delimiter `` arrives in two halves. let first = parser.push("reason` (id 1), opening reasoning before the stream. - parser.initialize(&[1]).unwrap(); + // Prompt prefills ``, opening reasoning before the stream. + parser.initialize(&[THINK_START_ID]).unwrap(); let delta = parser.push("This is a reasoning sectionThis is the rest").unwrap(); assert_eq!( @@ -146,7 +149,7 @@ mod tests { #[test] fn handles_unterminated_reasoning() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let pushed = parser.push("reason without end").unwrap(); @@ -159,7 +162,7 @@ mod tests { #[test] fn handles_empty_input() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let pushed = parser.push("").unwrap(); @@ -172,9 +175,9 @@ mod tests { fn complex_newline_pattern_trims_only_single_framing_newline_each_side() { // Only the immediately-adjacent framing `\n` is dropped on each side of // ``; surrounding newlines remain part of reasoning/content. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[1]).unwrap(); + parser.initialize(&[THINK_START_ID]).unwrap(); let delta = parser .push("\n This is a \n reasoning section\n\n\n\n\nThis is the rest") @@ -188,7 +191,7 @@ mod tests { #[test] fn drops_framing_newlines_in_single_push() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reason\n\nanswer").unwrap(); @@ -198,7 +201,7 @@ mod tests { #[test] fn drops_framing_newlines_across_pushes() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); // The trailing `\n` from the first push is held until we know whether @@ -219,7 +222,7 @@ mod tests { #[test] fn replays_held_newline_when_more_reasoning_follows() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let first = parser.push("reason\n").unwrap(); @@ -232,7 +235,7 @@ mod tests { #[test] fn finish_flushes_held_newline_in_unterminated_stream() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let first = parser.push("reason\n").unwrap(); @@ -245,7 +248,7 @@ mod tests { #[test] fn preserves_inner_newlines_in_reasoning() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("line1\nline2tail").unwrap(); @@ -257,7 +260,7 @@ mod tests { fn trims_only_one_trailing_reasoning_newline() { // Only the single framing newline immediately before `` is // dropped; earlier newlines in the reasoning body are preserved. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reason\n\nanswer").unwrap(); @@ -269,7 +272,7 @@ mod tests { fn drops_only_first_content_newline_after_transition() { // The leading-`\n` drop applies only to the first content delta after // ``; later deltas pass through untouched. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let first = parser.push("reason").unwrap(); @@ -288,7 +291,7 @@ mod tests { #[test] fn passes_through_clean_boundary_without_framing_newlines() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasontail").unwrap(); @@ -298,7 +301,7 @@ mod tests { #[test] fn handles_empty_reasoning_section() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("answer").unwrap(); diff --git a/rust/src/parser/src/reasoning/tests.rs b/rust/src/parser/src/reasoning/tests.rs index 22c026d358..5c148079e3 100644 --- a/rust/src/parser/src/reasoning/tests.rs +++ b/rust/src/parser/src/reasoning/tests.rs @@ -1,54 +1,42 @@ use std::sync::Arc; -use vllm_tokenizer::Tokenizer; +use vllm_tokenizer::test_utils::TestTokenizer; use super::{ DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser, Qwen3ReasoningParser, ReasoningParser, }; -pub(crate) struct FakeTokenizer; +pub(crate) const THINK_START_ID: u32 = 256; +pub(crate) const THINK_END_ID: u32 = 257; +pub(crate) const START_THINKING_ID: u32 = 258; +pub(crate) const END_THINKING_ID: u32 = 259; +pub(crate) const MINIMAX_THINK_START_ID: u32 = 260; +pub(crate) const MINIMAX_THINK_END_ID: u32 = 261; +pub(crate) const SPECIAL_BOUNDARY_ID: u32 = 262; +pub(crate) const MM_THINK_START_ID: u32 = 263; +pub(crate) const MM_THINK_END_ID: u32 = 264; +pub(crate) const SEED_THINK_START_ID: u32 = 265; +pub(crate) const SEED_THINK_END_ID: u32 = 266; -impl Tokenizer for FakeTokenizer { - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(1), - "" => Some(2), - "<|START_THINKING|>" => Some(3), - "<|END_THINKING|>" => Some(4), - "◁think▷" => Some(5), - "◁/think▷" => Some(6), - "" => Some(8), - "" => Some(9), - "" => Some(10), - "" => Some(11), - _ => None, - } - } - - fn is_special_id(&self, token_id: u32) -> bool { - token_id == 7 - } +pub(crate) fn fake_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("", THINK_START_ID) + .with_regular_token("", THINK_END_ID) + .with_regular_token("<|START_THINKING|>", START_THINKING_ID) + .with_regular_token("<|END_THINKING|>", END_THINKING_ID) + .with_regular_token("◁think▷", MINIMAX_THINK_START_ID) + .with_regular_token("◁/think▷", MINIMAX_THINK_END_ID) + .with_special_token("", SPECIAL_BOUNDARY_ID) + .with_regular_token("", MM_THINK_START_ID) + .with_regular_token("", MM_THINK_END_ID) + .with_regular_token("", SEED_THINK_START_ID) + .with_regular_token("", SEED_THINK_END_ID) } #[test] fn delimited_content_only_stream() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); @@ -60,7 +48,7 @@ fn delimited_content_only_stream() { #[test] fn delimited_single_chunk_with_reasoning_and_content() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); @@ -71,7 +59,7 @@ fn delimited_single_chunk_with_reasoning_and_content() { #[test] fn delimited_partial_tokens_across_chunks() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); @@ -83,10 +71,10 @@ fn delimited_partial_tokens_across_chunks() { #[test] fn delimited_finish_flushes_buffer() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); - parser.initialize(&[1]); + parser.initialize(&[THINK_START_ID]); let delta = parser.push("unfinishedanswer").unwrap(); @@ -106,9 +94,9 @@ fn qwen3_without_prompt_markers_expects_start_token() { #[test] fn qwen3_prompt_end_marker_starts_in_content() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[2]).unwrap(); + parser.initialize(&[THINK_END_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); @@ -117,7 +105,7 @@ fn qwen3_prompt_end_marker_starts_in_content() { #[test] fn qwen3_tolerates_old_and_new_formats() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap(); let old = old_parser.push("reasonanswer").unwrap(); @@ -125,7 +113,7 @@ fn qwen3_tolerates_old_and_new_formats() { assert_eq!(old.content.as_deref(), Some("answer")); let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap(); - new_parser.initialize(&[1]).unwrap(); + new_parser.initialize(&[THINK_START_ID]).unwrap(); let new = new_parser.push("reasonanswer").unwrap(); assert_eq!(new.reasoning.as_deref(), Some("reason")); assert_eq!(new.content.as_deref(), Some("answer")); @@ -133,10 +121,10 @@ fn qwen3_tolerates_old_and_new_formats() { #[test] fn qwen3_stops_scanning_at_last_special_token() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[1, 7]).unwrap(); + parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); @@ -145,7 +133,7 @@ fn qwen3_stops_scanning_at_last_special_token() { #[test] fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasonanswer").unwrap(); @@ -155,10 +143,10 @@ fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() { #[test] fn deepseek_r1_stops_scanning_at_last_special_token() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[2, 7]).unwrap(); + parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap(); let delta = parser.push("reasonanswer").unwrap(); assert_eq!(delta.reasoning.as_deref(), Some("reason")); @@ -167,7 +155,7 @@ fn deepseek_r1_stops_scanning_at_last_special_token() { #[test] fn minimax_m3_handles_explicit_think_delimiters() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasonanswer").unwrap(); @@ -177,7 +165,7 @@ fn minimax_m3_handles_explicit_think_delimiters() { #[test] fn minimax_m3_drops_leading_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("answer").unwrap(); @@ -187,7 +175,7 @@ fn minimax_m3_drops_leading_end_marker() { #[test] fn minimax_m3_preserves_non_leading_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("XXXYYY").unwrap(); @@ -197,7 +185,7 @@ fn minimax_m3_preserves_non_leading_end_marker() { #[test] fn minimax_m3_drops_split_leading_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); assert!(parser.push("answer").unwrap(); assert_eq!(delta.reasoning.as_deref(), Some("reason")); @@ -219,9 +207,9 @@ fn minimax_m3_uses_prompt_prefilled_start_marker() { #[test] fn minimax_m3_uses_prompt_prefilled_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[9]).unwrap(); + parser.initialize(&[MM_THINK_END_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs index 3f1c669013..f06334c72e 100644 --- a/rust/src/parser/src/unified/combined.rs +++ b/rust/src/parser/src/unified/combined.rs @@ -124,42 +124,17 @@ impl UnifiedParser for CombinedParser { mod tests { use std::sync::Arc; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::CombinedParser; use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser}; use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser}; use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; - struct FakeTokenizer; - - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(1), - "" => Some(2), - _ => None, - } - } + fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("", 256) + .with_regular_token("", 257) } fn test_tools() -> Vec { @@ -273,7 +248,7 @@ mod tests { #[test] fn combined_parser_emits_reasoning_and_text() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(tokenizer()); let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap(); let mut parser = CombinedParser::new(Some(reasoning), None); diff --git a/rust/src/parser/src/unified/gemma4.rs b/rust/src/parser/src/unified/gemma4.rs index 3276088913..71541f94c6 100644 --- a/rust/src/parser/src/unified/gemma4.rs +++ b/rust/src/parser/src/unified/gemma4.rs @@ -515,7 +515,7 @@ mod tests { use serde_json::{Value, json}; use thiserror_ext::AsReport; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use winnow::combinator::{eof, terminated}; use winnow::error::ErrMode; use winnow::prelude::*; @@ -527,66 +527,15 @@ mod tests { use crate::tool::Tool; use crate::unified::{UnifiedParserEvent, parsing_failed}; - struct FakeTokenizer; + const CHANNEL_START_ID: u32 = 256; + const CHANNEL_END_ID: u32 = 257; + const TURN_BOUNDARY_ID: u32 = 258; - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - CHANNEL_START => Some(100), - CHANNEL_END => Some(101), - _ => None, - } - } - - fn is_special_id(&self, token_id: u32) -> bool { - matches!(token_id, 100..=105) - } - } - - struct MissingTokenTokenizer; - - impl Tokenizer for MissingTokenTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } + fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_special_token(CHANNEL_START, CHANNEL_START_ID) + .with_special_token(CHANNEL_END, CHANNEL_END_ID) + .with_special_token("", TURN_BOUNDARY_ID) } trait UnifiedParserTestExt { @@ -716,12 +665,12 @@ mod tests { } fn test_parser() -> Gemma4UnifiedParser { - Gemma4UnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap() + Gemma4UnifiedParser::new(&test_tools(), Arc::new(tokenizer())).unwrap() } #[test] fn gemma4_create_requires_channel_start_token() { - let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(MissingTokenTokenizer)) { + let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(TestTokenizer::new())) { Ok(_) => panic!("expected missing token error"), Err(error) => error, }; @@ -1046,7 +995,7 @@ mod tests { #[test] fn gemma4_initialize_open_channel_prompt_starts_in_reasoning() { let mut parser = test_parser(); - parser.initialize(&[100, 3000, 3001]).unwrap(); + parser.initialize(&[CHANNEL_START_ID, 3000, 3001]).unwrap(); let output = parser.parse_complete("reasonanswer").unwrap(); @@ -1057,7 +1006,7 @@ mod tests { #[test] fn gemma4_initialize_turn_prompt_starts_in_text() { let mut parser = test_parser(); - parser.initialize(&[104, 3000, 3001]).unwrap(); + parser.initialize(&[TURN_BOUNDARY_ID, 3000, 3001]).unwrap(); let output = parser.parse_complete("<|channel>thought\nreasonanswer").unwrap(); @@ -1068,7 +1017,7 @@ mod tests { #[test] fn gemma4_initialize_special_token_caps_boundary_scan() { let mut parser = test_parser(); - parser.initialize(&[100, 3000, 104, 3001]).unwrap(); + parser.initialize(&[CHANNEL_START_ID, 3000, TURN_BOUNDARY_ID, 3001]).unwrap(); let output = parser.parse_complete("answer").unwrap(); @@ -1079,7 +1028,7 @@ mod tests { #[test] fn gemma4_initialize_closed_channel_prompt_starts_in_text() { let mut parser = test_parser(); - parser.initialize(&[100, 3000, 3001, 101]).unwrap(); + parser.initialize(&[CHANNEL_START_ID, 3000, 3001, CHANNEL_END_ID]).unwrap(); let output = parser.parse_complete("answer").unwrap(); diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index 00183d7f9c..f3e03863d4 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -64,6 +64,7 @@ tempfile.workspace = true tokio = { workspace = true, features = ["test-util"] } tower.workspace = true vllm-engine-core-client = { workspace = true, features = ["test-util"] } +vllm-tokenizer = { workspace = true, features = ["test-utils"] } zeromq.workspace = true [lints] diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 65eedbf7e8..48de928cfe 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{ use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::Llm; -use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; +use vllm_text::tokenizer::DynTokenizer; use vllm_text::{Prompt, TextBackend}; +use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; @@ -155,37 +156,9 @@ fn test_llm(client: EngineCoreClient) -> Llm { #[derive(Clone, Debug)] struct FakeTextBackend; -#[derive(Debug)] -struct FakeTokenizer; - -impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(text.bytes().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, token: &str) -> Option { - token.bytes().next().map(u32::from) - } -} - impl TextBackend for FakeTextBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(FakeTokenizer) + Arc::new(TestTokenizer::new()) } fn model_id(&self) -> &str { diff --git a/rust/src/server/src/routes/http_client_tests.rs b/rust/src/server/src/routes/http_client_tests.rs index b23c95fa6d..b3c6977f5e 100644 --- a/rust/src/server/src/routes/http_client_tests.rs +++ b/rust/src/server/src/routes/http_client_tests.rs @@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{ use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::Llm; -use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; +use vllm_text::tokenizer::DynTokenizer; use vllm_text::{Prompt, TextBackend}; +use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; @@ -151,37 +152,9 @@ fn test_llm(client: EngineCoreClient) -> Llm { #[derive(Clone, Debug)] struct FakeChatBackend; -#[derive(Debug)] -struct FakeChatTokenizer; - -impl Tokenizer for FakeChatTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(text.bytes().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, token: &str) -> Option { - token.bytes().next().map(u32::from) - } -} - impl TextBackend for FakeChatBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(FakeChatTokenizer) + Arc::new(TestTokenizer::new()) } fn model_id(&self) -> &str { diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 0541346438..46d552385a 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -194,7 +194,7 @@ mod tests { use axum::http::HeaderMap; use serde_json::json; use vllm_text::Prompt; - use vllm_text::tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::prepare_completion_request; use crate::lora::LoraModelResolution; @@ -212,32 +212,8 @@ mod tests { } } - #[derive(Debug)] - struct TestTokenizer; - - impl Tokenizer for TestTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(text.bytes().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } + fn test_tokenizer() -> TestTokenizer { + TestTokenizer::new() } fn base_request_json() -> serde_json::Value { @@ -297,7 +273,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -340,7 +316,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare") .text_request @@ -374,7 +350,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -399,7 +375,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -422,7 +398,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -446,7 +422,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -471,7 +447,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -498,7 +474,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -523,7 +499,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -549,7 +525,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1)); @@ -574,7 +550,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), request_context(&headers, None), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); assert_eq!(prepared.text_request.data_parallel_rank, Some(3)); @@ -593,7 +569,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); assert_eq!(prepared.text_request.data_parallel_rank, None); diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index d7e99e8fa9..b40542a746 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -40,8 +40,9 @@ use vllm_engine_core_client::{ }; use vllm_llm::Llm; use vllm_metrics::METRICS; -use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; +use vllm_text::tokenizer::DynTokenizer; use vllm_text::{Prompt, TextBackend}; +use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; @@ -417,82 +418,20 @@ struct FakeChatBackend { } /// Synthetic BOS id used when `add_special_tokens` is true in tests. -const FAKE_BOS_TOKEN_ID: u32 = 1; +const FAKE_BOS_TOKEN_ID: u32 = 256; +const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000; -#[derive(Debug)] -struct FakeChatTokenizer; - -impl Tokenizer for FakeChatTokenizer { - fn encode( - &self, - text: &str, - add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - let mut token_ids = Vec::new(); - if add_special_tokens { - token_ids.push(FAKE_BOS_TOKEN_ID); - } - let mut rest = text; - while !rest.is_empty() { - if let Some(stripped) = rest.strip_prefix("") { - token_ids.push(999); - rest = stripped; - continue; - } - if let Some(stripped) = rest.strip_prefix("<|image_pad|>") { - token_ids.push(151655); - rest = stripped; - continue; - } - - let ch = rest.chars().next().expect("rest is not empty"); - let mut buf = [0; 4]; - token_ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from)); - rest = &rest[ch.len_utf8()..]; - } - Ok(token_ids) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(999), - "<|image_pad|>" => Some(151655), - "" => Some(0xF001), - "" => Some(0xF002), - "<|START_THINKING|>" => Some(0xF003), - "<|END_THINKING|>" => Some(0xF004), - "◁think▷" => Some(0xF005), - "◁/think▷" => Some(0xF006), - _ => None, - } - } - - fn id_to_token(&self, id: u32) -> Option { - match id { - FAKE_BOS_TOKEN_ID => Some("".to_string()), - 999 => Some("".to_string()), - 151655 => Some("<|image_pad|>".to_string()), - 0xF001 => Some("".to_string()), - 0xF002 => Some("".to_string()), - 0xF003 => Some("<|START_THINKING|>".to_string()), - 0xF004 => Some("<|END_THINKING|>".to_string()), - 0xF005 => Some("◁think▷".to_string()), - 0xF006 => Some("◁/think▷".to_string()), - id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()), - _ => None, - } - } +fn fake_chat_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_bos_token("", FAKE_BOS_TOKEN_ID) + .with_regular_token("", 999) + .with_regular_token("<|image_pad|>", 151655) + .with_regular_token("", 0xF001) + .with_regular_token("", 0xF002) + .with_regular_token("<|START_THINKING|>", 0xF003) + .with_regular_token("<|END_THINKING|>", 0xF004) + .with_regular_token("◁think▷", 0xF005) + .with_regular_token("◁/think▷", 0xF006) } impl FakeChatBackend { @@ -530,7 +469,7 @@ impl fmt::Debug for FakeChatBackend { impl TextBackend for FakeChatBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(FakeChatTokenizer) + Arc::new(fake_chat_tokenizer()) } fn model_id(&self) -> &str { @@ -630,7 +569,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo { Some("qwen2_vl".to_string()), Some(&config_path), None, - Arc::new(FakeChatTokenizer), + Arc::new(fake_chat_tokenizer()), ) .expect("load multimodal info") .expect("qwen multimodal info is registered"); @@ -638,70 +577,6 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo { info } -#[derive(Clone, Debug)] -struct FailingDecodeChatBackend; - -#[derive(Debug)] -struct FailingDecodeTokenizer; - -impl Tokenizer for FailingDecodeTokenizer { - fn encode( - &self, - text: &str, - add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - FakeChatTokenizer.encode(text, add_special_tokens) - } - - fn decode( - &self, - token_ids: &[u32], - skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - if token_ids.contains(&(b'i' as u32)) { - return Err(vllm_text::tokenizer::TokenizerError( - "forced decode failure for streaming test".to_string(), - )); - } - - FakeChatTokenizer.decode(token_ids, skip_special_tokens) - } - - fn token_to_id(&self, token: &str) -> Option { - FakeChatTokenizer.token_to_id(token) - } -} - -impl TextBackend for FailingDecodeChatBackend { - fn tokenizer(&self) -> DynTokenizer { - Arc::new(FailingDecodeTokenizer) - } - - fn model_id(&self) -> &str { - "test-model" - } -} - -impl ChatBackend for FailingDecodeChatBackend { - fn chat_renderer(&self) -> DynChatRenderer { - Arc::new(self.clone()) - } - - fn new_chat_output_processor( - &self, - _request: &mut ChatRequest, - _options: NewChatOutputProcessorOptions<'_>, - ) -> vllm_chat::Result { - Ok(Box::new(DefaultChatOutputProcessor::plain_text_only())) - } -} - -impl ChatRenderer for FailingDecodeChatBackend { - fn render(&self, request: &ChatRequest) -> vllm_chat::Result { - FakeChatBackend::new().render(request) - } -} - async fn test_models_with_engine_outputs_and_backend_inner( engine_id: impl Into, output_specs: Vec<(Vec, Option)>, @@ -2755,8 +2630,8 @@ async fn load_endpoint_resets_when_stream_response_is_dropped() { #[serial] async fn stream_error_is_returned_as_openai_error_sse() { let (app, engine_task) = test_app_with_backend_and_stream_output_specs( - Arc::new(FailingDecodeChatBackend), - default_stream_output_specs(), + Arc::new(FakeChatBackend::new()), + vec![(vec![UNKNOWN_DECODE_TOKEN_ID], None)], ) .await; let response = app @@ -2789,7 +2664,9 @@ async fn stream_error_is_returned_as_openai_error_sse() { assert!(text.contains("\"role\":\"assistant\""), "{text}"); assert!(text.contains("\"type\":\"server_error\""), "{text}"); assert!( - text.contains("forced decode failure for streaming test"), + text.contains(&format!( + "test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}" + )), "{text}" ); assert!(!text.contains("\"usage\":"), "{text}"); diff --git a/rust/src/text/Cargo.toml b/rust/src/text/Cargo.toml index 7ed02c07fc..7bda7f976e 100644 --- a/rust/src/text/Cargo.toml +++ b/rust/src/text/Cargo.toml @@ -31,6 +31,7 @@ serial_test.workspace = true tempfile.workspace = true tokio.workspace = true vllm-llm = { workspace = true, features = ["test-util"] } +vllm-tokenizer = { workspace = true, features = ["test-utils"] } [lints] workspace = true diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 6cd18195bb..a36480edf6 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -275,6 +275,7 @@ mod tests { use std::collections::{BTreeSet, HashMap}; use serial_test::file_serial; + use vllm_tokenizer::test_utils::TestTokenizer; use super::*; use crate::backend::hf::HfTextBackend; @@ -282,60 +283,8 @@ mod tests { use crate::error::{LogprobsError, TokenIdsError}; use crate::request::{Prompt, TextRequest}; - /// Stub tokenizer that returns empty token IDs — sufficient for tests that - /// don't exercise bad-words tokenization. - struct StubTokenizer; - - impl Tokenizer for StubTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(vec![]) - } - - fn decode( - &self, - _token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(String::new()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } - } - - fn stub_tokenizer() -> StubTokenizer { - StubTokenizer - } - - struct FixedTokenizer { - token_ids: Vec, - } - - impl Tokenizer for FixedTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(self.token_ids.clone()) - } - - fn decode( - &self, - _token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(String::new()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } + fn stub_tokenizer() -> TestTokenizer { + TestTokenizer::new() } fn sample_request() -> TextRequest { @@ -952,9 +901,7 @@ mod tests { #[test] fn lower_sampling_params_rejects_out_of_vocab_bad_words() { - let tokenizer = FixedTokenizer { - token_ids: vec![1999, 2000], - }; + let tokenizer = TestTokenizer::new().with_regular_token("blocked", 2000); let error = lower_sampling_params( SamplingParams { bad_words: Some(vec!["blocked".to_string()]), diff --git a/rust/src/text/src/output/decoded.rs b/rust/src/text/src/output/decoded.rs index 5b96005860..203efab460 100644 --- a/rust/src/text/src/output/decoded.rs +++ b/rust/src/text/src/output/decoded.rs @@ -323,37 +323,11 @@ mod tests { use futures::{Stream, stream}; use vllm_engine_core_client::AbortCause; use vllm_llm::GenerateOutput; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::*; use crate::output::TextOutputStreamExt as _; - /// Backend that treats each token ID as a raw byte, producing lossy UTF-8. - struct ByteTokenizer; - - impl Tokenizer for ByteTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - unreachable!() - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - let bytes = token_ids.iter().map(|id| *id as u8).collect::>(); - Ok(String::from_utf8_lossy(&bytes).into_owned()) - } - - fn token_to_id(&self, _token: &str) -> Option { - unreachable!() - } - } - /// Helper: run `decoded_text_event_stream` to completion and return the /// collected output. async fn run_to_completion( @@ -366,7 +340,7 @@ mod tests { token_ids, Some(FinishReason::Length), ))]); - let tokenizer: DynTokenizer = Arc::new(ByteTokenizer); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); decoded_text_event_stream("test".into(), tokenizer, raw_stream, decode_options, false) .collect_output() .await @@ -419,7 +393,7 @@ mod tests { ))), dropped_cause: Arc::clone(&dropped_cause), }; - let tokenizer: DynTokenizer = Arc::new(ByteTokenizer); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); let output = decoded_text_event_stream( "test".into(), diff --git a/rust/src/text/src/output/logprobs.rs b/rust/src/text/src/output/logprobs.rs index 7024c52b77..069a8cba27 100644 --- a/rust/src/text/src/output/logprobs.rs +++ b/rust/src/text/src/output/logprobs.rs @@ -129,40 +129,13 @@ fn decode_position_logprobs( #[cfg(test)] mod tests { use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob}; + use vllm_tokenizer::test_utils::TestTokenizer; use super::*; - #[derive(Debug)] - struct ByteTokenizer; - - impl vllm_tokenizer::Tokenizer for ByteTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - unreachable!() - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(String::from_utf8_lossy( - &token_ids.iter().map(|token_id| *token_id as u8).collect::>(), - ) - .into_owned()) - } - - fn token_to_id(&self, _token: &str) -> Option { - unreachable!() - } - } - #[test] fn decode_logprobs_decodes_every_candidate_token() { - let tokenizer = ByteTokenizer; + let tokenizer = TestTokenizer::new(); let logprobs = Logprobs { positions: vec![PositionLogprobs { entries: vec![ @@ -205,7 +178,7 @@ mod tests { #[test] fn decode_prompt_logprobs_separates_first_prompt_token() { - let tokenizer = ByteTokenizer; + let tokenizer = TestTokenizer::new(); let logprobs = Logprobs { positions: vec![PositionLogprobs { entries: vec![TokenLogprob { diff --git a/rust/src/tokenizer/Cargo.toml b/rust/src/tokenizer/Cargo.toml index 7b54676f66..d18b66b231 100644 --- a/rust/src/tokenizer/Cargo.toml +++ b/rust/src/tokenizer/Cargo.toml @@ -4,6 +4,9 @@ version.workspace = true edition.workspace = true license.workspace = true +[features] +test-utils = [] + [dependencies] base64.workspace = true fastokens.workspace = true diff --git a/rust/src/tokenizer/src/incremental.rs b/rust/src/tokenizer/src/incremental.rs index 462475fc91..52f8734548 100644 --- a/rust/src/tokenizer/src/incremental.rs +++ b/rust/src/tokenizer/src/incremental.rs @@ -170,6 +170,10 @@ mod tests { fn token_to_id(&self, _token: &str) -> Option { unreachable!() } + + fn id_to_token(&self, _id: u32) -> Option { + unreachable!() + } } #[test] @@ -248,6 +252,10 @@ mod tests { fn token_to_id(&self, _token: &str) -> Option { unreachable!() } + + fn id_to_token(&self, _id: u32) -> Option { + unreachable!() + } } #[test] @@ -320,6 +328,10 @@ mod tests { fn token_to_id(&self, _token: &str) -> Option { unreachable!() } + + fn id_to_token(&self, _id: u32) -> Option { + unreachable!() + } } /// Without the char-boundary fix, this panics slicing mid-emoji. diff --git a/rust/src/tokenizer/src/lib.rs b/rust/src/tokenizer/src/lib.rs index 6f315bc01b..4f459450c6 100644 --- a/rust/src/tokenizer/src/lib.rs +++ b/rust/src/tokenizer/src/lib.rs @@ -8,6 +8,8 @@ mod error; mod hf; mod incremental; mod tekken; +#[cfg(any(test, feature = "test-utils"))] +pub mod test_utils; mod tiktoken; pub use error::{Result, TokenizerError}; @@ -28,11 +30,7 @@ pub trait Tokenizer: Send + Sync { fn token_to_id(&self, token: &str) -> Option; /// Convert one token ID into the tokenizer's raw token string. - fn id_to_token(&self, _id: u32) -> Option { - // TODO: remove default impl and require this to be implemented by all - // tokenizers - None - } + fn id_to_token(&self, id: u32) -> Option; /// Return the vocabulary size. Backends that cannot report it fall back to /// `usize::MAX`, an effectively unbounded value used only by test stubs. diff --git a/rust/src/tokenizer/src/test_utils.rs b/rust/src/tokenizer/src/test_utils.rs new file mode 100644 index 0000000000..efc4c11717 --- /dev/null +++ b/rust/src/tokenizer/src/test_utils.rs @@ -0,0 +1,434 @@ +use std::collections::BTreeMap; + +use crate::{Result, Tokenizer, TokenizerError}; + +const FIRST_CONFIGURED_TOKEN_ID: u32 = 256; + +/// Whether a configured test token should be treated as special. +/// +/// Special tokens are skipped by [`Tokenizer::decode`] when +/// `skip_special_tokens` is set. Regular configured tokens are always emitted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TestTokenKind { + /// Token is skipped when `skip_special_tokens = true`. + Special, + /// Token is emitted regardless of `skip_special_tokens`. + Regular, +} + +impl TestTokenKind { + fn is_special(self) -> bool { + matches!(self, Self::Special) + } +} + +/// Decode behavior for token ids that are neither configured tokens nor byte ids. +/// +/// The default is [`UnknownDecode::Error`] so tests notice missing tokenizer +/// fixtures instead of silently accepting impossible ids. Individual tests can +/// opt into empty or replacement output when they are explicitly modeling a +/// lenient detokenization path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnknownDecode { + /// Return a tokenizer error on the first unknown id. + Error, + /// Drop unknown ids from decoded output. + Empty, + /// Emit U+FFFD for each unknown id. + Replacement, +} + +#[derive(Debug, Clone)] +struct TestToken { + text: String, + kind: TestTokenKind, +} + +/// Configurable tokenizer for Rust frontend tests. +/// +/// `TestTokenizer` is intentionally small, but its methods obey the same basic +/// contract as production tokenizers: +/// +/// - ordinary text encodes as UTF-8 byte ids; +/// - configured token ids start at 256, leaving `0..=255` for byte fallback; +/// - configured token ids and token text are unique; +/// - configured tokens are matched before ordinary bytes, using longest-prefix matching so +/// multi-character markers such as `` work naturally; +/// - `token_to_id` and `id_to_token` are consistent for configured tokens; +/// - `decode` is strict by default for ids outside the byte range and the configured token table; +/// - `vocab_size` is an exclusive upper bound covering byte ids and configured token ids unless a +/// test sets it explicitly. +/// +/// Prefer this helper over ad-hoc fake tokenizers for tests that rely on +/// tokenizer semantics. Keep dedicated tiny fakes for error injection or for +/// tests that deliberately need a degenerate tokenizer. +#[derive(Debug, Clone)] +pub struct TestTokenizer { + token_to_id: BTreeMap, + id_to_token: BTreeMap, + unknown_decode: UnknownDecode, + vocab_size: Option, + bos_token_id: Option, +} + +impl Default for TestTokenizer { + fn default() -> Self { + Self::new() + } +} + +impl TestTokenizer { + /// Create a byte-level test tokenizer with strict unknown-id decode. + pub fn new() -> Self { + Self { + token_to_id: BTreeMap::new(), + id_to_token: BTreeMap::new(), + unknown_decode: UnknownDecode::Error, + vocab_size: None, + bos_token_id: None, + } + } + + /// Add a configured token and return the updated tokenizer. + /// + /// Configured tokens must use ids outside the byte range and may be marked + /// special or regular. + pub fn with_token(mut self, token: impl Into, id: u32, kind: TestTokenKind) -> Self { + self.insert_token(token, id, kind); + self + } + + /// Add a special configured token and return the updated tokenizer. + pub fn with_special_token(self, token: impl Into, id: u32) -> Self { + self.with_token(token, id, TestTokenKind::Special) + } + + /// Add a regular configured token and return the updated tokenizer. + pub fn with_regular_token(self, token: impl Into, id: u32) -> Self { + self.with_token(token, id, TestTokenKind::Regular) + } + + /// Add a special BOS token inserted by `encode(..., true)`. + /// + /// This also registers the token in the normal token/id maps so + /// `token_to_id`, `id_to_token`, `decode`, and `is_special_id` stay + /// consistent for the inserted id. + pub fn with_bos_token(mut self, token: impl Into, id: u32) -> Self { + self.insert_token(token, id, TestTokenKind::Special); + self.bos_token_id = Some(id); + self + } + + /// Set decode behavior for unknown non-byte ids. + pub fn with_unknown_decode(mut self, behavior: UnknownDecode) -> Self { + self.unknown_decode = behavior; + self + } + + /// Set an explicit vocabulary size. + /// + /// Use this when a test needs a model-like vocabulary bound that differs + /// from the highest configured token id plus one. + pub fn with_vocab_size(mut self, vocab_size: usize) -> Self { + self.vocab_size = Some(vocab_size); + self + } + + fn insert_token(&mut self, token: impl Into, id: u32, kind: TestTokenKind) { + let token = token.into(); + assert!( + !token.is_empty(), + "configured test token text must be non-empty" + ); + assert!( + id >= FIRST_CONFIGURED_TOKEN_ID, + "configured test token id {id} overlaps byte fallback range 0..=255" + ); + assert!( + token.len() > 1, + "configured test token text {token:?} overlaps byte fallback token text" + ); + if self.token_to_id.insert(token.clone(), id).is_some() { + panic!("configured test token text {token:?} was registered more than once"); + } + if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() { + panic!("configured test token id {id} was registered more than once"); + } + } + + fn byte_to_token(id: u32) -> Option { + u8::try_from(id).ok().map(|byte| String::from_utf8_lossy(&[byte]).into_owned()) + } + + fn flush_bytes(bytes: &mut Vec, output: &mut String) { + if !bytes.is_empty() { + output.push_str(&String::from_utf8_lossy(bytes)); + bytes.clear(); + } + } + + fn configured_token_prefix(&self, text: &str) -> Option<(&str, u32)> { + self.token_to_id + .iter() + .filter_map(|(token, &id)| text.starts_with(token).then_some((token.as_str(), id))) + .max_by_key(|(token, _)| token.len()) + } + + fn inferred_vocab_size(&self) -> usize { + let max_configured = + self.id_to_token.last_key_value().map(|(&id, _)| id as usize + 1).unwrap_or(0); + 256.max(max_configured) + } +} + +impl Tokenizer for TestTokenizer { + fn encode(&self, text: &str, add_special_tokens: bool) -> Result> { + let mut ids = Vec::new(); + if add_special_tokens && let Some(bos_token_id) = self.bos_token_id { + ids.push(bos_token_id); + } + + let mut rest = text; + while !rest.is_empty() { + if let Some((token, id)) = self.configured_token_prefix(rest) { + ids.push(id); + rest = &rest[token.len()..]; + continue; + } + + let ch = rest.chars().next().expect("rest is not empty"); + let mut buf = [0_u8; 4]; + ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from)); + rest = &rest[ch.len_utf8()..]; + } + + Ok(ids) + } + + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { + let mut output = String::new(); + let mut pending_bytes = Vec::new(); + for &id in token_ids { + if let Some(token) = self.id_to_token.get(&id) { + Self::flush_bytes(&mut pending_bytes, &mut output); + if !(skip_special_tokens && token.kind.is_special()) { + output.push_str(&token.text); + } + } else if let Ok(byte) = u8::try_from(id) { + pending_bytes.push(byte); + } else { + Self::flush_bytes(&mut pending_bytes, &mut output); + match self.unknown_decode { + UnknownDecode::Error => { + return Err(TokenizerError(format!( + "test tokenizer cannot decode unknown token id {id}" + ))); + } + UnknownDecode::Empty => {} + UnknownDecode::Replacement => output.push('\u{FFFD}'), + } + } + } + Self::flush_bytes(&mut pending_bytes, &mut output); + Ok(output) + } + + fn token_to_id(&self, token: &str) -> Option { + self.token_to_id.get(token).copied().or_else(|| { + let bytes = token.as_bytes(); + (bytes.len() == 1).then(|| u32::from(bytes[0])) + }) + } + + fn id_to_token(&self, id: u32) -> Option { + self.id_to_token + .get(&id) + .map(|token| token.text.clone()) + .or_else(|| Self::byte_to_token(id)) + } + + fn vocab_size(&self) -> usize { + self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size()) + } + + fn is_special_id(&self, token_id: u32) -> bool { + self.id_to_token.get(&token_id).is_some_and(|token| token.kind.is_special()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn byte_text_roundtrips_and_reports_byte_ids() { + let tokenizer = TestTokenizer::new(); + + let ids = tokenizer.encode("hi", false).unwrap(); + assert_eq!(ids, vec![b'h' as u32, b'i' as u32]); + assert_eq!(tokenizer.decode(&ids, false).unwrap(), "hi"); + assert_eq!(tokenizer.token_to_id("h"), Some(b'h' as u32)); + assert_eq!(tokenizer.id_to_token(b'h' as u32).as_deref(), Some("h")); + assert_eq!(tokenizer.vocab_size(), 256); + } + + #[test] + fn configured_tokens_use_longest_prefix_matching() { + let tokenizer = TestTokenizer::new() + .with_regular_token("", 999) + .with_regular_token("", 1000); + + assert_eq!( + tokenizer.encode("ab", false).unwrap(), + vec![b'a' as u32, 1000, b'b' as u32,] + ); + assert_eq!( + tokenizer.decode(&[b'a' as u32, 1000, b'b' as u32], false).unwrap(), + "ab" + ); + assert_eq!(tokenizer.token_to_id(""), Some(999)); + assert_eq!( + tokenizer.id_to_token(1000).as_deref(), + Some("") + ); + assert_eq!(tokenizer.vocab_size(), 1001); + } + + #[test] + fn non_ascii_text_roundtrips_through_buffered_byte_decode() { + let tokenizer = TestTokenizer::new(); + let text = "你好, café, 🚀"; + + let ids = tokenizer.encode(text, false).unwrap(); + assert_eq!( + ids, + text.as_bytes().iter().copied().map(u32::from).collect::>() + ); + assert_eq!(tokenizer.decode(&ids, false).unwrap(), text); + } + + #[test] + fn buffered_byte_decode_flushes_around_configured_tokens() { + let tokenizer = TestTokenizer::new() + .with_regular_token("", 999) + .with_special_token("", 1000); + + assert_eq!( + tokenizer.encode("你好🚀", false).unwrap(), + vec![228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128] + ); + assert_eq!( + tokenizer + .decode( + &[228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128], + false + ) + .unwrap(), + "你好🚀" + ); + assert_eq!( + tokenizer + .decode( + &[228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128], + true + ) + .unwrap(), + "你好🚀" + ); + } + + #[test] + fn invalid_utf8_bytes_decode_lossily_as_a_sequence() { + let tokenizer = TestTokenizer::new(); + + assert_eq!(tokenizer.decode(&[0xE4, 0xBD], false).unwrap(), "\u{FFFD}"); + assert_eq!( + tokenizer.decode(&[0xFF, b'a' as u32], false).unwrap(), + "\u{FFFD}a" + ); + } + + #[test] + fn special_tokens_respect_skip_special_tokens() { + let tokenizer = TestTokenizer::new() + .with_bos_token("", 256) + .with_special_token("", 0xF001) + .with_regular_token("", 0xF002); + + assert_eq!( + tokenizer.encode("x", true).unwrap(), + vec![256, 0xF001, b'x' as u32, 0xF002,] + ); + assert_eq!( + tokenizer.decode(&[256, 0xF001, b'x' as u32, 0xF002], false).unwrap(), + "x" + ); + assert_eq!( + tokenizer.decode(&[256, 0xF001, b'x' as u32, 0xF002], true).unwrap(), + "x" + ); + assert!(tokenizer.is_special_id(0xF001)); + assert!(!tokenizer.is_special_id(0xF002)); + } + + #[test] + #[should_panic(expected = "configured test token id 255 overlaps byte fallback range 0..=255")] + fn configured_token_id_must_stay_outside_byte_range() { + let _ = TestTokenizer::new().with_regular_token("", 255); + } + + #[test] + #[should_panic(expected = "configured test token text \"a\" overlaps byte fallback token text")] + fn configured_token_text_must_not_shadow_byte_tokens() { + let _ = TestTokenizer::new().with_regular_token("a", 256); + } + + #[test] + #[should_panic( + expected = "configured test token text \"\" was registered more than once" + )] + fn configured_token_text_must_be_unique() { + let _ = TestTokenizer::new() + .with_regular_token("", 256) + .with_regular_token("", 257); + } + + #[test] + #[should_panic(expected = "configured test token id 256 was registered more than once")] + fn configured_token_id_must_be_unique() { + let _ = TestTokenizer::new() + .with_regular_token("", 256) + .with_regular_token("", 256); + } + + #[test] + fn unknown_decode_is_strict_by_default_and_configurable() { + let strict = TestTokenizer::new(); + assert!(strict.decode(&[300], false).is_err()); + assert_eq!( + TestTokenizer::new() + .with_unknown_decode(UnknownDecode::Empty) + .decode(&[b'a' as u32, 300, b'b' as u32], false) + .unwrap(), + "ab" + ); + assert_eq!( + TestTokenizer::new() + .with_unknown_decode(UnknownDecode::Replacement) + .decode(&[300], false) + .unwrap(), + "\u{FFFD}" + ); + assert_eq!(strict.id_to_token(300), None); + } + + #[test] + fn explicit_vocab_size_overrides_inferred_bound() { + let tokenizer = TestTokenizer::new() + .with_regular_token("", 10_000) + .with_vocab_size(20_000); + + assert_eq!(tokenizer.vocab_size(), 20_000); + assert_eq!(tokenizer.id_to_token(10_000).as_deref(), Some("")); + } +}