[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-30 15:34:43 +01:00
committed by GitHub
parent 1ab9522935
commit 7a327f0b4f
28 changed files with 680 additions and 826 deletions
+1
View File
@@ -50,6 +50,7 @@ tokio.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
zeromq.workspace = true
[lints]
+3 -26
View File
@@ -154,7 +154,8 @@ mod tests {
use thiserror_ext::AsReport as _;
use vllm_text::Prompt;
use vllm_text::backend::hf::TokenizerSource;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::HfChatBackend;
use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};
@@ -196,32 +197,8 @@ mod tests {
}
}
struct TestTokenizer;
impl Tokenizer for TestTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(Vec::new())
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
}
fn test_tokenizer() -> DynTokenizer {
Arc::new(TestTokenizer)
Arc::new(TestTokenizer::new())
}
fn backend_for_selection(
+10 -56
View File
@@ -563,7 +563,7 @@ mod tests {
use llm_multimodal::TokenId;
use vllm_engine_core_client::protocol::tensor::WireArrayData;
use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError};
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
@@ -574,60 +574,14 @@ mod tests {
const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;
struct TestTokenizer;
impl Tokenizer for TestTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> std::result::Result<Vec<u32>, TokenizerError> {
Ok(match text {
"<|image|>" => vec![LLAMA4_IMAGE_ID],
text => text.bytes().map(u32::from).collect(),
})
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> std::result::Result<String, TokenizerError> {
Ok(String::new())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|image_start|>" => Some(LLAMA4_IMAGE_START_ID),
"<|image_end|>" => Some(LLAMA4_IMAGE_END_ID),
"<|image|>" => Some(LLAMA4_IMAGE_ID),
"<|patch|>" => Some(LLAMA4_PATCH_ID),
"<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID),
"<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
match id {
LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()),
LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()),
LLAMA4_IMAGE_ID => Some("<|image|>".to_string()),
LLAMA4_PATCH_ID => Some("<|patch|>".to_string()),
LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()),
LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()),
_ => None,
}
}
fn create_decode_stream(
&self,
_prompt_token_ids: &[u32],
_skip_special_tokens: bool,
_min_bytes_to_buffer: usize,
) -> Box<dyn IncrementalDecoder + '_> {
unreachable!("not used")
}
fn llama4_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
.with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID)
.with_regular_token("<|image|>", LLAMA4_IMAGE_ID)
.with_regular_token("<|patch|>", LLAMA4_PATCH_ID)
.with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID)
.with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID)
}
fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {
@@ -635,7 +589,7 @@ mod tests {
model_id: format!("{model_type}-test"),
model_type: Some(model_type.to_string()),
config,
tokenizer: TokenizerResolver(Arc::new(TestTokenizer)),
tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())),
};
let spec = context
.resolve_model_spec()
+7 -34
View File
@@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
mod tests {
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::DefaultChatOutputProcessor;
use crate::Error;
use crate::parser::ParserSelection;
use crate::request::ChatRequest;
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|channel>" => Some(1),
"<channel|>" => Some(2),
_ => None,
}
}
}
fn tokenizer() -> Arc<FakeTokenizer> {
Arc::new(FakeTokenizer)
fn tokenizer() -> Arc<TestTokenizer> {
Arc::new(
TestTokenizer::new()
.with_regular_token("<|channel>", 256)
.with_regular_token("<channel|>", 257),
)
}
#[test]
+2 -25
View File
@@ -1,32 +1,9 @@
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::{ReasoningParserFactory, names};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
}
#[test]
fn factory_contains_and_lists_registered_parsers() {
let factory = ReasoningParserFactory::new();
@@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() {
#[test]
fn factory_rejects_unknown_parser_names() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(TestTokenizer::new());
let factory = ReasoningParserFactory::new();
let error = match factory.create("missing", tokenizer) {
Ok(_) => panic!("expected parser lookup to fail"),
+6 -31
View File
@@ -75,39 +75,14 @@ impl UnifiedParserFactory {
mod tests {
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::{UnifiedParserFactory, names};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|channel>" => Some(1),
"<channel|>" => Some(2),
_ => None,
}
}
fn tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<|channel>", 256)
.with_regular_token("<channel|>", 257)
}
#[test]
@@ -119,6 +94,6 @@ mod tests {
factory.resolve_name_for_model("google/gemma-4-27b-it"),
Some(names::GEMMA4)
);
factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap();
factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
}
}
+34 -103
View File
@@ -21,15 +21,17 @@ use vllm_engine_core_client::protocol::{
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
use vllm_llm::Llm;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, Prompt,
TextBackend,
};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
const SPECIAL_STOP_TOKEN_ID: u32 = 256;
const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;
fn request_output(
request_id: &str,
@@ -158,45 +160,18 @@ async fn connect_chat_llm_with_ipc(
struct FakeChatBackend {
has_template: bool,
model_id: String,
tokenizer: DynTokenizer,
}
#[derive(Debug)]
struct FakeChatTokenizer;
impl Tokenizer for FakeChatTokenizer {
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
let bytes = token_ids
.iter()
.filter_map(|id| {
if skip_special_tokens && *id == SPECIAL_STOP_TOKEN_ID {
None
} else {
Some(*id as u8)
}
})
.collect::<Vec<_>>();
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(0xF001),
"</think>" => Some(0xF002),
"<|START_THINKING|>" => Some(0xF003),
"<|END_THINKING|>" => Some(0xF004),
"◁think▷" => Some(0xF005),
"◁/think▷" => Some(0xF006),
_ => None,
}
}
fn fake_chat_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_special_token("<stop>", SPECIAL_STOP_TOKEN_ID)
.with_regular_token("<think>", 0xF001)
.with_regular_token("</think>", 0xF002)
.with_regular_token("<|START_THINKING|>", 0xF003)
.with_regular_token("<|END_THINKING|>", 0xF004)
.with_regular_token("◁think▷", 0xF005)
.with_regular_token("◁/think▷", 0xF006)
}
impl fmt::Debug for FakeChatBackend {
@@ -210,6 +185,7 @@ impl FakeChatBackend {
Self {
has_template: true,
model_id: "test-model".to_string(),
tokenizer: Arc::new(fake_chat_tokenizer()),
}
}
@@ -217,6 +193,7 @@ impl FakeChatBackend {
Self {
has_template: false,
model_id: "test-model".to_string(),
tokenizer: Arc::new(fake_chat_tokenizer()),
}
}
@@ -224,13 +201,19 @@ impl FakeChatBackend {
Self {
has_template: true,
model_id: model_id.into(),
tokenizer: Arc::new(fake_chat_tokenizer()),
}
}
fn with_tokenizer(mut self, tokenizer: DynTokenizer) -> Self {
self.tokenizer = tokenizer;
self
}
}
impl TextBackend for FakeChatBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeChatTokenizer)
Arc::clone(&self.tokenizer)
}
fn model_id(&self) -> &str {
@@ -282,65 +265,6 @@ impl ChatRenderer for FakeChatBackend {
}
}
#[derive(Clone, Debug)]
struct FailingDecodeBackend {
inner: FakeChatBackend,
}
#[derive(Debug)]
struct FailingDecodeTokenizer;
impl Tokenizer for FailingDecodeTokenizer {
fn encode(&self, text: &str, add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
FakeChatTokenizer.encode(text, add_special_tokens)
}
fn decode(
&self,
token_ids: &[u32],
skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
if token_ids.contains(&(b'i' as u32)) {
return Err(vllm_tokenizer::TokenizerError("decode failed".to_string()));
}
FakeChatTokenizer.decode(token_ids, skip_special_tokens)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
FakeChatTokenizer.token_to_id(token)
}
}
impl TextBackend for FailingDecodeBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FailingDecodeTokenizer)
}
fn model_id(&self) -> &str {
self.inner.model_id()
}
}
impl ChatBackend for FailingDecodeBackend {
fn chat_renderer(&self) -> DynChatRenderer {
Arc::new(self.clone())
}
fn new_chat_output_processor(
&self,
_request: &mut ChatRequest,
_options: NewChatOutputProcessorOptions<'_>,
) -> vllm_chat::Result<DynChatOutputProcessor> {
Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))
}
}
impl ChatRenderer for FailingDecodeBackend {
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
self.inner.render(request)
}
}
/// Skip `LogprobsDelta` events that carry only token_ids (no logprobs),
/// returning the next semantically interesting event.
async fn next_semantic<S>(stream: &mut S) -> Option<Result<ChatEvent, vllm_chat::Error>>
@@ -738,7 +662,12 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
send_outputs(
push,
EngineCoreOutputs {
outputs: vec![request_output("chat-4", vec![b'i' as u32], None, None)],
outputs: vec![request_output(
"chat-4",
vec![UNKNOWN_DECODE_TOKEN_ID],
None,
None,
)],
..Default::default()
},
)
@@ -747,9 +676,8 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
},
);
let backend: Arc<dyn ChatTextBackend> = Arc::new(FailingDecodeBackend {
inner: FakeChatBackend::new(),
});
let backend: Arc<dyn ChatTextBackend> =
Arc::new(FakeChatBackend::new().with_tokenizer(Arc::new(TestTokenizer::new())));
let chat = connect_chat_llm_with_ipc(
EngineCoreClientConfig::new_single(handshake_address),
&ipc,
@@ -769,7 +697,10 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
match timeout(Duration::from_secs(2), stream.next()).await.unwrap() {
Some(Err(vllm_chat::Error::Text(vllm_text::Error::Tokenizer(message)))) => {
assert_eq!(message, "decode failed");
assert_eq!(
message,
format!("test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}")
);
}
other => panic!("unexpected event after close: {other:?}"),
}