[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-30 15:34:43 +01:00
committed by GitHub
parent 1ab9522935
commit 7a327f0b4f
28 changed files with 680 additions and 826 deletions
+1
View File
@@ -64,6 +64,7 @@ tempfile.workspace = true
tokio = { workspace = true, features = ["test-util"] }
tower.workspace = true
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
zeromq.workspace = true
[lints]
+3 -30
View File
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::Llm;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{Prompt, TextBackend};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -155,37 +156,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
#[derive(Clone, Debug)]
struct FakeTextBackend;
#[derive(Debug)]
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
token.bytes().next().map(u32::from)
}
}
impl TextBackend for FakeTextBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeTokenizer)
Arc::new(TestTokenizer::new())
}
fn model_id(&self) -> &str {
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::Llm;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{Prompt, TextBackend};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -151,37 +152,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
#[derive(Clone, Debug)]
struct FakeChatBackend;
#[derive(Debug)]
struct FakeChatTokenizer;
impl Tokenizer for FakeChatTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
token.bytes().next().map(u32::from)
}
}
impl TextBackend for FakeChatBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeChatTokenizer)
Arc::new(TestTokenizer::new())
}
fn model_id(&self) -> &str {
@@ -194,7 +194,7 @@ mod tests {
use axum::http::HeaderMap;
use serde_json::json;
use vllm_text::Prompt;
use vllm_text::tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::prepare_completion_request;
use crate::lora::LoraModelResolution;
@@ -212,32 +212,8 @@ mod tests {
}
}
#[derive(Debug)]
struct TestTokenizer;
impl Tokenizer for TestTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
fn test_tokenizer() -> TestTokenizer {
TestTokenizer::new()
}
fn base_request_json() -> serde_json::Value {
@@ -297,7 +273,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -340,7 +316,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare")
.text_request
@@ -374,7 +350,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -399,7 +375,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -422,7 +398,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -446,7 +422,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -471,7 +447,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -498,7 +474,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -523,7 +499,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -549,7 +525,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1));
@@ -574,7 +550,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
@@ -593,7 +569,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.data_parallel_rank, None);
+22 -145
View File
@@ -40,8 +40,9 @@ use vllm_engine_core_client::{
};
use vllm_llm::Llm;
use vllm_metrics::METRICS;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{Prompt, TextBackend};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -417,82 +418,20 @@ struct FakeChatBackend {
}
/// Synthetic BOS id used when `add_special_tokens` is true in tests.
const FAKE_BOS_TOKEN_ID: u32 = 1;
const FAKE_BOS_TOKEN_ID: u32 = 256;
const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;
#[derive(Debug)]
struct FakeChatTokenizer;
impl Tokenizer for FakeChatTokenizer {
fn encode(
&self,
text: &str,
add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
let mut token_ids = Vec::new();
if add_special_tokens {
token_ids.push(FAKE_BOS_TOKEN_ID);
}
let mut rest = text;
while !rest.is_empty() {
if let Some(stripped) = rest.strip_prefix("<image>") {
token_ids.push(999);
rest = stripped;
continue;
}
if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {
token_ids.push(151655);
rest = stripped;
continue;
}
let ch = rest.chars().next().expect("rest is not empty");
let mut buf = [0; 4];
token_ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from));
rest = &rest[ch.len_utf8()..];
}
Ok(token_ids)
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<image>" => Some(999),
"<|image_pad|>" => Some(151655),
"<think>" => Some(0xF001),
"</think>" => Some(0xF002),
"<|START_THINKING|>" => Some(0xF003),
"<|END_THINKING|>" => Some(0xF004),
"◁think▷" => Some(0xF005),
"◁/think▷" => Some(0xF006),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
match id {
FAKE_BOS_TOKEN_ID => Some("<bos>".to_string()),
999 => Some("<image>".to_string()),
151655 => Some("<|image_pad|>".to_string()),
0xF001 => Some("<think>".to_string()),
0xF002 => Some("</think>".to_string()),
0xF003 => Some("<|START_THINKING|>".to_string()),
0xF004 => Some("<|END_THINKING|>".to_string()),
0xF005 => Some("◁think▷".to_string()),
0xF006 => Some("◁/think▷".to_string()),
id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()),
_ => None,
}
}
fn fake_chat_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_bos_token("<bos>", FAKE_BOS_TOKEN_ID)
.with_regular_token("<image>", 999)
.with_regular_token("<|image_pad|>", 151655)
.with_regular_token("<think>", 0xF001)
.with_regular_token("</think>", 0xF002)
.with_regular_token("<|START_THINKING|>", 0xF003)
.with_regular_token("<|END_THINKING|>", 0xF004)
.with_regular_token("◁think▷", 0xF005)
.with_regular_token("◁/think▷", 0xF006)
}
impl FakeChatBackend {
@@ -530,7 +469,7 @@ impl fmt::Debug for FakeChatBackend {
impl TextBackend for FakeChatBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeChatTokenizer)
Arc::new(fake_chat_tokenizer())
}
fn model_id(&self) -> &str {
@@ -630,7 +569,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
Some("qwen2_vl".to_string()),
Some(&config_path),
None,
Arc::new(FakeChatTokenizer),
Arc::new(fake_chat_tokenizer()),
)
.expect("load multimodal info")
.expect("qwen multimodal info is registered");
@@ -638,70 +577,6 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
info
}
#[derive(Clone, Debug)]
struct FailingDecodeChatBackend;
#[derive(Debug)]
struct FailingDecodeTokenizer;
impl Tokenizer for FailingDecodeTokenizer {
fn encode(
&self,
text: &str,
add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
FakeChatTokenizer.encode(text, add_special_tokens)
}
fn decode(
&self,
token_ids: &[u32],
skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
if token_ids.contains(&(b'i' as u32)) {
return Err(vllm_text::tokenizer::TokenizerError(
"forced decode failure for streaming test".to_string(),
));
}
FakeChatTokenizer.decode(token_ids, skip_special_tokens)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
FakeChatTokenizer.token_to_id(token)
}
}
impl TextBackend for FailingDecodeChatBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FailingDecodeTokenizer)
}
fn model_id(&self) -> &str {
"test-model"
}
}
impl ChatBackend for FailingDecodeChatBackend {
fn chat_renderer(&self) -> DynChatRenderer {
Arc::new(self.clone())
}
fn new_chat_output_processor(
&self,
_request: &mut ChatRequest,
_options: NewChatOutputProcessorOptions<'_>,
) -> vllm_chat::Result<DynChatOutputProcessor> {
Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))
}
}
impl ChatRenderer for FailingDecodeChatBackend {
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<vllm_chat::RenderedPrompt> {
FakeChatBackend::new().render(request)
}
}
async fn test_models_with_engine_outputs_and_backend_inner(
engine_id: impl Into<EngineId>,
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
@@ -2755,8 +2630,8 @@ async fn load_endpoint_resets_when_stream_response_is_dropped() {
#[serial]
async fn stream_error_is_returned_as_openai_error_sse() {
let (app, engine_task) = test_app_with_backend_and_stream_output_specs(
Arc::new(FailingDecodeChatBackend),
default_stream_output_specs(),
Arc::new(FakeChatBackend::new()),
vec![(vec![UNKNOWN_DECODE_TOKEN_ID], None)],
)
.await;
let response = app
@@ -2789,7 +2664,9 @@ async fn stream_error_is_returned_as_openai_error_sse() {
assert!(text.contains("\"role\":\"assistant\""), "{text}");
assert!(text.contains("\"type\":\"server_error\""), "{text}");
assert!(
text.contains("forced decode failure for streaming test"),
text.contains(&format!(
"test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}"
)),
"{text}"
);
assert!(!text.contains("\"usage\":"), "{text}");