[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -50,6 +50,7 @@ tokio.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
|
||||
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
|
||||
zeromq.workspace = true
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -154,7 +154,8 @@ mod tests {
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::backend::hf::TokenizerSource;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::HfChatBackend;
|
||||
use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};
|
||||
@@ -196,32 +197,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct TestTokenizer;
|
||||
|
||||
impl Tokenizer for TestTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn test_tokenizer() -> DynTokenizer {
|
||||
Arc::new(TestTokenizer)
|
||||
Arc::new(TestTokenizer::new())
|
||||
}
|
||||
|
||||
fn backend_for_selection(
|
||||
|
||||
@@ -563,7 +563,7 @@ mod tests {
|
||||
|
||||
use llm_multimodal::TokenId;
|
||||
use vllm_engine_core_client::protocol::tensor::WireArrayData;
|
||||
use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -574,60 +574,14 @@ mod tests {
|
||||
const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
|
||||
const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;
|
||||
|
||||
struct TestTokenizer;
|
||||
|
||||
impl Tokenizer for TestTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> std::result::Result<Vec<u32>, TokenizerError> {
|
||||
Ok(match text {
|
||||
"<|image|>" => vec![LLAMA4_IMAGE_ID],
|
||||
text => text.bytes().map(u32::from).collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> std::result::Result<String, TokenizerError> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|image_start|>" => Some(LLAMA4_IMAGE_START_ID),
|
||||
"<|image_end|>" => Some(LLAMA4_IMAGE_END_ID),
|
||||
"<|image|>" => Some(LLAMA4_IMAGE_ID),
|
||||
"<|patch|>" => Some(LLAMA4_PATCH_ID),
|
||||
"<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID),
|
||||
"<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
match id {
|
||||
LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()),
|
||||
LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()),
|
||||
LLAMA4_IMAGE_ID => Some("<|image|>".to_string()),
|
||||
LLAMA4_PATCH_ID => Some("<|patch|>".to_string()),
|
||||
LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()),
|
||||
LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_decode_stream(
|
||||
&self,
|
||||
_prompt_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
_min_bytes_to_buffer: usize,
|
||||
) -> Box<dyn IncrementalDecoder + '_> {
|
||||
unreachable!("not used")
|
||||
}
|
||||
fn llama4_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
|
||||
.with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID)
|
||||
.with_regular_token("<|image|>", LLAMA4_IMAGE_ID)
|
||||
.with_regular_token("<|patch|>", LLAMA4_PATCH_ID)
|
||||
.with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID)
|
||||
.with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID)
|
||||
}
|
||||
|
||||
fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {
|
||||
@@ -635,7 +589,7 @@ mod tests {
|
||||
model_id: format!("{model_type}-test"),
|
||||
model_type: Some(model_type.to_string()),
|
||||
config,
|
||||
tokenizer: TokenizerResolver(Arc::new(TestTokenizer)),
|
||||
tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())),
|
||||
};
|
||||
let spec = context
|
||||
.resolve_model_spec()
|
||||
|
||||
@@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::DefaultChatOutputProcessor;
|
||||
use crate::Error;
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|channel>" => Some(1),
|
||||
"<channel|>" => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenizer() -> Arc<FakeTokenizer> {
|
||||
Arc::new(FakeTokenizer)
|
||||
fn tokenizer() -> Arc<TestTokenizer> {
|
||||
Arc::new(
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<|channel>", 256)
|
||||
.with_regular_token("<channel|>", 257),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,32 +1,9 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::{ReasoningParserFactory, names};
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_contains_and_lists_registered_parsers() {
|
||||
let factory = ReasoningParserFactory::new();
|
||||
@@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() {
|
||||
|
||||
#[test]
|
||||
fn factory_rejects_unknown_parser_names() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(TestTokenizer::new());
|
||||
let factory = ReasoningParserFactory::new();
|
||||
let error = match factory.create("missing", tokenizer) {
|
||||
Ok(_) => panic!("expected parser lookup to fail"),
|
||||
|
||||
@@ -75,39 +75,14 @@ impl UnifiedParserFactory {
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::{UnifiedParserFactory, names};
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|channel>" => Some(1),
|
||||
"<channel|>" => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<|channel>", 256)
|
||||
.with_regular_token("<channel|>", 257)
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -119,6 +94,6 @@ mod tests {
|
||||
factory.resolve_name_for_model("google/gemma-4-27b-it"),
|
||||
Some(names::GEMMA4)
|
||||
);
|
||||
factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap();
|
||||
factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
+34
-103
@@ -21,15 +21,17 @@ use vllm_engine_core_client::protocol::{
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, Prompt,
|
||||
TextBackend,
|
||||
};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
const SPECIAL_STOP_TOKEN_ID: u32 = 256;
|
||||
const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;
|
||||
|
||||
fn request_output(
|
||||
request_id: &str,
|
||||
@@ -158,45 +160,18 @@ async fn connect_chat_llm_with_ipc(
|
||||
struct FakeChatBackend {
|
||||
has_template: bool,
|
||||
model_id: String,
|
||||
tokenizer: DynTokenizer,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FakeChatTokenizer;
|
||||
|
||||
impl Tokenizer for FakeChatTokenizer {
|
||||
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
let bytes = token_ids
|
||||
.iter()
|
||||
.filter_map(|id| {
|
||||
if skip_special_tokens && *id == SPECIAL_STOP_TOKEN_ID {
|
||||
None
|
||||
} else {
|
||||
Some(*id as u8)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<think>" => Some(0xF001),
|
||||
"</think>" => Some(0xF002),
|
||||
"<|START_THINKING|>" => Some(0xF003),
|
||||
"<|END_THINKING|>" => Some(0xF004),
|
||||
"◁think▷" => Some(0xF005),
|
||||
"◁/think▷" => Some(0xF006),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn fake_chat_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_special_token("<stop>", SPECIAL_STOP_TOKEN_ID)
|
||||
.with_regular_token("<think>", 0xF001)
|
||||
.with_regular_token("</think>", 0xF002)
|
||||
.with_regular_token("<|START_THINKING|>", 0xF003)
|
||||
.with_regular_token("<|END_THINKING|>", 0xF004)
|
||||
.with_regular_token("◁think▷", 0xF005)
|
||||
.with_regular_token("◁/think▷", 0xF006)
|
||||
}
|
||||
|
||||
impl fmt::Debug for FakeChatBackend {
|
||||
@@ -210,6 +185,7 @@ impl FakeChatBackend {
|
||||
Self {
|
||||
has_template: true,
|
||||
model_id: "test-model".to_string(),
|
||||
tokenizer: Arc::new(fake_chat_tokenizer()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,6 +193,7 @@ impl FakeChatBackend {
|
||||
Self {
|
||||
has_template: false,
|
||||
model_id: "test-model".to_string(),
|
||||
tokenizer: Arc::new(fake_chat_tokenizer()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,13 +201,19 @@ impl FakeChatBackend {
|
||||
Self {
|
||||
has_template: true,
|
||||
model_id: model_id.into(),
|
||||
tokenizer: Arc::new(fake_chat_tokenizer()),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_tokenizer(mut self, tokenizer: DynTokenizer) -> Self {
|
||||
self.tokenizer = tokenizer;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for FakeChatBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FakeChatTokenizer)
|
||||
Arc::clone(&self.tokenizer)
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
@@ -282,65 +265,6 @@ impl ChatRenderer for FakeChatBackend {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct FailingDecodeBackend {
|
||||
inner: FakeChatBackend,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FailingDecodeTokenizer;
|
||||
|
||||
impl Tokenizer for FailingDecodeTokenizer {
|
||||
fn encode(&self, text: &str, add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
FakeChatTokenizer.encode(text, add_special_tokens)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
if token_ids.contains(&(b'i' as u32)) {
|
||||
return Err(vllm_tokenizer::TokenizerError("decode failed".to_string()));
|
||||
}
|
||||
FakeChatTokenizer.decode(token_ids, skip_special_tokens)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
FakeChatTokenizer.token_to_id(token)
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for FailingDecodeBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FailingDecodeTokenizer)
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
self.inner.model_id()
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for FailingDecodeBackend {
|
||||
fn chat_renderer(&self) -> DynChatRenderer {
|
||||
Arc::new(self.clone())
|
||||
}
|
||||
|
||||
fn new_chat_output_processor(
|
||||
&self,
|
||||
_request: &mut ChatRequest,
|
||||
_options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> vllm_chat::Result<DynChatOutputProcessor> {
|
||||
Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for FailingDecodeBackend {
|
||||
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
|
||||
self.inner.render(request)
|
||||
}
|
||||
}
|
||||
|
||||
/// Skip `LogprobsDelta` events that carry only token_ids (no logprobs),
|
||||
/// returning the next semantically interesting event.
|
||||
async fn next_semantic<S>(stream: &mut S) -> Option<Result<ChatEvent, vllm_chat::Error>>
|
||||
@@ -738,7 +662,12 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output("chat-4", vec![b'i' as u32], None, None)],
|
||||
outputs: vec![request_output(
|
||||
"chat-4",
|
||||
vec![UNKNOWN_DECODE_TOKEN_ID],
|
||||
None,
|
||||
None,
|
||||
)],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -747,9 +676,8 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
|
||||
},
|
||||
);
|
||||
|
||||
let backend: Arc<dyn ChatTextBackend> = Arc::new(FailingDecodeBackend {
|
||||
inner: FakeChatBackend::new(),
|
||||
});
|
||||
let backend: Arc<dyn ChatTextBackend> =
|
||||
Arc::new(FakeChatBackend::new().with_tokenizer(Arc::new(TestTokenizer::new())));
|
||||
let chat = connect_chat_llm_with_ipc(
|
||||
EngineCoreClientConfig::new_single(handshake_address),
|
||||
&ipc,
|
||||
@@ -769,7 +697,10 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
|
||||
|
||||
match timeout(Duration::from_secs(2), stream.next()).await.unwrap() {
|
||||
Some(Err(vllm_chat::Error::Text(vllm_text::Error::Tokenizer(message)))) => {
|
||||
assert_eq!(message, "decode failed");
|
||||
assert_eq!(
|
||||
message,
|
||||
format!("test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}")
|
||||
);
|
||||
}
|
||||
other => panic!("unexpected event after close: {other:?}"),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user