[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -64,6 +64,7 @@ tempfile.workspace = true
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
tower.workspace = true
|
||||
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
|
||||
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
|
||||
zeromq.workspace = true
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_text::{Prompt, TextBackend};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
@@ -155,37 +156,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
|
||||
#[derive(Clone, Debug)]
|
||||
struct FakeTextBackend;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(
|
||||
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
|
||||
.into_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
token.bytes().next().map(u32::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for FakeTextBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FakeTokenizer)
|
||||
Arc::new(TestTokenizer::new())
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
|
||||
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_text::{Prompt, TextBackend};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
@@ -151,37 +152,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
|
||||
#[derive(Clone, Debug)]
|
||||
struct FakeChatBackend;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FakeChatTokenizer;
|
||||
|
||||
impl Tokenizer for FakeChatTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(
|
||||
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
|
||||
.into_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
token.bytes().next().map(u32::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for FakeChatBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FakeChatTokenizer)
|
||||
Arc::new(TestTokenizer::new())
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
|
||||
@@ -194,7 +194,7 @@ mod tests {
|
||||
use axum::http::HeaderMap;
|
||||
use serde_json::json;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::prepare_completion_request;
|
||||
use crate::lora::LoraModelResolution;
|
||||
@@ -212,32 +212,8 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestTokenizer;
|
||||
|
||||
impl Tokenizer for TestTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(
|
||||
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
|
||||
.into_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
fn test_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
}
|
||||
|
||||
fn base_request_json() -> serde_json::Value {
|
||||
@@ -297,7 +273,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -340,7 +316,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare")
|
||||
.text_request
|
||||
@@ -374,7 +350,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -399,7 +375,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -422,7 +398,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -446,7 +422,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -471,7 +447,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -498,7 +474,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -523,7 +499,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -549,7 +525,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1));
|
||||
@@ -574,7 +550,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
request_context(&headers, None),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
|
||||
@@ -593,7 +569,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&TestTokenizer,
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare");
|
||||
assert_eq!(prepared.text_request.data_parallel_rank, None);
|
||||
|
||||
@@ -40,8 +40,9 @@ use vllm_engine_core_client::{
|
||||
};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_metrics::METRICS;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_text::{Prompt, TextBackend};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
@@ -417,82 +418,20 @@ struct FakeChatBackend {
|
||||
}
|
||||
|
||||
/// Synthetic BOS id used when `add_special_tokens` is true in tests.
|
||||
const FAKE_BOS_TOKEN_ID: u32 = 1;
|
||||
const FAKE_BOS_TOKEN_ID: u32 = 256;
|
||||
const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FakeChatTokenizer;
|
||||
|
||||
impl Tokenizer for FakeChatTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
let mut token_ids = Vec::new();
|
||||
if add_special_tokens {
|
||||
token_ids.push(FAKE_BOS_TOKEN_ID);
|
||||
}
|
||||
let mut rest = text;
|
||||
while !rest.is_empty() {
|
||||
if let Some(stripped) = rest.strip_prefix("<image>") {
|
||||
token_ids.push(999);
|
||||
rest = stripped;
|
||||
continue;
|
||||
}
|
||||
if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {
|
||||
token_ids.push(151655);
|
||||
rest = stripped;
|
||||
continue;
|
||||
}
|
||||
|
||||
let ch = rest.chars().next().expect("rest is not empty");
|
||||
let mut buf = [0; 4];
|
||||
token_ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from));
|
||||
rest = &rest[ch.len_utf8()..];
|
||||
}
|
||||
Ok(token_ids)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(
|
||||
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
|
||||
.into_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<image>" => Some(999),
|
||||
"<|image_pad|>" => Some(151655),
|
||||
"<think>" => Some(0xF001),
|
||||
"</think>" => Some(0xF002),
|
||||
"<|START_THINKING|>" => Some(0xF003),
|
||||
"<|END_THINKING|>" => Some(0xF004),
|
||||
"◁think▷" => Some(0xF005),
|
||||
"◁/think▷" => Some(0xF006),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
match id {
|
||||
FAKE_BOS_TOKEN_ID => Some("<bos>".to_string()),
|
||||
999 => Some("<image>".to_string()),
|
||||
151655 => Some("<|image_pad|>".to_string()),
|
||||
0xF001 => Some("<think>".to_string()),
|
||||
0xF002 => Some("</think>".to_string()),
|
||||
0xF003 => Some("<|START_THINKING|>".to_string()),
|
||||
0xF004 => Some("<|END_THINKING|>".to_string()),
|
||||
0xF005 => Some("◁think▷".to_string()),
|
||||
0xF006 => Some("◁/think▷".to_string()),
|
||||
id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn fake_chat_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_bos_token("<bos>", FAKE_BOS_TOKEN_ID)
|
||||
.with_regular_token("<image>", 999)
|
||||
.with_regular_token("<|image_pad|>", 151655)
|
||||
.with_regular_token("<think>", 0xF001)
|
||||
.with_regular_token("</think>", 0xF002)
|
||||
.with_regular_token("<|START_THINKING|>", 0xF003)
|
||||
.with_regular_token("<|END_THINKING|>", 0xF004)
|
||||
.with_regular_token("◁think▷", 0xF005)
|
||||
.with_regular_token("◁/think▷", 0xF006)
|
||||
}
|
||||
|
||||
impl FakeChatBackend {
|
||||
@@ -530,7 +469,7 @@ impl fmt::Debug for FakeChatBackend {
|
||||
|
||||
impl TextBackend for FakeChatBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FakeChatTokenizer)
|
||||
Arc::new(fake_chat_tokenizer())
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
@@ -630,7 +569,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
Some("qwen2_vl".to_string()),
|
||||
Some(&config_path),
|
||||
None,
|
||||
Arc::new(FakeChatTokenizer),
|
||||
Arc::new(fake_chat_tokenizer()),
|
||||
)
|
||||
.expect("load multimodal info")
|
||||
.expect("qwen multimodal info is registered");
|
||||
@@ -638,70 +577,6 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
|
||||
info
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct FailingDecodeChatBackend;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FailingDecodeTokenizer;
|
||||
|
||||
impl Tokenizer for FailingDecodeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
FakeChatTokenizer.encode(text, add_special_tokens)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
if token_ids.contains(&(b'i' as u32)) {
|
||||
return Err(vllm_text::tokenizer::TokenizerError(
|
||||
"forced decode failure for streaming test".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
FakeChatTokenizer.decode(token_ids, skip_special_tokens)
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
FakeChatTokenizer.token_to_id(token)
|
||||
}
|
||||
}
|
||||
|
||||
impl TextBackend for FailingDecodeChatBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
Arc::new(FailingDecodeTokenizer)
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
"test-model"
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for FailingDecodeChatBackend {
|
||||
fn chat_renderer(&self) -> DynChatRenderer {
|
||||
Arc::new(self.clone())
|
||||
}
|
||||
|
||||
fn new_chat_output_processor(
|
||||
&self,
|
||||
_request: &mut ChatRequest,
|
||||
_options: NewChatOutputProcessorOptions<'_>,
|
||||
) -> vllm_chat::Result<DynChatOutputProcessor> {
|
||||
Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for FailingDecodeChatBackend {
|
||||
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<vllm_chat::RenderedPrompt> {
|
||||
FakeChatBackend::new().render(request)
|
||||
}
|
||||
}
|
||||
|
||||
async fn test_models_with_engine_outputs_and_backend_inner(
|
||||
engine_id: impl Into<EngineId>,
|
||||
output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
|
||||
@@ -2755,8 +2630,8 @@ async fn load_endpoint_resets_when_stream_response_is_dropped() {
|
||||
#[serial]
|
||||
async fn stream_error_is_returned_as_openai_error_sse() {
|
||||
let (app, engine_task) = test_app_with_backend_and_stream_output_specs(
|
||||
Arc::new(FailingDecodeChatBackend),
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
vec![(vec![UNKNOWN_DECODE_TOKEN_ID], None)],
|
||||
)
|
||||
.await;
|
||||
let response = app
|
||||
@@ -2789,7 +2664,9 @@ async fn stream_error_is_returned_as_openai_error_sse() {
|
||||
assert!(text.contains("\"role\":\"assistant\""), "{text}");
|
||||
assert!(text.contains("\"type\":\"server_error\""), "{text}");
|
||||
assert!(
|
||||
text.contains("forced decode failure for streaming test"),
|
||||
text.contains(&format!(
|
||||
"test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}"
|
||||
)),
|
||||
"{text}"
|
||||
);
|
||||
assert!(!text.contains("\"usage\":"), "{text}");
|
||||
|
||||
Reference in New Issue
Block a user