[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-30 15:34:43 +01:00
committed by GitHub
parent 1ab9522935
commit 7a327f0b4f
28 changed files with 680 additions and 826 deletions
+1
View File
@@ -31,6 +31,7 @@ serial_test.workspace = true
tempfile.workspace = true
tokio.workspace = true
vllm-llm = { workspace = true, features = ["test-util"] }
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
[lints]
workspace = true
+4 -57
View File
@@ -275,6 +275,7 @@ mod tests {
use std::collections::{BTreeSet, HashMap};
use serial_test::file_serial;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
use crate::backend::hf::HfTextBackend;
@@ -282,60 +283,8 @@ mod tests {
use crate::error::{LogprobsError, TokenIdsError};
use crate::request::{Prompt, TextRequest};
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
/// don't exercise bad-words tokenization.
struct StubTokenizer;
impl Tokenizer for StubTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(vec![])
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
}
fn stub_tokenizer() -> StubTokenizer {
StubTokenizer
}
struct FixedTokenizer {
token_ids: Vec<u32>,
}
impl Tokenizer for FixedTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(self.token_ids.clone())
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
fn stub_tokenizer() -> TestTokenizer {
TestTokenizer::new()
}
fn sample_request() -> TextRequest {
@@ -952,9 +901,7 @@ mod tests {
#[test]
fn lower_sampling_params_rejects_out_of_vocab_bad_words() {
let tokenizer = FixedTokenizer {
token_ids: vec![1999, 2000],
};
let tokenizer = TestTokenizer::new().with_regular_token("blocked", 2000);
let error = lower_sampling_params(
SamplingParams {
bad_words: Some(vec!["blocked".to_string()]),
+3 -29
View File
@@ -323,37 +323,11 @@ mod tests {
use futures::{Stream, stream};
use vllm_engine_core_client::AbortCause;
use vllm_llm::GenerateOutput;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
use crate::output::TextOutputStreamExt as _;
/// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
struct ByteTokenizer;
impl Tokenizer for ByteTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
unreachable!()
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
/// Helper: run `decoded_text_event_stream` to completion and return the
/// collected output.
async fn run_to_completion(
@@ -366,7 +340,7 @@ mod tests {
token_ids,
Some(FinishReason::Length),
))]);
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
decoded_text_event_stream("test".into(), tokenizer, raw_stream, decode_options, false)
.collect_output()
.await
@@ -419,7 +393,7 @@ mod tests {
))),
dropped_cause: Arc::clone(&dropped_cause),
};
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
let output = decoded_text_event_stream(
"test".into(),
+3 -30
View File
@@ -129,40 +129,13 @@ fn decode_position_logprobs<T: Tokenizer + ?Sized>(
#[cfg(test)]
mod tests {
use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob};
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
#[derive(Debug)]
struct ByteTokenizer;
impl vllm_tokenizer::Tokenizer for ByteTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
unreachable!()
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::from_utf8_lossy(
&token_ids.iter().map(|token_id| *token_id as u8).collect::<Vec<_>>(),
)
.into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
#[test]
fn decode_logprobs_decodes_every_candidate_token() {
let tokenizer = ByteTokenizer;
let tokenizer = TestTokenizer::new();
let logprobs = Logprobs {
positions: vec![PositionLogprobs {
entries: vec![
@@ -205,7 +178,7 @@ mod tests {
#[test]
fn decode_prompt_logprobs_separates_first_prompt_token() {
let tokenizer = ByteTokenizer;
let tokenizer = TestTokenizer::new();
let logprobs = Logprobs {
positions: vec![PositionLogprobs {
entries: vec![TokenLogprob {