[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -31,6 +31,7 @@ serial_test.workspace = true
|
||||
tempfile.workspace = true
|
||||
tokio.workspace = true
|
||||
vllm-llm = { workspace = true, features = ["test-util"] }
|
||||
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -275,6 +275,7 @@ mod tests {
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use serial_test::file_serial;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
use crate::backend::hf::HfTextBackend;
|
||||
@@ -282,60 +283,8 @@ mod tests {
|
||||
use crate::error::{LogprobsError, TokenIdsError};
|
||||
use crate::request::{Prompt, TextRequest};
|
||||
|
||||
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
|
||||
/// don't exercise bad-words tokenization.
|
||||
struct StubTokenizer;
|
||||
|
||||
impl Tokenizer for StubTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn stub_tokenizer() -> StubTokenizer {
|
||||
StubTokenizer
|
||||
}
|
||||
|
||||
struct FixedTokenizer {
|
||||
token_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
impl Tokenizer for FixedTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(self.token_ids.clone())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
fn stub_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
}
|
||||
|
||||
fn sample_request() -> TextRequest {
|
||||
@@ -952,9 +901,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_out_of_vocab_bad_words() {
|
||||
let tokenizer = FixedTokenizer {
|
||||
token_ids: vec![1999, 2000],
|
||||
};
|
||||
let tokenizer = TestTokenizer::new().with_regular_token("blocked", 2000);
|
||||
let error = lower_sampling_params(
|
||||
SamplingParams {
|
||||
bad_words: Some(vec!["blocked".to_string()]),
|
||||
|
||||
@@ -323,37 +323,11 @@ mod tests {
|
||||
use futures::{Stream, stream};
|
||||
use vllm_engine_core_client::AbortCause;
|
||||
use vllm_llm::GenerateOutput;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
use crate::output::TextOutputStreamExt as _;
|
||||
|
||||
/// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
|
||||
struct ByteTokenizer;
|
||||
|
||||
impl Tokenizer for ByteTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
|
||||
Ok(String::from_utf8_lossy(&bytes).into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper: run `decoded_text_event_stream` to completion and return the
|
||||
/// collected output.
|
||||
async fn run_to_completion(
|
||||
@@ -366,7 +340,7 @@ mod tests {
|
||||
token_ids,
|
||||
Some(FinishReason::Length),
|
||||
))]);
|
||||
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
decoded_text_event_stream("test".into(), tokenizer, raw_stream, decode_options, false)
|
||||
.collect_output()
|
||||
.await
|
||||
@@ -419,7 +393,7 @@ mod tests {
|
||||
))),
|
||||
dropped_cause: Arc::clone(&dropped_cause),
|
||||
};
|
||||
let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
|
||||
let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
|
||||
|
||||
let output = decoded_text_event_stream(
|
||||
"test".into(),
|
||||
|
||||
@@ -129,40 +129,13 @@ fn decode_position_logprobs<T: Tokenizer + ?Sized>(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ByteTokenizer;
|
||||
|
||||
impl vllm_tokenizer::Tokenizer for ByteTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::from_utf8_lossy(
|
||||
&token_ids.iter().map(|token_id| *token_id as u8).collect::<Vec<_>>(),
|
||||
)
|
||||
.into_owned())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_logprobs_decodes_every_candidate_token() {
|
||||
let tokenizer = ByteTokenizer;
|
||||
let tokenizer = TestTokenizer::new();
|
||||
let logprobs = Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![
|
||||
@@ -205,7 +178,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn decode_prompt_logprobs_separates_first_prompt_token() {
|
||||
let tokenizer = ByteTokenizer;
|
||||
let tokenizer = TestTokenizer::new();
|
||||
let logprobs = Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![TokenLogprob {
|
||||
|
||||
Reference in New Issue
Block a user