[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-30 15:34:43 +01:00
committed by GitHub
parent 1ab9522935
commit 7a327f0b4f
28 changed files with 680 additions and 826 deletions
+1
View File
@@ -23,6 +23,7 @@ expect-test.workspace = true
futures.workspace = true
openai-protocol.workspace = true
tool-parser.workspace = true
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
[[bench]]
name = "deepseek_v3"
+4
View File
@@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer {
fn token_to_id(&self, _token: &str) -> Option<u32> {
Some(u32::MAX)
}
fn id_to_token(&self, _id: u32) -> Option<String> {
Some("\u{FFFD}".to_string())
}
}
/// Bench-only adapter that exposes a unified parser through the tool-parser
+15 -12
View File
@@ -49,11 +49,14 @@ mod tests {
use std::sync::Arc;
use super::SeedOssReasoningParser;
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
use crate::reasoning::{
ReasoningParser,
tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer},
};
#[test]
fn without_prompt_markers_expects_start_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("implicit reasoning</seed:think>answer").unwrap();
@@ -66,10 +69,10 @@ mod tests {
#[test]
fn picks_up_prompt_start_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
// Prompt prefills `<seed:think>` (id 10), opening reasoning before the stream.
parser.initialize(&[10]).unwrap();
// Prompt prefills `<seed:think>`, opening reasoning before the stream.
parser.initialize(&[SEED_THINK_START_ID]).unwrap();
let delta = parser.push("reason</seed:think>answer").unwrap();
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -78,10 +81,10 @@ mod tests {
#[test]
fn respects_prompt_end_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
// Prompt already closed reasoning with `</seed:think>` (id 11).
parser.initialize(&[11]).unwrap();
// Prompt already closed reasoning with `</seed:think>`.
parser.initialize(&[SEED_THINK_END_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
@@ -91,7 +94,7 @@ mod tests {
#[test]
fn handles_explicit_start_token() {
// An explicit start delimiter must not leak into reasoning text.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<seed:think>reason</seed:think>answer").unwrap();
@@ -103,7 +106,7 @@ mod tests {
fn streams_explicit_start_token_across_pushes() {
// Start token, reasoning body, end token, and content arrive in separate
// streaming deltas.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
let mut reasoning = String::new();
@@ -131,9 +134,9 @@ mod tests {
#[test]
fn handles_partial_delimiters_across_pushes() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[10]).unwrap();
parser.initialize(&[SEED_THINK_START_ID]).unwrap();
// Closing delimiter `</seed:think>` arrives in two halves.
let first = parser.push("reason</seed:").unwrap();
+20 -17
View File
@@ -127,14 +127,17 @@ mod tests {
use std::sync::Arc;
use super::Step3p5ReasoningParser;
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
use crate::reasoning::{
ReasoningParser,
tests::{THINK_START_ID, fake_tokenizer},
};
#[test]
fn picks_up_prompt_start_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
// Prompt prefills `<think>` (id 1), opening reasoning before the stream.
parser.initialize(&[1]).unwrap();
// Prompt prefills `<think>`, opening reasoning before the stream.
parser.initialize(&[THINK_START_ID]).unwrap();
let delta = parser.push("This is a reasoning section</think>This is the rest").unwrap();
assert_eq!(
@@ -146,7 +149,7 @@ mod tests {
#[test]
fn handles_unterminated_reasoning() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let pushed = parser.push("<think>reason without end").unwrap();
@@ -159,7 +162,7 @@ mod tests {
#[test]
fn handles_empty_input() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let pushed = parser.push("").unwrap();
@@ -172,9 +175,9 @@ mod tests {
fn complex_newline_pattern_trims_only_single_framing_newline_each_side() {
// Only the immediately-adjacent framing `\n` is dropped on each side of
// `</think>`; surrounding newlines remain part of reasoning/content.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[1]).unwrap();
parser.initialize(&[THINK_START_ID]).unwrap();
let delta = parser
.push("\n This is a \n reasoning section\n\n\n</think>\n\nThis is the rest")
@@ -188,7 +191,7 @@ mod tests {
#[test]
fn drops_framing_newlines_in_single_push() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>reason\n</think>\nanswer").unwrap();
@@ -198,7 +201,7 @@ mod tests {
#[test]
fn drops_framing_newlines_across_pushes() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
// The trailing `\n` from the first push is held until we know whether
@@ -219,7 +222,7 @@ mod tests {
#[test]
fn replays_held_newline_when_more_reasoning_follows() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let first = parser.push("<think>reason\n").unwrap();
@@ -232,7 +235,7 @@ mod tests {
#[test]
fn finish_flushes_held_newline_in_unterminated_stream() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let first = parser.push("<think>reason\n").unwrap();
@@ -245,7 +248,7 @@ mod tests {
#[test]
fn preserves_inner_newlines_in_reasoning() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>line1\nline2</think>tail").unwrap();
@@ -257,7 +260,7 @@ mod tests {
fn trims_only_one_trailing_reasoning_newline() {
// Only the single framing newline immediately before `</think>` is
// dropped; earlier newlines in the reasoning body are preserved.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>reason\n\n</think>answer").unwrap();
@@ -269,7 +272,7 @@ mod tests {
fn drops_only_first_content_newline_after_transition() {
// The leading-`\n` drop applies only to the first content delta after
// `</think>`; later deltas pass through untouched.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let first = parser.push("<think>reason</think>").unwrap();
@@ -288,7 +291,7 @@ mod tests {
#[test]
fn passes_through_clean_boundary_without_framing_newlines() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>reason</think>tail").unwrap();
@@ -298,7 +301,7 @@ mod tests {
#[test]
fn handles_empty_reasoning_section() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think></think>answer").unwrap();
+48 -60
View File
@@ -1,54 +1,42 @@
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::{
DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser,
Qwen3ReasoningParser, ReasoningParser,
};
pub(crate) struct FakeTokenizer;
pub(crate) const THINK_START_ID: u32 = 256;
pub(crate) const THINK_END_ID: u32 = 257;
pub(crate) const START_THINKING_ID: u32 = 258;
pub(crate) const END_THINKING_ID: u32 = 259;
pub(crate) const MINIMAX_THINK_START_ID: u32 = 260;
pub(crate) const MINIMAX_THINK_END_ID: u32 = 261;
pub(crate) const SPECIAL_BOUNDARY_ID: u32 = 262;
pub(crate) const MM_THINK_START_ID: u32 = 263;
pub(crate) const MM_THINK_END_ID: u32 = 264;
pub(crate) const SEED_THINK_START_ID: u32 = 265;
pub(crate) const SEED_THINK_END_ID: u32 = 266;
impl Tokenizer for FakeTokenizer {
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(1),
"</think>" => Some(2),
"<|START_THINKING|>" => Some(3),
"<|END_THINKING|>" => Some(4),
"◁think▷" => Some(5),
"◁/think▷" => Some(6),
"<mm:think>" => Some(8),
"</mm:think>" => Some(9),
"<seed:think>" => Some(10),
"</seed:think>" => Some(11),
_ => None,
}
}
fn is_special_id(&self, token_id: u32) -> bool {
token_id == 7
}
pub(crate) fn fake_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<think>", THINK_START_ID)
.with_regular_token("</think>", THINK_END_ID)
.with_regular_token("<|START_THINKING|>", START_THINKING_ID)
.with_regular_token("<|END_THINKING|>", END_THINKING_ID)
.with_regular_token("◁think▷", MINIMAX_THINK_START_ID)
.with_regular_token("◁/think▷", MINIMAX_THINK_END_ID)
.with_special_token("<special-boundary>", SPECIAL_BOUNDARY_ID)
.with_regular_token("<mm:think>", MM_THINK_START_ID)
.with_regular_token("</mm:think>", MM_THINK_END_ID)
.with_regular_token("<seed:think>", SEED_THINK_START_ID)
.with_regular_token("</seed:think>", SEED_THINK_END_ID)
}
#[test]
fn delimited_content_only_stream() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
@@ -60,7 +48,7 @@ fn delimited_content_only_stream() {
#[test]
fn delimited_single_chunk_with_reasoning_and_content() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
@@ -71,7 +59,7 @@ fn delimited_single_chunk_with_reasoning_and_content() {
#[test]
fn delimited_partial_tokens_across_chunks() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
@@ -83,10 +71,10 @@ fn delimited_partial_tokens_across_chunks() {
#[test]
fn delimited_finish_flushes_buffer() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
parser.initialize(&[1]);
parser.initialize(&[THINK_START_ID]);
let delta = parser.push("unfinished</thi");
assert_eq!(delta.reasoning.as_deref(), Some("unfinished"));
@@ -96,7 +84,7 @@ fn delimited_finish_flushes_buffer() {
#[test]
fn qwen3_without_prompt_markers_expects_start_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("reason</think>answer").unwrap();
@@ -106,9 +94,9 @@ fn qwen3_without_prompt_markers_expects_start_token() {
#[test]
fn qwen3_prompt_end_marker_starts_in_content() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[2]).unwrap();
parser.initialize(&[THINK_END_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
@@ -117,7 +105,7 @@ fn qwen3_prompt_end_marker_starts_in_content() {
#[test]
fn qwen3_tolerates_old_and_new_formats() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
let old = old_parser.push("<think>reason</think>answer").unwrap();
@@ -125,7 +113,7 @@ fn qwen3_tolerates_old_and_new_formats() {
assert_eq!(old.content.as_deref(), Some("answer"));
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
new_parser.initialize(&[1]).unwrap();
new_parser.initialize(&[THINK_START_ID]).unwrap();
let new = new_parser.push("reason</think>answer").unwrap();
assert_eq!(new.reasoning.as_deref(), Some("reason"));
assert_eq!(new.content.as_deref(), Some("answer"));
@@ -133,10 +121,10 @@ fn qwen3_tolerates_old_and_new_formats() {
#[test]
fn qwen3_stops_scanning_at_last_special_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[1, 7]).unwrap();
parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
@@ -145,7 +133,7 @@ fn qwen3_stops_scanning_at_last_special_token() {
#[test]
fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("reason</think>answer").unwrap();
@@ -155,10 +143,10 @@ fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
#[test]
fn deepseek_r1_stops_scanning_at_last_special_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[2, 7]).unwrap();
parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap();
let delta = parser.push("reason</think>answer").unwrap();
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -167,7 +155,7 @@ fn deepseek_r1_stops_scanning_at_last_special_token() {
#[test]
fn minimax_m3_handles_explicit_think_delimiters() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<mm:think>reason</mm:think>answer").unwrap();
@@ -177,7 +165,7 @@ fn minimax_m3_handles_explicit_think_delimiters() {
#[test]
fn minimax_m3_drops_leading_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("</mm:think>answer").unwrap();
@@ -187,7 +175,7 @@ fn minimax_m3_drops_leading_end_marker() {
#[test]
fn minimax_m3_preserves_non_leading_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("XXX</mm:think>YYY").unwrap();
@@ -197,7 +185,7 @@ fn minimax_m3_preserves_non_leading_end_marker() {
#[test]
fn minimax_m3_drops_split_leading_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
assert!(parser.push("</mm").unwrap().is_empty());
@@ -208,9 +196,9 @@ fn minimax_m3_drops_split_leading_end_marker() {
#[test]
fn minimax_m3_uses_prompt_prefilled_start_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[8]).unwrap();
parser.initialize(&[MM_THINK_START_ID]).unwrap();
let delta = parser.push("reason</mm:think>answer").unwrap();
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -219,9 +207,9 @@ fn minimax_m3_uses_prompt_prefilled_start_marker() {
#[test]
fn minimax_m3_uses_prompt_prefilled_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[9]).unwrap();
parser.initialize(&[MM_THINK_END_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
+6 -31
View File
@@ -124,42 +124,17 @@ impl UnifiedParser for CombinedParser {
mod tests {
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::CombinedParser;
use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};
use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};
use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(1),
"</think>" => Some(2),
_ => None,
}
}
fn tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<think>", 256)
.with_regular_token("</think>", 257)
}
fn test_tools() -> Vec<Tool> {
@@ -273,7 +248,7 @@ mod tests {
#[test]
fn combined_parser_emits_reasoning_and_text() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(tokenizer());
let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
let mut parser = CombinedParser::new(Some(reasoning), None);
+15 -66
View File
@@ -515,7 +515,7 @@ mod tests {
use serde_json::{Value, json};
use thiserror_ext::AsReport;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use winnow::combinator::{eof, terminated};
use winnow::error::ErrMode;
use winnow::prelude::*;
@@ -527,66 +527,15 @@ mod tests {
use crate::tool::Tool;
use crate::unified::{UnifiedParserEvent, parsing_failed};
struct FakeTokenizer;
const CHANNEL_START_ID: u32 = 256;
const CHANNEL_END_ID: u32 = 257;
const TURN_BOUNDARY_ID: u32 = 258;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
CHANNEL_START => Some(100),
CHANNEL_END => Some(101),
_ => None,
}
}
fn is_special_id(&self, token_id: u32) -> bool {
matches!(token_id, 100..=105)
}
}
struct MissingTokenTokenizer;
impl Tokenizer for MissingTokenTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
fn tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_special_token(CHANNEL_START, CHANNEL_START_ID)
.with_special_token(CHANNEL_END, CHANNEL_END_ID)
.with_special_token("<turn-boundary>", TURN_BOUNDARY_ID)
}
trait UnifiedParserTestExt {
@@ -716,12 +665,12 @@ mod tests {
}
fn test_parser() -> Gemma4UnifiedParser {
Gemma4UnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()
Gemma4UnifiedParser::new(&test_tools(), Arc::new(tokenizer())).unwrap()
}
#[test]
fn gemma4_create_requires_channel_start_token() {
let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(MissingTokenTokenizer)) {
let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(TestTokenizer::new())) {
Ok(_) => panic!("expected missing token error"),
Err(error) => error,
};
@@ -1046,7 +995,7 @@ mod tests {
#[test]
fn gemma4_initialize_open_channel_prompt_starts_in_reasoning() {
let mut parser = test_parser();
parser.initialize(&[100, 3000, 3001]).unwrap();
parser.initialize(&[CHANNEL_START_ID, 3000, 3001]).unwrap();
let output = parser.parse_complete("reason<channel|>answer").unwrap();
@@ -1057,7 +1006,7 @@ mod tests {
#[test]
fn gemma4_initialize_turn_prompt_starts_in_text() {
let mut parser = test_parser();
parser.initialize(&[104, 3000, 3001]).unwrap();
parser.initialize(&[TURN_BOUNDARY_ID, 3000, 3001]).unwrap();
let output = parser.parse_complete("<|channel>thought\nreason<channel|>answer").unwrap();
@@ -1068,7 +1017,7 @@ mod tests {
#[test]
fn gemma4_initialize_special_token_caps_boundary_scan() {
let mut parser = test_parser();
parser.initialize(&[100, 3000, 104, 3001]).unwrap();
parser.initialize(&[CHANNEL_START_ID, 3000, TURN_BOUNDARY_ID, 3001]).unwrap();
let output = parser.parse_complete("answer").unwrap();
@@ -1079,7 +1028,7 @@ mod tests {
#[test]
fn gemma4_initialize_closed_channel_prompt_starts_in_text() {
let mut parser = test_parser();
parser.initialize(&[100, 3000, 3001, 101]).unwrap();
parser.initialize(&[CHANNEL_START_ID, 3000, 3001, CHANNEL_END_ID]).unwrap();
let output = parser.parse_complete("answer").unwrap();