[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -23,6 +23,7 @@ expect-test.workspace = true
|
||||
futures.workspace = true
|
||||
openai-protocol.workspace = true
|
||||
tool-parser.workspace = true
|
||||
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
|
||||
|
||||
[[bench]]
|
||||
name = "deepseek_v3"
|
||||
|
||||
@@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer {
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
Some(u32::MAX)
|
||||
}
|
||||
|
||||
fn id_to_token(&self, _id: u32) -> Option<String> {
|
||||
Some("\u{FFFD}".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bench-only adapter that exposes a unified parser through the tool-parser
|
||||
|
||||
@@ -49,11 +49,14 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::SeedOssReasoningParser;
|
||||
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
|
||||
use crate::reasoning::{
|
||||
ReasoningParser,
|
||||
tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn without_prompt_markers_expects_start_token() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("implicit reasoning</seed:think>answer").unwrap();
|
||||
@@ -66,10 +69,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn picks_up_prompt_start_boundary() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
|
||||
// Prompt prefills `<seed:think>` (id 10), opening reasoning before the stream.
|
||||
parser.initialize(&[10]).unwrap();
|
||||
// Prompt prefills `<seed:think>`, opening reasoning before the stream.
|
||||
parser.initialize(&[SEED_THINK_START_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("reason</seed:think>answer").unwrap();
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
|
||||
@@ -78,10 +81,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn respects_prompt_end_boundary() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
|
||||
// Prompt already closed reasoning with `</seed:think>` (id 11).
|
||||
parser.initialize(&[11]).unwrap();
|
||||
// Prompt already closed reasoning with `</seed:think>`.
|
||||
parser.initialize(&[SEED_THINK_END_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("answer").unwrap();
|
||||
assert_eq!(delta.reasoning, None);
|
||||
@@ -91,7 +94,7 @@ mod tests {
|
||||
#[test]
|
||||
fn handles_explicit_start_token() {
|
||||
// An explicit start delimiter must not leak into reasoning text.
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("<seed:think>reason</seed:think>answer").unwrap();
|
||||
@@ -103,7 +106,7 @@ mod tests {
|
||||
fn streams_explicit_start_token_across_pushes() {
|
||||
// Start token, reasoning body, end token, and content arrive in separate
|
||||
// streaming deltas.
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let mut reasoning = String::new();
|
||||
@@ -131,9 +134,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn handles_partial_delimiters_across_pushes() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[10]).unwrap();
|
||||
parser.initialize(&[SEED_THINK_START_ID]).unwrap();
|
||||
|
||||
// Closing delimiter `</seed:think>` arrives in two halves.
|
||||
let first = parser.push("reason</seed:").unwrap();
|
||||
|
||||
@@ -127,14 +127,17 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::Step3p5ReasoningParser;
|
||||
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
|
||||
use crate::reasoning::{
|
||||
ReasoningParser,
|
||||
tests::{THINK_START_ID, fake_tokenizer},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn picks_up_prompt_start_boundary() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
// Prompt prefills `<think>` (id 1), opening reasoning before the stream.
|
||||
parser.initialize(&[1]).unwrap();
|
||||
// Prompt prefills `<think>`, opening reasoning before the stream.
|
||||
parser.initialize(&[THINK_START_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("This is a reasoning section</think>This is the rest").unwrap();
|
||||
assert_eq!(
|
||||
@@ -146,7 +149,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn handles_unterminated_reasoning() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let pushed = parser.push("<think>reason without end").unwrap();
|
||||
@@ -159,7 +162,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn handles_empty_input() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let pushed = parser.push("").unwrap();
|
||||
@@ -172,9 +175,9 @@ mod tests {
|
||||
fn complex_newline_pattern_trims_only_single_framing_newline_each_side() {
|
||||
// Only the immediately-adjacent framing `\n` is dropped on each side of
|
||||
// `</think>`; surrounding newlines remain part of reasoning/content.
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[1]).unwrap();
|
||||
parser.initialize(&[THINK_START_ID]).unwrap();
|
||||
|
||||
let delta = parser
|
||||
.push("\n This is a \n reasoning section\n\n\n</think>\n\nThis is the rest")
|
||||
@@ -188,7 +191,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn drops_framing_newlines_in_single_push() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("<think>reason\n</think>\nanswer").unwrap();
|
||||
@@ -198,7 +201,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn drops_framing_newlines_across_pushes() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
// The trailing `\n` from the first push is held until we know whether
|
||||
@@ -219,7 +222,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replays_held_newline_when_more_reasoning_follows() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let first = parser.push("<think>reason\n").unwrap();
|
||||
@@ -232,7 +235,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn finish_flushes_held_newline_in_unterminated_stream() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let first = parser.push("<think>reason\n").unwrap();
|
||||
@@ -245,7 +248,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn preserves_inner_newlines_in_reasoning() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("<think>line1\nline2</think>tail").unwrap();
|
||||
@@ -257,7 +260,7 @@ mod tests {
|
||||
fn trims_only_one_trailing_reasoning_newline() {
|
||||
// Only the single framing newline immediately before `</think>` is
|
||||
// dropped; earlier newlines in the reasoning body are preserved.
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("<think>reason\n\n</think>answer").unwrap();
|
||||
@@ -269,7 +272,7 @@ mod tests {
|
||||
fn drops_only_first_content_newline_after_transition() {
|
||||
// The leading-`\n` drop applies only to the first content delta after
|
||||
// `</think>`; later deltas pass through untouched.
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let first = parser.push("<think>reason</think>").unwrap();
|
||||
@@ -288,7 +291,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn passes_through_clean_boundary_without_framing_newlines() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("<think>reason</think>tail").unwrap();
|
||||
@@ -298,7 +301,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn handles_empty_reasoning_section() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("<think></think>answer").unwrap();
|
||||
|
||||
@@ -1,54 +1,42 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::{
|
||||
DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser,
|
||||
Qwen3ReasoningParser, ReasoningParser,
|
||||
};
|
||||
|
||||
pub(crate) struct FakeTokenizer;
|
||||
pub(crate) const THINK_START_ID: u32 = 256;
|
||||
pub(crate) const THINK_END_ID: u32 = 257;
|
||||
pub(crate) const START_THINKING_ID: u32 = 258;
|
||||
pub(crate) const END_THINKING_ID: u32 = 259;
|
||||
pub(crate) const MINIMAX_THINK_START_ID: u32 = 260;
|
||||
pub(crate) const MINIMAX_THINK_END_ID: u32 = 261;
|
||||
pub(crate) const SPECIAL_BOUNDARY_ID: u32 = 262;
|
||||
pub(crate) const MM_THINK_START_ID: u32 = 263;
|
||||
pub(crate) const MM_THINK_END_ID: u32 = 264;
|
||||
pub(crate) const SEED_THINK_START_ID: u32 = 265;
|
||||
pub(crate) const SEED_THINK_END_ID: u32 = 266;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<think>" => Some(1),
|
||||
"</think>" => Some(2),
|
||||
"<|START_THINKING|>" => Some(3),
|
||||
"<|END_THINKING|>" => Some(4),
|
||||
"◁think▷" => Some(5),
|
||||
"◁/think▷" => Some(6),
|
||||
"<mm:think>" => Some(8),
|
||||
"</mm:think>" => Some(9),
|
||||
"<seed:think>" => Some(10),
|
||||
"</seed:think>" => Some(11),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
token_id == 7
|
||||
}
|
||||
pub(crate) fn fake_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<think>", THINK_START_ID)
|
||||
.with_regular_token("</think>", THINK_END_ID)
|
||||
.with_regular_token("<|START_THINKING|>", START_THINKING_ID)
|
||||
.with_regular_token("<|END_THINKING|>", END_THINKING_ID)
|
||||
.with_regular_token("◁think▷", MINIMAX_THINK_START_ID)
|
||||
.with_regular_token("◁/think▷", MINIMAX_THINK_END_ID)
|
||||
.with_special_token("<special-boundary>", SPECIAL_BOUNDARY_ID)
|
||||
.with_regular_token("<mm:think>", MM_THINK_START_ID)
|
||||
.with_regular_token("</mm:think>", MM_THINK_END_ID)
|
||||
.with_regular_token("<seed:think>", SEED_THINK_START_ID)
|
||||
.with_regular_token("</seed:think>", SEED_THINK_END_ID)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delimited_content_only_stream() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
|
||||
@@ -60,7 +48,7 @@ fn delimited_content_only_stream() {
|
||||
|
||||
#[test]
|
||||
fn delimited_single_chunk_with_reasoning_and_content() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
|
||||
@@ -71,7 +59,7 @@ fn delimited_single_chunk_with_reasoning_and_content() {
|
||||
|
||||
#[test]
|
||||
fn delimited_partial_tokens_across_chunks() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
|
||||
@@ -83,10 +71,10 @@ fn delimited_partial_tokens_across_chunks() {
|
||||
|
||||
#[test]
|
||||
fn delimited_finish_flushes_buffer() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser =
|
||||
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
|
||||
parser.initialize(&[1]);
|
||||
parser.initialize(&[THINK_START_ID]);
|
||||
|
||||
let delta = parser.push("unfinished</thi");
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("unfinished"));
|
||||
@@ -96,7 +84,7 @@ fn delimited_finish_flushes_buffer() {
|
||||
|
||||
#[test]
|
||||
fn qwen3_without_prompt_markers_expects_start_token() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("reason</think>answer").unwrap();
|
||||
@@ -106,9 +94,9 @@ fn qwen3_without_prompt_markers_expects_start_token() {
|
||||
|
||||
#[test]
|
||||
fn qwen3_prompt_end_marker_starts_in_content() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[2]).unwrap();
|
||||
parser.initialize(&[THINK_END_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("answer").unwrap();
|
||||
assert_eq!(delta.reasoning, None);
|
||||
@@ -117,7 +105,7 @@ fn qwen3_prompt_end_marker_starts_in_content() {
|
||||
|
||||
#[test]
|
||||
fn qwen3_tolerates_old_and_new_formats() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
|
||||
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
|
||||
let old = old_parser.push("<think>reason</think>answer").unwrap();
|
||||
@@ -125,7 +113,7 @@ fn qwen3_tolerates_old_and_new_formats() {
|
||||
assert_eq!(old.content.as_deref(), Some("answer"));
|
||||
|
||||
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
new_parser.initialize(&[1]).unwrap();
|
||||
new_parser.initialize(&[THINK_START_ID]).unwrap();
|
||||
let new = new_parser.push("reason</think>answer").unwrap();
|
||||
assert_eq!(new.reasoning.as_deref(), Some("reason"));
|
||||
assert_eq!(new.content.as_deref(), Some("answer"));
|
||||
@@ -133,10 +121,10 @@ fn qwen3_tolerates_old_and_new_formats() {
|
||||
|
||||
#[test]
|
||||
fn qwen3_stops_scanning_at_last_special_token() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
parser.initialize(&[1, 7]).unwrap();
|
||||
parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("answer").unwrap();
|
||||
assert_eq!(delta.reasoning, None);
|
||||
@@ -145,7 +133,7 @@ fn qwen3_stops_scanning_at_last_special_token() {
|
||||
|
||||
#[test]
|
||||
fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("reason</think>answer").unwrap();
|
||||
@@ -155,10 +143,10 @@ fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
|
||||
|
||||
#[test]
|
||||
fn deepseek_r1_stops_scanning_at_last_special_token() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
parser.initialize(&[2, 7]).unwrap();
|
||||
parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("reason</think>answer").unwrap();
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
|
||||
@@ -167,7 +155,7 @@ fn deepseek_r1_stops_scanning_at_last_special_token() {
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_handles_explicit_think_delimiters() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("<mm:think>reason</mm:think>answer").unwrap();
|
||||
@@ -177,7 +165,7 @@ fn minimax_m3_handles_explicit_think_delimiters() {
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_drops_leading_end_marker() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("</mm:think>answer").unwrap();
|
||||
@@ -187,7 +175,7 @@ fn minimax_m3_drops_leading_end_marker() {
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_preserves_non_leading_end_marker() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
let delta = parser.push("XXX</mm:think>YYY").unwrap();
|
||||
@@ -197,7 +185,7 @@ fn minimax_m3_preserves_non_leading_end_marker() {
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_drops_split_leading_end_marker() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
|
||||
assert!(parser.push("</mm").unwrap().is_empty());
|
||||
@@ -208,9 +196,9 @@ fn minimax_m3_drops_split_leading_end_marker() {
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_uses_prompt_prefilled_start_marker() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[8]).unwrap();
|
||||
parser.initialize(&[MM_THINK_START_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("reason</mm:think>answer").unwrap();
|
||||
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
|
||||
@@ -219,9 +207,9 @@ fn minimax_m3_uses_prompt_prefilled_start_marker() {
|
||||
|
||||
#[test]
|
||||
fn minimax_m3_uses_prompt_prefilled_end_marker() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(fake_tokenizer());
|
||||
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
|
||||
parser.initialize(&[9]).unwrap();
|
||||
parser.initialize(&[MM_THINK_END_ID]).unwrap();
|
||||
|
||||
let delta = parser.push("answer").unwrap();
|
||||
assert_eq!(delta.reasoning, None);
|
||||
|
||||
@@ -124,42 +124,17 @@ impl UnifiedParser for CombinedParser {
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::CombinedParser;
|
||||
use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};
|
||||
use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};
|
||||
use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<think>" => Some(1),
|
||||
"</think>" => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<think>", 256)
|
||||
.with_regular_token("</think>", 257)
|
||||
}
|
||||
|
||||
fn test_tools() -> Vec<Tool> {
|
||||
@@ -273,7 +248,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn combined_parser_emits_reasoning_and_text() {
|
||||
let tokenizer = Arc::new(FakeTokenizer);
|
||||
let tokenizer = Arc::new(tokenizer());
|
||||
let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
|
||||
let mut parser = CombinedParser::new(Some(reasoning), None);
|
||||
|
||||
|
||||
@@ -515,7 +515,7 @@ mod tests {
|
||||
|
||||
use serde_json::{Value, json};
|
||||
use thiserror_ext::AsReport;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
use winnow::combinator::{eof, terminated};
|
||||
use winnow::error::ErrMode;
|
||||
use winnow::prelude::*;
|
||||
@@ -527,66 +527,15 @@ mod tests {
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{UnifiedParserEvent, parsing_failed};
|
||||
|
||||
struct FakeTokenizer;
|
||||
const CHANNEL_START_ID: u32 = 256;
|
||||
const CHANNEL_END_ID: u32 = 257;
|
||||
const TURN_BOUNDARY_ID: u32 = 258;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
CHANNEL_START => Some(100),
|
||||
CHANNEL_END => Some(101),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
matches!(token_id, 100..=105)
|
||||
}
|
||||
}
|
||||
|
||||
struct MissingTokenTokenizer;
|
||||
|
||||
impl Tokenizer for MissingTokenTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, _token: &str) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
fn tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_special_token(CHANNEL_START, CHANNEL_START_ID)
|
||||
.with_special_token(CHANNEL_END, CHANNEL_END_ID)
|
||||
.with_special_token("<turn-boundary>", TURN_BOUNDARY_ID)
|
||||
}
|
||||
|
||||
trait UnifiedParserTestExt {
|
||||
@@ -716,12 +665,12 @@ mod tests {
|
||||
}
|
||||
|
||||
fn test_parser() -> Gemma4UnifiedParser {
|
||||
Gemma4UnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()
|
||||
Gemma4UnifiedParser::new(&test_tools(), Arc::new(tokenizer())).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemma4_create_requires_channel_start_token() {
|
||||
let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(MissingTokenTokenizer)) {
|
||||
let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(TestTokenizer::new())) {
|
||||
Ok(_) => panic!("expected missing token error"),
|
||||
Err(error) => error,
|
||||
};
|
||||
@@ -1046,7 +995,7 @@ mod tests {
|
||||
#[test]
|
||||
fn gemma4_initialize_open_channel_prompt_starts_in_reasoning() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[100, 3000, 3001]).unwrap();
|
||||
parser.initialize(&[CHANNEL_START_ID, 3000, 3001]).unwrap();
|
||||
|
||||
let output = parser.parse_complete("reason<channel|>answer").unwrap();
|
||||
|
||||
@@ -1057,7 +1006,7 @@ mod tests {
|
||||
#[test]
|
||||
fn gemma4_initialize_turn_prompt_starts_in_text() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[104, 3000, 3001]).unwrap();
|
||||
parser.initialize(&[TURN_BOUNDARY_ID, 3000, 3001]).unwrap();
|
||||
|
||||
let output = parser.parse_complete("<|channel>thought\nreason<channel|>answer").unwrap();
|
||||
|
||||
@@ -1068,7 +1017,7 @@ mod tests {
|
||||
#[test]
|
||||
fn gemma4_initialize_special_token_caps_boundary_scan() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[100, 3000, 104, 3001]).unwrap();
|
||||
parser.initialize(&[CHANNEL_START_ID, 3000, TURN_BOUNDARY_ID, 3001]).unwrap();
|
||||
|
||||
let output = parser.parse_complete("answer").unwrap();
|
||||
|
||||
@@ -1079,7 +1028,7 @@ mod tests {
|
||||
#[test]
|
||||
fn gemma4_initialize_closed_channel_prompt_starts_in_text() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[100, 3000, 3001, 101]).unwrap();
|
||||
parser.initialize(&[CHANNEL_START_ID, 3000, 3001, CHANNEL_END_ID]).unwrap();
|
||||
|
||||
let output = parser.parse_complete("answer").unwrap();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user