[Rust Frontend] Simplify unit tests with shared TestTokenizer (#47125)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-30 15:34:43 +01:00
committed by GitHub
parent 1ab9522935
commit 7a327f0b4f
28 changed files with 680 additions and 826 deletions
+4
View File
@@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer {
fn token_to_id(&self, _token: &str) -> Option<u32> {
Some(u32::MAX)
}
fn id_to_token(&self, _id: u32) -> Option<String> {
Some("\u{FFFD}".to_string())
}
}
/// Bench-only adapter that exposes a unified parser through the tool-parser