[Rust Frontend] Add ordinary-text tokenizer encoding (#49992)

Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-28 15:46:51 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 61ac368021
commit 5ed3faa43d
10 changed files with 448 additions and 2 deletions
@@ -31,6 +31,10 @@ impl Tokenizer for FixtureTokenizer {
Ok(text.bytes().map(u32::from).collect())
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
token_ids: &[u32],