[Rust Frontend] Add ordinary-text tokenizer encoding (#49992)

Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-28 15:46:51 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 61ac368021
commit 5ed3faa43d
10 changed files with 448 additions and 2 deletions
+4
View File
@@ -177,6 +177,10 @@ mod tests {
Ok(vec![])
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
_token_ids: &[u32],