[Rust Frontend] Add ordinary-text tokenizer encoding (#49992)

Co-authored-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-28 15:46:51 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 61ac368021
commit 5ed3faa43d
10 changed files with 448 additions and 2 deletions
+4
View File
@@ -19,6 +19,10 @@ impl Tokenizer for BenchTokenizer {
Ok(text.chars().map(|_| u32::MAX).collect())
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
token_ids: &[u32],
+8
View File
@@ -414,6 +414,10 @@ mod tests {
Ok(text.chars().map(u32::from).collect())
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
token_ids: &[u32],
@@ -733,6 +737,10 @@ mod tests {
Ok(vec![])
}
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
self.encode(text, false)
}
fn decode(
&self,
_token_ids: &[u32],