[Rust Frontend] Add ordinary-text tokenizer encoding (#49992)
Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
61ac368021
commit
5ed3faa43d
@@ -19,6 +19,10 @@ impl Tokenizer for BenchTokenizer {
|
||||
Ok(text.chars().map(|_| u32::MAX).collect())
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
|
||||
@@ -414,6 +414,10 @@ mod tests {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
@@ -733,6 +737,10 @@ mod tests {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
|
||||
Reference in New Issue
Block a user