[Rust Frontend] Add ordinary-text tokenizer encoding (#49992)
Co-authored-by: OpenAI Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
61ac368021
commit
5ed3faa43d
@@ -177,6 +177,10 @@ mod tests {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
self.encode(text, false)
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
|
||||
Reference in New Issue
Block a user