[Rust Frontend] Migrate to new tekken crate (#53056)

Signed-off-by: Jorge Menjivar <jorge@menjivar.ai>
This commit is contained in:
Jorge Menjivar
2026-09-01 02:35:21 +00:00
committed by GitHub
parent 45aed9b0cd
commit 225aec4809
2 changed files with 10 additions and 62 deletions
+9 -61
View File
@@ -1737,12 +1737,6 @@ dependencies = [
"digest",
]
[[package]]
name = "hound"
version = "3.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
[[package]]
name = "http"
version = "1.4.0"
@@ -2307,7 +2301,7 @@ dependencies = [
"hf-hub",
"image",
"libloading",
"ndarray 0.17.2",
"ndarray",
"once_cell",
"pkg-config",
"rayon",
@@ -2677,21 +2671,6 @@ dependencies = [
"tempfile",
]
[[package]]
name = "ndarray"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
dependencies = [
"matrixmultiply",
"num-complex",
"num-integer",
"num-traits",
"portable-atomic",
"portable-atomic-util",
"rawpointer",
]
[[package]]
name = "ndarray"
version = "0.17.2"
@@ -3846,18 +3825,6 @@ dependencies = [
"serde_bytes",
]
[[package]]
name = "rubato"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5258099699851cfd0082aeb645feb9c084d9a5e1f1b8d5372086b989fc5e56a1"
dependencies = [
"num-complex",
"num-integer",
"num-traits",
"realfft",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
@@ -4814,25 +4781,21 @@ dependencies = [
]
[[package]]
name = "tekken-rs"
version = "0.1.1"
name = "tekken"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49623843103837f53f7ebe8cfafc19ccff28ff0e15e7c4b9f6ad21e36fbfde3a"
checksum = "1d1481e6bbb98c7e1b04124420b6e7c367c87fc897aae340edcdeb4e91b639d2"
dependencies = [
"anyhow",
"base64 0.22.1",
"env_logger",
"hound",
"log",
"ndarray 0.16.1",
"regex",
"rubato",
"rustc-hash 1.1.0",
"rustfft",
"serde",
"serde_json",
"thiserror 2.0.18",
"tiktoken-rs 0.7.0",
"tiktoken-rs",
]
[[package]]
@@ -4933,21 +4896,6 @@ dependencies = [
"zune-jpeg",
]
[[package]]
name = "tiktoken-rs"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25563eeba904d770acf527e8b370fe9a5547bacd20ff84a0b6c3bc41288e5625"
dependencies = [
"anyhow",
"base64 0.22.1",
"bstr",
"fancy-regex 0.13.0",
"lazy_static",
"regex",
"rustc-hash 1.1.0",
]
[[package]]
name = "tiktoken-rs"
version = "0.9.1"
@@ -5779,7 +5727,7 @@ dependencies = [
"serde_json",
"thiserror 2.0.18",
"thiserror-ext",
"tiktoken-rs 0.9.1",
"tiktoken-rs",
"tokenizers",
"tokio",
"tokio-stream",
@@ -5810,7 +5758,7 @@ dependencies = [
"llm-multimodal",
"minijinja",
"minijinja-contrib",
"ndarray 0.17.2",
"ndarray",
"oss-harmony",
"paste",
"reqwest 0.13.4",
@@ -6095,11 +6043,11 @@ dependencies = [
"rustc-hash 1.1.0",
"serde",
"serde_json",
"tekken-rs",
"tekken",
"tempfile",
"thiserror 2.0.18",
"thiserror-ext",
"tiktoken-rs 0.9.1",
"tiktoken-rs",
"tokenizers",
"tokio",
"tracing",
+1 -1
View File
@@ -104,7 +104,7 @@ strum = { version = "0.27.2", features = ["derive"] }
subenum = "1.1.3"
subtle = "2.6"
task-local = "0.1.1"
tekken = { package = "tekken-rs", version = "0.1.1", default-features = false }
tekken = { version = "0.2.0", default-features = false }
tempfile = "3.23.0"
thiserror = "2.0.16"
thiserror-ext = "0.3.0"