[Rust Frontend] Integrate MM audio support (#48554)

Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Bugen Zhao
2026-07-15 15:00:17 +08:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent adce068118
commit e281ac663a
18 changed files with 1275 additions and 258 deletions
+403 -11
View File
@@ -182,7 +182,7 @@ dependencies = [
"hex",
"hmac",
"rand 0.9.2",
"reqwest",
"reqwest 0.12.28",
"reqwest-eventsource",
"secrecy",
"serde",
@@ -379,6 +379,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64ct"
version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bit-set"
version = "0.5.3"
@@ -665,6 +671,35 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
[[package]]
name = "cookie"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
dependencies = [
"percent-encoding",
"time",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
dependencies = [
"cookie",
"document-features",
"idna",
"indexmap 2.13.0",
"log",
"serde",
"serde_derive",
"serde_json",
"time",
"url",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -888,6 +923,16 @@ dependencies = [
"serde",
]
[[package]]
name = "der"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
dependencies = [
"pem-rfc7468",
"zeroize",
]
[[package]]
name = "deranged"
version = "0.5.8"
@@ -1171,6 +1216,12 @@ dependencies = [
"once_cell",
]
[[package]]
name = "extended"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fancy-regex"
version = "0.13.0"
@@ -1589,13 +1640,15 @@ dependencies = [
"indicatif",
"libc",
"log",
"native-tls",
"num_cpus",
"rand 0.9.2",
"reqwest",
"reqwest 0.12.28",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"ureq",
"windows-sys 0.61.2",
]
@@ -2167,22 +2220,27 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "llm-multimodal"
version = "1.7.1"
source = "git+https://github.com/smg-project/llm-multimodal?rev=c8a29dcc755139fdc26185f400ea48c6d6d48273#c8a29dcc755139fdc26185f400ea48c6d6d48273"
source = "git+https://github.com/smg-project/llm-multimodal?rev=7df38e53f99aefaebe86934f010aa8084ec99b2f#7df38e53f99aefaebe86934f010aa8084ec99b2f"
dependencies = [
"anyhow",
"base64 0.22.1",
"blake3",
"bytes",
"cc",
"fast_image_resize",
"hf-hub",
"image",
"libloading",
"ndarray 0.17.2",
"once_cell",
"reqwest",
"pkg-config",
"rayon",
"reqwest 0.13.4",
"rustfft",
"serde",
"serde_json",
"serde_with",
"symphonia",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -2740,6 +2798,15 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "pem-rfc7468"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
dependencies = [
"base64ct",
]
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -3245,9 +3312,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rayon"
version = "1.11.0"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
@@ -3346,6 +3413,12 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "regex-lite"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
[[package]]
name = "regex-syntax"
version = "0.8.10"
@@ -3392,7 +3465,43 @@ dependencies = [
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"wasm-streams 0.4.2",
"web-sys",
]
[[package]]
name = "reqwest"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-core",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-tls",
"hyper-util",
"js-sys",
"log",
"native-tls",
"percent-encoding",
"pin-project-lite",
"rustls-pki-types",
"sync_wrapper",
"tokio",
"tokio-native-tls",
"tokio-util",
"tower",
"tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams 0.5.0",
"web-sys",
]
@@ -3408,7 +3517,7 @@ dependencies = [
"mime",
"nom",
"pin-project-lite",
"reqwest",
"reqwest 0.12.28",
"thiserror 1.0.69",
]
@@ -3526,7 +3635,9 @@ version = "0.23.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
dependencies = [
"log",
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
@@ -4052,6 +4163,17 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "socks"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
dependencies = [
"byteorder",
"libc",
"winapi",
]
[[package]]
name = "spm_precompiled"
version = "0.1.4"
@@ -4127,6 +4249,203 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symphonia"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-adpcm",
"symphonia-codec-alac",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-caf",
"symphonia-format-isomp4",
"symphonia-format-mkv",
"symphonia-format-ogg",
"symphonia-format-riff",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76"
dependencies = [
"lazy_static",
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-adpcm"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebbdfd76d6cc5a601c6292a44357c5b7c82f2cd7cdc0f171421f5c5cff0ea1f"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-alac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-common"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-core"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1"
dependencies = [
"bitflags",
"bytemuck",
"lazy_static",
"log",
"num-complex",
"smallvec",
]
[[package]]
name = "symphonia-format-caf"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde3ca76633d3400ab57195456c09f8a58d775ff5452329f3f212b6efc8622f5"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-format-mkv"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74"
dependencies = [
"lazy_static",
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-format-riff"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0"
dependencies = [
"extended",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681"
dependencies = [
"lazy_static",
"log",
"regex-lite",
"smallvec",
"symphonia-core",
]
[[package]]
name = "syn"
version = "1.0.109"
@@ -4982,6 +5301,42 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
dependencies = [
"base64 0.22.1",
"cookie_store",
"der",
"flate2",
"log",
"native-tls",
"percent-encoding",
"rustls",
"rustls-pki-types",
"serde",
"serde_json",
"socks",
"ureq-proto",
"utf8-zero",
"webpki-root-certs",
"webpki-roots",
]
[[package]]
name = "ureq-proto"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
dependencies = [
"base64 0.22.1",
"http",
"httparse",
"log",
]
[[package]]
name = "url"
version = "2.5.8"
@@ -5006,6 +5361,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]]
name = "utf8-zero"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -5097,7 +5458,7 @@ dependencies = [
"ndarray 0.17.2",
"oss-harmony",
"paste",
"reqwest",
"reqwest 0.13.4",
"rmp-serde",
"serde",
"serde-json-fmt",
@@ -5344,7 +5705,7 @@ dependencies = [
"futures",
"hf-hub",
"itertools 0.14.0",
"reqwest",
"reqwest 0.12.28",
"serde",
"serde_json",
"serde_with",
@@ -5368,7 +5729,7 @@ dependencies = [
"criterion",
"fastokens",
"hf-hub",
"reqwest",
"reqwest 0.12.28",
"riptoken",
"rustc-hash 1.1.0",
"serde",
@@ -5531,6 +5892,19 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wasm-streams"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
@@ -5563,6 +5937,24 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki-root-certs"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "weezl"
version = "0.1.12"