[Rust][Benchmark] Load HF datasets from parquet shards via hf-hub, fixing truncated-cache sampling (#51570)

This commit is contained in:
Roy Wang
2026-08-21 05:07:43 +00:00
committed by GitHub
parent ba07e4a48f
commit df344bf36f
8 changed files with 732 additions and 183 deletions
+133
View File
@@ -15,6 +15,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"const-random",
"getrandom 0.3.4",
"once_cell",
"serde",
@@ -31,6 +32,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "android_system_properties"
version = "0.1.5"
@@ -379,6 +395,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "base64ct"
version = "1.8.3"
@@ -450,6 +472,27 @@ dependencies = [
"generic-array",
]
[[package]]
name = "brotli"
version = "8.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "5.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "bstr"
version = "1.12.1"
@@ -671,6 +714,26 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "const-random"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
dependencies = [
"const-random-macro",
]
[[package]]
name = "const-random-macro"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
dependencies = [
"getrandom 0.2.17",
"once_cell",
"tiny-keccak",
]
[[package]]
name = "constant_time_eq"
version = "0.4.2"
@@ -1584,6 +1647,7 @@ dependencies = [
"bytemuck",
"cfg-if",
"crunchy",
"num-traits",
"zerocopy",
]
@@ -1617,6 +1681,12 @@ version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "heck"
version = "0.5.0"
@@ -2270,6 +2340,15 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lz4_flex"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
dependencies = [
"twox-hash",
]
[[package]]
name = "logos"
version = "0.15.1"
@@ -2648,6 +2727,16 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "num-bigint"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.4.6"
@@ -2866,6 +2955,31 @@ dependencies = [
"windows-link",
]
[[package]]
name = "parquet"
version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c"
dependencies = [
"ahash",
"base64 0.23.1",
"brotli",
"bytes",
"chrono",
"flate2",
"half",
"hashbrown 0.17.1",
"lz4_flex",
"num-bigint",
"num-integer",
"num-traits",
"seq-macro",
"serde_json",
"snap",
"twox-hash",
"zstd",
]
[[package]]
name = "paste"
version = "1.0.15"
@@ -4044,6 +4158,12 @@ version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "seq-macro"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "serde"
version = "1.0.228"
@@ -4316,6 +4436,12 @@ dependencies = [
"version_check",
]
[[package]]
name = "snap"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
[[package]]
name = "socket2"
version = "0.6.3"
@@ -5341,6 +5467,12 @@ dependencies = [
"utf-8",
]
[[package]]
name = "twox-hash"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9"
[[package]]
name = "typenum"
version = "1.19.0"
@@ -5636,6 +5768,7 @@ dependencies = [
"image",
"indicatif",
"mimalloc",
"parquet",
"rand 0.9.2",
"rand_distr",
"rayon",