Make dedup critical crates compilation-compat with wasm

Cargo.toml

@@ -19,7 +19,7 @@ members = [
     "xet_threadpool", 
     "deduplication"]
 
-exclude = ["hf_xet", "chunk_cache_bench"]
+exclude = ["hf_xet", "chunk_cache_bench", "wasm_xet"]
 
 [profile.release]
 opt-level = 3

cas_client/Cargo.toml

@@ -21,7 +21,7 @@ merklehash = { path = "../merklehash" }
 xet_threadpool = { path = "../xet_threadpool" }
 deduplication = {path = "../deduplication" }
 thiserror = "2.0"
-tokio = { version = "1.41", features = ["sync"] }
+tokio = { version = "1.44", features = ["sync"] }
 async-trait = "0.1.9"
 anyhow = "1"
 http = "1.1.0"

cas_object/Cargo.toml

@@ -25,18 +25,20 @@ anyhow = "1.0.88"
 tracing = "0.1.40"
 lz4_flex = "0.11.3"
 bytes = "1.7.2"
-rand = "0.8.5"
+rand = "0.9"
 blake3 = "1.5.4"
 futures = { version = "0.3.31" }
-tokio-util = { version = "0.7.12", features = ["io"] }
-tokio = { version = "1.41.1" }
+tokio = { version = "1.44", features = ["sync", "time", "rt", "macros"] }
 half = "2.4.1"
 serde = { version = "1.0.129", features = ["derive"] }
 countio = { version = "0.2.19", features = ["futures"] }
 clap = { version = "4.1", features = ["derive"] }
 csv = "1.1"
 more-asserts = "0.3.1"
 
+[target.'cfg(not(target_family = "wasm"))'.dependencies]
+tokio-util = { version = "0.7.12", features = ["io"] }
+
 [[bin]]
 path = "src/byte_grouping/compression_stats/collect_compression_stats.rs"
 name = "collect_compression_stats"

cas_object/src/byte_grouping/bg4.rs

@@ -267,10 +267,10 @@ mod tests {
 
     #[test]
     fn test_split_regroup_separate() {
-        let mut rng = rand::thread_rng();
+        let mut rng = rand::rng();
 
         for n in [64 * 1024, 64 * 1024 - 53, 64 * 1024 + 135] {
-            let data: Vec<_> = (0..n).map(|_| rng.gen_range(0..255)).collect();
+            let data: Vec<_> = (0..n).map(|_| rng.random_range(0..255)).collect();
             let groups = bg4_split_separate(&data);
             let regrouped = bg4_regroup_separate(&groups);
 
@@ -280,10 +280,10 @@ mod tests {
 
     #[test]
     fn test_split_regroup_together() {
-        let mut rng = rand::thread_rng();
+        let mut rng = rand::rng();
 
         for n in [64 * 1024, 64 * 1024 - 53, 64 * 1024 + 135] {
-            let data: Vec<_> = (0..n).map(|_| rng.gen_range(0..255)).collect();
+            let data: Vec<_> = (0..n).map(|_| rng.random_range(0..255)).collect();
             let groups = bg4_split_together(&data);
 
             let regrouped = bg4_regroup_together(&groups);

cas_object/src/byte_grouping/compression_stats/collect_compression_stats.rs

@@ -214,9 +214,9 @@ async fn main() -> Result<()> {
         // For each random block
         for idx in 0..nblocks {
             // Choose random block size in [32K..96K], clamp to file size
-            let block_size = rng.gen_range(32_768..=98_304).min(file_size);
+            let block_size = rng.random_range(32_768..=98_304).min(file_size);
 
-            let offset = rng.gen_range(0..=(file_size - block_size));
+            let offset = rng.random_range(0..=(file_size - block_size));
 
             let load_permit = max_limiter.clone().acquire_owned().await.unwrap();
 

cas_object/src/cas_chunk_format.rs

@@ -267,7 +267,7 @@ mod tests {
     }
 
     fn gen_random_bytes(uncompressed_chunk_size: u32) -> Vec<u8> {
-        let mut rng = rand::thread_rng();
+        let mut rng = rand::rng();
         let mut data = vec![0u8; uncompressed_chunk_size as usize];
         rng.fill(&mut data[..]);
         data

cas_object/src/cas_chunk_format/deserialize_async.rs

@@ -120,7 +120,7 @@ where
 mod tests {
     use bytes::Bytes;
     use futures::Stream;
-    use rand::{thread_rng, Rng};
+    use rand::{rng, Rng};
 
     use crate::deserialize_async::deserialize_chunks_to_writer_from_stream;
     use crate::{serialize_chunk, CompressionScheme};
@@ -149,7 +149,7 @@ mod tests {
     ) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Unpin {
         let data = get_chunks(rng, num_chunks, compression_scheme);
         let it = data
-            .chunks(data.len() / (2 + rng.gen::<usize>() % 8))
+            .chunks(data.len() / (2 + rng.random::<u64>() % 8) as usize)
             .map(|chunk| Ok(Bytes::copy_from_slice(chunk)))
             .collect::<Vec<_>>();
         futures::stream::iter(it)
@@ -165,7 +165,7 @@ mod tests {
             (100, CompressionScheme::LZ4),
             (1000, CompressionScheme::LZ4),
         ];
-        let rng = &mut thread_rng();
+        let rng = &mut rng();
         for (num_chunks, compression_scheme) in cases {
             let stream = get_stream(rng, num_chunks, compression_scheme);
             let mut buf = Vec::new();

cas_object/src/cas_object_format.rs

@@ -1311,7 +1311,7 @@ pub mod test_utils {
     use crate::cas_chunk_format::serialize_chunk;
 
     pub fn gen_random_bytes(size: u32) -> Vec<u8> {
-        let mut rng = rand::thread_rng();
+        let mut rng = rand::rng();
         let mut data = vec![0u8; size as usize];
         rng.fill(&mut data[..]);
         data
@@ -1354,8 +1354,8 @@ pub mod test_utils {
         for _idx in 0..num_chunks {
             let chunk_size: u32 = match chunk_size {
                 ChunkSize::Random(a, b) => {
-                    let mut rng = rand::thread_rng();
-                    rng.gen_range(a..=b)
+                    let mut rng = rand::rng();
+                    rng.random_range(a..=b)
                 },
                 ChunkSize::Fixed(size) => size,
             };

cas_object/src/compression_scheme.rs

@@ -278,55 +278,55 @@ mod tests {
 
     #[test]
     fn test_bg4_lz4() {
-        let mut rng = rand::thread_rng();
+        let mut rng = rand::rng();
 
         for i in 0..4 {
             let n = 64 * 1024 + i * 23;
             let all_zeros = vec![0u8; n];
             let all_ones = vec![1u8; n];
             let all_0xff = vec![0xFF; n];
-            let random_u8s: Vec<_> = (0..n).map(|_| rng.gen_range(0..255)).collect();
+            let random_u8s: Vec<_> = (0..n).map(|_| rng.random_range(0..255)).collect();
             let random_f32s_ng1_1: Vec<_> = (0..n / size_of::<f32>())
-                .map(|_| rng.gen_range(-1.0f32..=1.0))
+                .map(|_| rng.random_range(-1.0f32..=1.0))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();
             let random_f32s_0_2: Vec<_> = (0..n / size_of::<f32>())
-                .map(|_| rng.gen_range(0f32..=2.0))
+                .map(|_| rng.random_range(0f32..=2.0))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();
             let random_f64s_ng1_1: Vec<_> = (0..n / size_of::<f64>())
-                .map(|_| rng.gen_range(-1.0f64..=1.0))
+                .map(|_| rng.random_range(-1.0f64..=1.0))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();
             let random_f64s_0_2: Vec<_> = (0..n / size_of::<f64>())
-                .map(|_| rng.gen_range(0f64..=2.0))
+                .map(|_| rng.random_range(0f64..=2.0))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();
 
             // f16, a.k.a binary16 format: sign (1 bit), exponent (5 bit), mantissa (10 bit)
             let random_f16s_ng1_1: Vec<_> = (0..n / size_of::<f16>())
-                .map(|_| f16::from_f32(rng.gen_range(-1.0f32..=1.0)))
+                .map(|_| f16::from_f32(rng.random_range(-1.0f32..=1.0)))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();
             let random_f16s_0_2: Vec<_> = (0..n / size_of::<f16>())
-                .map(|_| f16::from_f32(rng.gen_range(0f32..=2.0)))
+                .map(|_| f16::from_f32(rng.random_range(0f32..=2.0)))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();
 
             // bf16 format: sign (1 bit), exponent (8 bit), mantissa (7 bit)
             let random_bf16s_ng1_1: Vec<_> = (0..n / size_of::<bf16>())
-                .map(|_| bf16::from_f32(rng.gen_range(-1.0f32..=1.0)))
+                .map(|_| bf16::from_f32(rng.random_range(-1.0f32..=1.0)))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();
             let random_bf16s_0_2: Vec<_> = (0..n / size_of::<bf16>())
-                .map(|_| bf16::from_f32(rng.gen_range(0f32..=2.0)))
+                .map(|_| bf16::from_f32(rng.random_range(0f32..=2.0)))
                 .map(|f| f.to_le_bytes())
                 .flatten()
                 .collect();

cas_object/src/validate_xorb_stream.rs

@@ -343,7 +343,7 @@ mod tests {
         buf.resize(serialized_chunks_length as usize, 0);
 
         // Now add some maybe existing but less than 8 bytes
-        let mut rng = rand::thread_rng();
+        let mut rng = rand::rng();
         let some_nonzero_size_less_than_8 = rng.next_u32() % 7 + 1;
         let mut some_bytes = vec![0u8; some_nonzero_size_less_than_8 as usize];
         rng.fill_bytes(&mut some_bytes);

chunk_cache/Cargo.toml

@@ -20,7 +20,7 @@ crc32fast = "1.4.2"
 log = "0.4.22"
 
 [dev-dependencies]
-tokio = { version = "1.36", features = ["full"] }
+tokio = { version = "1.44", features = ["full"] }
 tempdir = "0.3.7"
 
 [[bin]]