diff --git a/Cargo.lock b/Cargo.lock index 0cf5841..08fdf73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,7 +38,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -49,7 +49,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -67,6 +67,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + [[package]] name = "cfg-if" version = "1.0.4" @@ -119,6 +125,19 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -158,6 +177,12 @@ dependencies = [ "md4", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "generic-array" version = "0.14.7" @@ -174,6 +199,19 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -186,13 +224,24 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "l-s" -version = "0.2.1" +version = "0.3.0" dependencies = [ "anyhow", "clap", "ed2k", + "indicatif", "md-5", "serde", "serde_json", @@ -232,12 +281,30 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + [[package]] name = "once_cell_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + [[package]] name = "proc-macro2" version = "1.0.103" @@ -256,6 +323,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.20" @@ -356,6 +429,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "utf8parse" version = "0.2.2" @@ -368,12 +447,76 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -383,6 +526,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index 718c7ad..1430d97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "l-s" -version = "0.2.1" +version = "0.3.0" authors = ["licsber "] edition = "2021" @@ -14,6 +14,7 @@ license = "AGPL-3.0-only" anyhow = "1.0" clap = { version = "4.5", features = ["derive"] } ed2k = "1.0.1" +indicatif = "0.17" md-5 = "0.10" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/meta/file.rs b/src/meta/file.rs index d1f8c25..2b64647 100644 --- a/src/meta/file.rs +++ b/src/meta/file.rs @@ -33,6 +33,14 @@ pub struct FileMeta { impl FileMeta { pub fn from_path(path: &Path) -> Result { + Self::from_path_with_callback(path, |_| {}, || {}) + } + + pub fn from_path_with_callback(path: &Path, mut on_bytes_read: F1, mut on_iop: F2) -> Result + where + F1: FnMut(u64), + F2: FnMut(), + { let info = fs::metadata(path).with_context(|| format!("无法读取文件信息: {}", path.display()))?; if !info.is_file() { @@ -78,6 +86,9 @@ impl FileMeta { head115.feed(chunk); head_baidu.feed(chunk); + + on_bytes_read(read_len as u64); + on_iop(); // 每次 read 调用算一次 IOPS } let head_115 = calc_head_115(head115.as_slice()); @@ -114,6 +125,14 @@ impl FileMeta { } pub fn calc_xxh128(path: &Path) -> Result { + calc_xxh128_with_callback(path, |_| {}, || {}) +} + +pub fn calc_xxh128_with_callback(path: &Path, mut on_bytes_read: F1, mut on_iop: F2) -> Result +where + F1: FnMut(u64), + F2: FnMut(), +{ let mut file = File::open(path).with_context(|| format!("无法打开文件: {}", path.display()))?; let mut buffer = vec![0u8; DEFAULT_BUFFER_SIZE]; let mut hasher = Xxh3::new(); @@ -124,6 +143,8 @@ pub fn calc_xxh128(path: &Path) -> Result { break; } hasher.update(&buffer[..read_len]); + on_bytes_read(read_len as u64); + on_iop(); // 每次 read 调用算一次 IOPS } Ok(hex_upper(hasher.digest128().to_be_bytes())) diff --git a/src/meta/tree.rs b/src/meta/tree.rs index ffe3a4e..68c3f7f 100644 --- a/src/meta/tree.rs +++ b/src/meta/tree.rs @@ -1,13 +1,16 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; use anyhow::{Context, Result}; +use indicatif::{ProgressBar, ProgressStyle}; use serde::{Deserialize, Serialize}; -use super::file::{calc_xxh128, FileMeta}; +use super::file::{calc_xxh128_with_callback, FileMeta}; use crate::constants::META_VERSION; -use crate::utils::{basename, should_skip_dir, should_skip_file}; +use crate::utils::{basename, friendly_size, should_skip_dir, should_skip_file}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DirSnapshot { @@ -20,8 +23,34 @@ pub struct DirSnapshot { impl DirSnapshot { pub fn build_root(path: &Path) -> Result { - let mut node = Self::build_node(path)?; + // 先统计总文件数 + let total_files = count_files(path)?; + + // 创建进度条 + let pb = if total_files > 0 { + let pb = ProgressBar::new(total_files); + pb.set_style( + ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({percent}%) {msg}") + .unwrap() + .progress_chars("#>-"), + ); + pb.set_message("构建中..."); + Some(pb) + } else { + None + }; + + let start = Instant::now(); + let total_bytes_read = Arc::new(Mutex::new(0u64)); + let total_iops = Arc::new(Mutex::new(0u64)); + let mut node = Self::build_node(path, &pb, &start, &total_bytes_read, &total_iops)?; node.v = Some(META_VERSION.to_string()); + + if let Some(pb) = &pb { + pb.finish_with_message("构建完成"); + } + Ok(node) } @@ -29,7 +58,13 @@ impl DirSnapshot { Ok(serde_json::from_reader(reader)?) } - fn build_node(path: &Path) -> Result { + fn build_node( + path: &Path, + pb: &Option, + start: &Instant, + total_bytes_read: &Arc>, + total_iops: &Arc>, + ) -> Result { let dir_name = path .file_name() .map(basename) @@ -57,8 +92,7 @@ impl DirSnapshot { if should_skip_dir(&name) { continue; } - println!("目录: {}", full_path.display()); - dirs.push(Self::build_node(&full_path)?); + dirs.push(Self::build_node(&full_path, pb, start, total_bytes_read, total_iops)?); continue; } @@ -66,9 +100,28 @@ impl DirSnapshot { continue; } - let meta = FileMeta::from_path(&full_path)?; - println!("文件: {} {}", meta.friendly_size, full_path.display()); + let total_bytes_read_clone = total_bytes_read.clone(); + let total_iops_clone = total_iops.clone(); + let meta = FileMeta::from_path_with_callback(&full_path, move |bytes| { + *total_bytes_read_clone.lock().unwrap() += bytes; + }, move || { + *total_iops_clone.lock().unwrap() += 1; + })?; files.push(meta); + + // 更新进度条 + if let Some(pb) = pb { + pb.inc(1); + let elapsed = start.elapsed().as_secs_f64(); + let total_bytes = *total_bytes_read.lock().unwrap(); + let total_ops = *total_iops.lock().unwrap(); + if total_bytes > 0 && elapsed > 0.0 { + let speed_bytes_per_sec = total_bytes as f64 / elapsed; + let speed_str = friendly_size(speed_bytes_per_sec as u64); + let iops = total_ops as f64 / elapsed; + pb.set_message(format!("IO速度: {}/s | IOPS: {:.0}", speed_str, iops)); + } + } } Ok(Self { @@ -98,12 +151,80 @@ impl DirSnapshot { } pub fn scan_dir_xxh128(path: &Path) -> Result> { + // 先统计总文件数 + let total_files = count_files(path)?; + + // 创建进度条 + let pb = if total_files > 0 { + let pb = ProgressBar::new(total_files); + pb.set_style( + ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({percent}%) {msg}") + .unwrap() + .progress_chars("#>-"), + ); + pb.set_message("扫描中..."); + Some(pb) + } else { + None + }; + let mut map = BTreeMap::new(); - walk_dir(path, &mut map)?; + let start = Instant::now(); + let total_bytes_read = Arc::new(Mutex::new(0u64)); + let total_iops = Arc::new(Mutex::new(0u64)); + walk_dir_with_progress(path, &mut map, &pb, &start, &total_bytes_read, &total_iops)?; + + if let Some(pb) = &pb { + pb.finish_with_message("扫描完成"); + } + Ok(map) } -fn walk_dir(path: &Path, map: &mut BTreeMap) -> Result<()> { +fn count_files(path: &Path) -> Result { + let mut count = 0u64; + count_files_recursive(path, &mut count)?; + Ok(count) +} + +fn count_files_recursive(path: &Path, count: &mut u64) -> Result<()> { + let entries = fs::read_dir(path) + .with_context(|| format!("无法遍历目录: {}", path.display()))? + .collect::, _>>() + .with_context(|| format!("读取目录失败: {}", path.display()))?; + + for entry in entries { + let file_name = entry.file_name(); + let name = file_name.to_string_lossy().to_string(); + let full_path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("无法读取类型: {}", full_path.display()))?; + + if file_type.is_dir() { + if should_skip_dir(&name) { + continue; + } + count_files_recursive(&full_path, count)?; + } else { + if !should_skip_file(&name) { + *count += 1; + } + } + } + + Ok(()) +} + +fn walk_dir_with_progress( + path: &Path, + map: &mut BTreeMap, + pb: &Option, + start: &Instant, + total_bytes_read: &Arc>, + total_iops: &Arc>, +) -> Result<()> { let mut entries = fs::read_dir(path) .with_context(|| format!("无法遍历目录: {}", path.display()))? .collect::, _>>() @@ -122,7 +243,7 @@ fn walk_dir(path: &Path, map: &mut BTreeMap) -> Result<()> { if should_skip_dir(&name) { continue; } - walk_dir(&full_path, map)?; + walk_dir_with_progress(&full_path, map, pb, start, total_bytes_read, total_iops)?; continue; } @@ -130,8 +251,28 @@ fn walk_dir(path: &Path, map: &mut BTreeMap) -> Result<()> { continue; } - let hash = calc_xxh128(&full_path)?; + let total_bytes_read_clone = total_bytes_read.clone(); + let total_iops_clone = total_iops.clone(); + let hash = calc_xxh128_with_callback(&full_path, move |bytes| { + *total_bytes_read_clone.lock().unwrap() += bytes; + }, move || { + *total_iops_clone.lock().unwrap() += 1; + })?; map.insert(full_path, hash); + + // 更新进度条 + if let Some(pb) = pb { + pb.inc(1); + let elapsed = start.elapsed().as_secs_f64(); + let total_bytes = *total_bytes_read.lock().unwrap(); + let total_ops = *total_iops.lock().unwrap(); + if total_bytes > 0 && elapsed > 0.0 { + let speed_bytes_per_sec = total_bytes as f64 / elapsed; + let speed_str = friendly_size(speed_bytes_per_sec as u64); + let iops = total_ops as f64 / elapsed; + pb.set_message(format!("IO速度: {}/s | IOPS: {:.0}", speed_str, iops)); + } + } } Ok(())