refactor: code readable.
This commit is contained in:
2
Cargo.lock
generated
2
Cargo.lock
generated
@@ -236,7 +236,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "l-s"
|
name = "l-s"
|
||||||
version = "0.3.0"
|
version = "0.3.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"clap",
|
"clap",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "l-s"
|
name = "l-s"
|
||||||
version = "0.3.0"
|
version = "0.3.1"
|
||||||
authors = ["licsber <admin@licsber.site>"]
|
authors = ["licsber <admin@licsber.site>"]
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
mod file;
|
mod file;
|
||||||
|
mod progress;
|
||||||
mod tree;
|
mod tree;
|
||||||
|
|
||||||
pub use file::{calc_xxh128, FileMeta};
|
pub use file::{calc_xxh128, FileMeta};
|
||||||
|
|||||||
87
src/meta/progress.rs
Normal file
87
src/meta/progress.rs
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use indicatif::{ProgressBar, ProgressStyle};
|
||||||
|
|
||||||
|
use crate::utils::friendly_size;
|
||||||
|
|
||||||
|
/// 进度跟踪器,封装进度条和 IO 统计信息
|
||||||
|
pub struct ProgressTracker {
|
||||||
|
progress_bar: Option<ProgressBar>,
|
||||||
|
bytes_read: Arc<Mutex<u64>>,
|
||||||
|
iops: Arc<Mutex<u64>>,
|
||||||
|
start_time: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProgressTracker {
|
||||||
|
/// 创建新的进度跟踪器
|
||||||
|
pub fn new(total_files: u64, message: &str) -> Self {
|
||||||
|
let progress_bar = if total_files > 0 {
|
||||||
|
let pb = ProgressBar::new(total_files);
|
||||||
|
pb.set_style(
|
||||||
|
ProgressStyle::default_bar()
|
||||||
|
.template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({percent}%) {msg}")
|
||||||
|
.unwrap()
|
||||||
|
.progress_chars("#>-"),
|
||||||
|
);
|
||||||
|
pb.set_message(message.to_string());
|
||||||
|
Some(pb)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
Self {
|
||||||
|
progress_bar,
|
||||||
|
bytes_read: Arc::new(Mutex::new(0u64)),
|
||||||
|
iops: Arc::new(Mutex::new(0u64)),
|
||||||
|
start_time: Instant::now(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 完成一个文件的处理
|
||||||
|
pub fn finish_file(&self) {
|
||||||
|
if let Some(pb) = &self.progress_bar {
|
||||||
|
pb.inc(1);
|
||||||
|
self.update_message();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 更新进度条消息
|
||||||
|
fn update_message(&self) {
|
||||||
|
if let Some(pb) = &self.progress_bar {
|
||||||
|
let elapsed = self.start_time.elapsed().as_secs_f64();
|
||||||
|
let total_bytes = *self.bytes_read.lock().unwrap();
|
||||||
|
let total_ops = *self.iops.lock().unwrap();
|
||||||
|
|
||||||
|
if total_bytes > 0 && elapsed > 0.0 {
|
||||||
|
let speed_bytes_per_sec = total_bytes as f64 / elapsed;
|
||||||
|
let speed_str = friendly_size(speed_bytes_per_sec as u64);
|
||||||
|
let iops = total_ops as f64 / elapsed;
|
||||||
|
pb.set_message(format!("IO速度: {}/s | IOPS: {:.0}", speed_str, iops));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 完成所有处理
|
||||||
|
pub fn finish(&self, message: &str) {
|
||||||
|
if let Some(pb) = &self.progress_bar {
|
||||||
|
pb.finish_with_message(message.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取字节读取回调(可 move)
|
||||||
|
pub fn bytes_callback(&self) -> impl FnMut(u64) {
|
||||||
|
let bytes_read = self.bytes_read.clone();
|
||||||
|
move |bytes| {
|
||||||
|
*bytes_read.lock().unwrap() += bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取 IO 操作回调(可 move)
|
||||||
|
pub fn iop_callback(&self) -> impl FnMut() {
|
||||||
|
let iops = self.iops.clone();
|
||||||
|
move || {
|
||||||
|
*iops.lock().unwrap() += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
127
src/meta/tree.rs
127
src/meta/tree.rs
@@ -1,16 +1,14 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
use std::time::Instant;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use indicatif::{ProgressBar, ProgressStyle};
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use super::file::{calc_xxh128_with_callback, FileMeta};
|
use super::file::{calc_xxh128_with_callback, FileMeta};
|
||||||
|
use super::progress::ProgressTracker;
|
||||||
use crate::constants::META_VERSION;
|
use crate::constants::META_VERSION;
|
||||||
use crate::utils::{basename, friendly_size, should_skip_dir, should_skip_file};
|
use crate::utils::{basename, should_skip_dir, should_skip_file};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct DirSnapshot {
|
pub struct DirSnapshot {
|
||||||
@@ -23,34 +21,13 @@ pub struct DirSnapshot {
|
|||||||
|
|
||||||
impl DirSnapshot {
|
impl DirSnapshot {
|
||||||
pub fn build_root(path: &Path) -> Result<Self> {
|
pub fn build_root(path: &Path) -> Result<Self> {
|
||||||
// 先统计总文件数
|
|
||||||
let total_files = count_files(path)?;
|
let total_files = count_files(path)?;
|
||||||
|
let tracker = ProgressTracker::new(total_files, "构建中...");
|
||||||
|
|
||||||
// 创建进度条
|
let mut node = Self::build_node(path, &tracker)?;
|
||||||
let pb = if total_files > 0 {
|
|
||||||
let pb = ProgressBar::new(total_files);
|
|
||||||
pb.set_style(
|
|
||||||
ProgressStyle::default_bar()
|
|
||||||
.template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({percent}%) {msg}")
|
|
||||||
.unwrap()
|
|
||||||
.progress_chars("#>-"),
|
|
||||||
);
|
|
||||||
pb.set_message("构建中...");
|
|
||||||
Some(pb)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let start = Instant::now();
|
|
||||||
let total_bytes_read = Arc::new(Mutex::new(0u64));
|
|
||||||
let total_iops = Arc::new(Mutex::new(0u64));
|
|
||||||
let mut node = Self::build_node(path, &pb, &start, &total_bytes_read, &total_iops)?;
|
|
||||||
node.v = Some(META_VERSION.to_string());
|
node.v = Some(META_VERSION.to_string());
|
||||||
|
|
||||||
if let Some(pb) = &pb {
|
tracker.finish("构建完成");
|
||||||
pb.finish_with_message("构建完成");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(node)
|
Ok(node)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,13 +35,7 @@ impl DirSnapshot {
|
|||||||
Ok(serde_json::from_reader(reader)?)
|
Ok(serde_json::from_reader(reader)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_node(
|
fn build_node(path: &Path, tracker: &ProgressTracker) -> Result<Self> {
|
||||||
path: &Path,
|
|
||||||
pb: &Option<ProgressBar>,
|
|
||||||
start: &Instant,
|
|
||||||
total_bytes_read: &Arc<Mutex<u64>>,
|
|
||||||
total_iops: &Arc<Mutex<u64>>,
|
|
||||||
) -> Result<Self> {
|
|
||||||
let dir_name = path
|
let dir_name = path
|
||||||
.file_name()
|
.file_name()
|
||||||
.map(basename)
|
.map(basename)
|
||||||
@@ -92,7 +63,7 @@ impl DirSnapshot {
|
|||||||
if should_skip_dir(&name) {
|
if should_skip_dir(&name) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
dirs.push(Self::build_node(&full_path, pb, start, total_bytes_read, total_iops)?);
|
dirs.push(Self::build_node(&full_path, tracker)?);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,28 +71,11 @@ impl DirSnapshot {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let total_bytes_read_clone = total_bytes_read.clone();
|
let on_bytes = tracker.bytes_callback();
|
||||||
let total_iops_clone = total_iops.clone();
|
let on_iop = tracker.iop_callback();
|
||||||
let meta = FileMeta::from_path_with_callback(&full_path, move |bytes| {
|
let meta = FileMeta::from_path_with_callback(&full_path, on_bytes, on_iop)?;
|
||||||
*total_bytes_read_clone.lock().unwrap() += bytes;
|
|
||||||
}, move || {
|
|
||||||
*total_iops_clone.lock().unwrap() += 1;
|
|
||||||
})?;
|
|
||||||
files.push(meta);
|
files.push(meta);
|
||||||
|
tracker.finish_file();
|
||||||
// 更新进度条
|
|
||||||
if let Some(pb) = pb {
|
|
||||||
pb.inc(1);
|
|
||||||
let elapsed = start.elapsed().as_secs_f64();
|
|
||||||
let total_bytes = *total_bytes_read.lock().unwrap();
|
|
||||||
let total_ops = *total_iops.lock().unwrap();
|
|
||||||
if total_bytes > 0 && elapsed > 0.0 {
|
|
||||||
let speed_bytes_per_sec = total_bytes as f64 / elapsed;
|
|
||||||
let speed_str = friendly_size(speed_bytes_per_sec as u64);
|
|
||||||
let iops = total_ops as f64 / elapsed;
|
|
||||||
pb.set_message(format!("IO速度: {}/s | IOPS: {:.0}", speed_str, iops));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
@@ -151,34 +105,13 @@ impl DirSnapshot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn scan_dir_xxh128(path: &Path) -> Result<BTreeMap<PathBuf, String>> {
|
pub fn scan_dir_xxh128(path: &Path) -> Result<BTreeMap<PathBuf, String>> {
|
||||||
// 先统计总文件数
|
|
||||||
let total_files = count_files(path)?;
|
let total_files = count_files(path)?;
|
||||||
|
let tracker = ProgressTracker::new(total_files, "扫描中...");
|
||||||
// 创建进度条
|
|
||||||
let pb = if total_files > 0 {
|
|
||||||
let pb = ProgressBar::new(total_files);
|
|
||||||
pb.set_style(
|
|
||||||
ProgressStyle::default_bar()
|
|
||||||
.template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({percent}%) {msg}")
|
|
||||||
.unwrap()
|
|
||||||
.progress_chars("#>-"),
|
|
||||||
);
|
|
||||||
pb.set_message("扫描中...");
|
|
||||||
Some(pb)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut map = BTreeMap::new();
|
let mut map = BTreeMap::new();
|
||||||
let start = Instant::now();
|
walk_dir_with_progress(path, &mut map, &tracker)?;
|
||||||
let total_bytes_read = Arc::new(Mutex::new(0u64));
|
|
||||||
let total_iops = Arc::new(Mutex::new(0u64));
|
|
||||||
walk_dir_with_progress(path, &mut map, &pb, &start, &total_bytes_read, &total_iops)?;
|
|
||||||
|
|
||||||
if let Some(pb) = &pb {
|
|
||||||
pb.finish_with_message("扫描完成");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
tracker.finish("扫描完成");
|
||||||
Ok(map)
|
Ok(map)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,10 +153,7 @@ fn count_files_recursive(path: &Path, count: &mut u64) -> Result<()> {
|
|||||||
fn walk_dir_with_progress(
|
fn walk_dir_with_progress(
|
||||||
path: &Path,
|
path: &Path,
|
||||||
map: &mut BTreeMap<PathBuf, String>,
|
map: &mut BTreeMap<PathBuf, String>,
|
||||||
pb: &Option<ProgressBar>,
|
tracker: &ProgressTracker,
|
||||||
start: &Instant,
|
|
||||||
total_bytes_read: &Arc<Mutex<u64>>,
|
|
||||||
total_iops: &Arc<Mutex<u64>>,
|
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let mut entries = fs::read_dir(path)
|
let mut entries = fs::read_dir(path)
|
||||||
.with_context(|| format!("无法遍历目录: {}", path.display()))?
|
.with_context(|| format!("无法遍历目录: {}", path.display()))?
|
||||||
@@ -243,7 +173,7 @@ fn walk_dir_with_progress(
|
|||||||
if should_skip_dir(&name) {
|
if should_skip_dir(&name) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
walk_dir_with_progress(&full_path, map, pb, start, total_bytes_read, total_iops)?;
|
walk_dir_with_progress(&full_path, map, tracker)?;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,28 +181,11 @@ fn walk_dir_with_progress(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let total_bytes_read_clone = total_bytes_read.clone();
|
let on_bytes = tracker.bytes_callback();
|
||||||
let total_iops_clone = total_iops.clone();
|
let on_iop = tracker.iop_callback();
|
||||||
let hash = calc_xxh128_with_callback(&full_path, move |bytes| {
|
let hash = calc_xxh128_with_callback(&full_path, on_bytes, on_iop)?;
|
||||||
*total_bytes_read_clone.lock().unwrap() += bytes;
|
|
||||||
}, move || {
|
|
||||||
*total_iops_clone.lock().unwrap() += 1;
|
|
||||||
})?;
|
|
||||||
map.insert(full_path, hash);
|
map.insert(full_path, hash);
|
||||||
|
tracker.finish_file();
|
||||||
// 更新进度条
|
|
||||||
if let Some(pb) = pb {
|
|
||||||
pb.inc(1);
|
|
||||||
let elapsed = start.elapsed().as_secs_f64();
|
|
||||||
let total_bytes = *total_bytes_read.lock().unwrap();
|
|
||||||
let total_ops = *total_iops.lock().unwrap();
|
|
||||||
if total_bytes > 0 && elapsed > 0.0 {
|
|
||||||
let speed_bytes_per_sec = total_bytes as f64 / elapsed;
|
|
||||||
let speed_str = friendly_size(speed_bytes_per_sec as u64);
|
|
||||||
let iops = total_ops as f64 / elapsed;
|
|
||||||
pb.set_message(format!("IO速度: {}/s | IOPS: {:.0}", speed_str, iops));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
Reference in New Issue
Block a user