feat(security): [#1981] l-s v0.5.2 Unix 符号链接防护及安全文件操作

This commit is contained in:
2026-05-10 02:08:33 +08:00
parent 30186c23fb
commit e3afbbe3be
7 changed files with 697 additions and 84 deletions
Generated
+1
View File
@@ -267,6 +267,7 @@ dependencies = [
"clap", "clap",
"ed2k", "ed2k",
"indicatif", "indicatif",
"libc",
"md-5", "md-5",
"serde", "serde",
"serde_json", "serde_json",
+2 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "l-s" name = "l-s"
version = "0.5.1" version = "0.5.2"
authors = ["licsber <admin@licsber.site>"] authors = ["licsber <admin@licsber.site>"]
edition = "2021" edition = "2021"
@@ -15,6 +15,7 @@ anyhow = "1.0"
clap = { version = "4.5", features = ["derive"] } clap = { version = "4.5", features = ["derive"] }
ed2k = "1.0.1" ed2k = "1.0.1"
indicatif = "0.18" indicatif = "0.18"
libc = "0.2"
md-5 = "0.10" md-5 = "0.10"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
+16 -3
View File
@@ -1,7 +1,7 @@
use std::path::PathBuf; use std::path::PathBuf;
use std::{env, fs}; use std::{env, fs};
use anyhow::{Context, Result}; use anyhow::{anyhow, Context, Result};
use clap::Parser; use clap::Parser;
/// 负责解析命令行参数 /// 负责解析命令行参数
@@ -20,7 +20,20 @@ impl Cli {
None => env::current_dir().context("无法获取当前工作目录")?, None => env::current_dir().context("无法获取当前工作目录")?,
}; };
fs::canonicalize(&candidate) let path = if candidate.is_absolute() {
.with_context(|| format!("无法解析路径: {}", candidate.display())) candidate
} else {
env::current_dir()
.context("无法获取当前工作目录")?
.join(candidate)
};
let info = fs::symlink_metadata(&path)
.with_context(|| format!("无法读取路径信息: {}", path.display()))?;
if info.file_type().is_symlink() {
return Err(anyhow!("不支持扫描符号链接: {}", path.display()));
}
Ok(path)
} }
} }
+78 -17
View File
@@ -5,13 +5,16 @@ mod meta;
mod utils; mod utils;
use std::fs::{self, File, OpenOptions}; use std::fs::{self, File, OpenOptions};
use std::io::Write; use std::io::{self, Write};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::time::{Instant, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result}; use anyhow::{anyhow, Context, Result};
use clap::Parser; use clap::Parser;
use meta::{calc_xxh128_with_callback, scan_dir_xxh128, DirSnapshot, FileMeta, ProgressTracker}; use meta::{
calc_xxh128_from_file_with_callback, open_regular_file_nofollow, scan_dir_xxh128, DirSnapshot,
FileMeta, ProgressTracker,
};
fn main() -> Result<()> { fn main() -> Result<()> {
let started = Instant::now(); let started = Instant::now();
@@ -19,10 +22,14 @@ fn main() -> Result<()> {
let target = cli.resolve_path()?; let target = cli.resolve_path()?;
println!("目标: {}", target.display()); println!("目标: {}", target.display());
if target.is_dir() { let target_info = fs::symlink_metadata(&target)
.with_context(|| format!("无法读取路径信息: {}", target.display()))?;
if target_info.is_dir() {
process_dir(&target)?; process_dir(&target)?;
} else { } else if target_info.is_file() {
process_file(&target)?; process_file(&target)?;
} else {
return Err(anyhow!("{} 不是文件或目录", target.display()));
} }
println!("耗时: {:?}", started.elapsed()); println!("耗时: {:?}", started.elapsed());
@@ -35,8 +42,19 @@ fn process_file(path: &Path) -> Result<()> {
.map(Path::to_path_buf) .map(Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from(".")) .unwrap_or_else(|| PathBuf::from("."))
.join("meta"); .join("meta");
match symlink_metadata_optional(&meta_dir)? {
Some(info) if info.file_type().is_symlink() => {
return Err(anyhow!("不支持符号链接目录: {}", meta_dir.display()));
}
Some(info) if !info.is_dir() => {
return Err(anyhow!("{} 不是目录", meta_dir.display()));
}
Some(_) => {}
None => {
fs::create_dir_all(&meta_dir) fs::create_dir_all(&meta_dir)
.with_context(|| format!("无法创建目录: {}", meta_dir.display()))?; .with_context(|| format!("无法创建目录: {}", meta_dir.display()))?;
}
}
let basename = path let basename = path
.file_name() .file_name()
@@ -44,32 +62,45 @@ fn process_file(path: &Path) -> Result<()> {
.unwrap_or_else(|| "unknown".to_string()); .unwrap_or_else(|| "unknown".to_string());
let save_path = meta_dir.join(format!("{basename}.json")); let save_path = meta_dir.join(format!("{basename}.json"));
// 获取文件大小 let existing_save = symlink_metadata_optional(&save_path)?;
let file_size = fs::metadata(path) if existing_save.is_none() {
let file = open_regular_file_nofollow(path)?;
let file_size = file
.metadata()
.with_context(|| format!("无法读取文件信息: {}", path.display()))? .with_context(|| format!("无法读取文件信息: {}", path.display()))?
.len(); .len();
if !save_path.exists() {
let tracker = ProgressTracker::new_single_file(file_size, &basename); let tracker = ProgressTracker::new_single_file(file_size, &basename);
let on_bytes = tracker.bytes_callback(); let on_bytes = tracker.bytes_callback();
let on_iop = tracker.iop_callback(); let on_iop = tracker.iop_callback();
let meta = FileMeta::from_path_with_callback(path, on_bytes, on_iop)?; let meta = FileMeta::from_open_file_with_callback(path, file, on_bytes, on_iop)?;
tracker.finish("处理完成"); tracker.finish("处理完成");
let json = meta.to_pretty_json()?; let json = meta.to_pretty_json()?;
println!("{}", json); println!("{}", json);
write_atomic(&save_path, &json)?; write_atomic(&save_path, &json)?;
return Ok(()); return Ok(());
} }
let save_info = existing_save.expect("checked as Some");
if save_info.file_type().is_symlink() {
return Err(anyhow!("不支持符号链接元数据文件: {}", save_path.display()));
}
if !save_info.is_file() {
return Err(anyhow!("{} 不是文件", save_path.display()));
}
let existing = File::open(&save_path) let existing = open_regular_file_nofollow(&save_path)
.with_context(|| format!("无法读取历史元数据: {}", save_path.display()))?; .with_context(|| format!("无法读取历史元数据: {}", save_path.display()))?;
let old_meta = FileMeta::from_reader(existing)?; let old_meta = FileMeta::from_reader(existing)?;
// 使用进度条计算快速哈希 // 使用进度条计算快速哈希
let file = open_regular_file_nofollow(path)?;
let file_size = file
.metadata()
.with_context(|| format!("无法读取文件信息: {}", path.display()))?
.len();
let tracker = ProgressTracker::new_single_file(file_size, &basename); let tracker = ProgressTracker::new_single_file(file_size, &basename);
let on_bytes = tracker.bytes_callback(); let on_bytes = tracker.bytes_callback();
let on_iop = tracker.iop_callback(); let on_iop = tracker.iop_callback();
let fast_hash = calc_xxh128_with_callback(path, on_bytes, on_iop)?; let fast_hash = calc_xxh128_from_file_with_callback(path, file, on_bytes, on_iop)?;
tracker.finish("校验完成"); tracker.finish("校验完成");
if fast_hash == old_meta.xxh128 { if fast_hash == old_meta.xxh128 {
@@ -79,10 +110,15 @@ fn process_file(path: &Path) -> Result<()> {
println!("校验失败!"); println!("校验失败!");
println!("现校验文件:"); println!("现校验文件:");
let file = open_regular_file_nofollow(path)?;
let file_size = file
.metadata()
.with_context(|| format!("无法读取文件信息: {}", path.display()))?
.len();
let tracker = ProgressTracker::new_single_file(file_size, &basename); let tracker = ProgressTracker::new_single_file(file_size, &basename);
let on_bytes = tracker.bytes_callback(); let on_bytes = tracker.bytes_callback();
let on_iop = tracker.iop_callback(); let on_iop = tracker.iop_callback();
let meta = FileMeta::from_path_with_callback(path, on_bytes, on_iop)?; let meta = FileMeta::from_open_file_with_callback(path, file, on_bytes, on_iop)?;
tracker.finish("处理完成"); tracker.finish("处理完成");
println!("{}", meta.to_pretty_json()?); println!("{}", meta.to_pretty_json()?);
println!("原校验文件:"); println!("原校验文件:");
@@ -95,23 +131,30 @@ fn process_dir(path: &Path) -> Result<()> {
let meta_path = path.join("meta.json"); let meta_path = path.join("meta.json");
let backup_path = path.join("meta-old.json"); let backup_path = path.join("meta-old.json");
if !meta_path.exists() { let Some(meta_info) = symlink_metadata_optional(&meta_path)? else {
let snapshot = DirSnapshot::build_root(path)?; let snapshot = DirSnapshot::build_root(path)?;
let json = serde_json::to_string_pretty(&snapshot)?; let json = serde_json::to_string_pretty(&snapshot)?;
write_atomic(&meta_path, &json)?; write_atomic(&meta_path, &json)?;
return Ok(()); return Ok(());
};
if meta_info.file_type().is_symlink() {
return Err(anyhow!("不支持符号链接元数据文件: {}", meta_path.display()));
}
if !meta_info.is_file() {
return Err(anyhow!("{} 不是文件", meta_path.display()));
} }
if backup_path.exists() { if symlink_metadata_optional(&backup_path)?.is_some() {
fs::remove_file(&backup_path)?; fs::remove_file(&backup_path)?;
} }
fs::rename(&meta_path, &backup_path) fs::rename(&meta_path, &backup_path)
.with_context(|| format!("无法重命名旧meta: {}", meta_path.display()))?; .with_context(|| format!("无法重命名旧meta: {}", meta_path.display()))?;
sync_parent_dir(&backup_path)?;
println!("发现旧元数据,已暂存为 meta-old.json,开始校验..."); println!("发现旧元数据,已暂存为 meta-old.json,开始校验...");
let meta_file = let meta_file = open_regular_file_nofollow(&backup_path)
File::open(&backup_path).with_context(|| format!("无法读取: {}", backup_path.display()))?; .with_context(|| format!("无法读取: {}", backup_path.display()))?;
let snapshot = DirSnapshot::from_reader(meta_file)?; let snapshot = DirSnapshot::from_reader(meta_file)?;
let mut stored = snapshot.collect_file_map(path); let mut stored = snapshot.collect_file_map(path);
let current = scan_dir_xxh128(path)?; let current = scan_dir_xxh128(path)?;
@@ -145,6 +188,7 @@ fn process_dir(path: &Path) -> Result<()> {
println!("校验通过."); println!("校验通过.");
fs::rename(&backup_path, &meta_path) fs::rename(&backup_path, &meta_path)
.with_context(|| format!("无法恢复meta: {}", meta_path.display()))?; .with_context(|| format!("无法恢复meta: {}", meta_path.display()))?;
sync_parent_dir(&meta_path)?;
} }
Ok(()) Ok(())
@@ -180,6 +224,7 @@ fn write_atomic(path: &Path, contents: &str) -> Result<()> {
path.display() path.display()
) )
})?; })?;
sync_parent_dir(path)?;
Ok(()) Ok(())
})(); })();
@@ -189,3 +234,19 @@ fn write_atomic(path: &Path, contents: &str) -> Result<()> {
result result
} }
fn symlink_metadata_optional(path: &Path) -> Result<Option<fs::Metadata>> {
match fs::symlink_metadata(path) {
Ok(info) => Ok(Some(info)),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(err).with_context(|| format!("无法读取路径信息: {}", path.display())),
}
}
fn sync_parent_dir(path: &Path) -> Result<()> {
let parent = path.parent().unwrap_or_else(|| Path::new("."));
let dir = File::open(parent)
.with_context(|| format!("无法打开父目录以同步: {}", parent.display()))?;
dir.sync_all()
.with_context(|| format!("无法同步父目录: {}", parent.display()))
}
+77 -7
View File
@@ -1,4 +1,4 @@
use std::fs::{self, File}; use std::fs::{File, OpenOptions};
use std::io::Read; use std::io::Read;
use std::path::Path; use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
@@ -16,6 +16,11 @@ use crate::constants::{DEFAULT_BUFFER_SIZE, HEAD_115_BYTES, HEAD_BAIDU_BYTES};
use crate::head_hash::{calc_head_115, calc_head_baidu, HeadChunk}; use crate::head_hash::{calc_head_115, calc_head_baidu, HeadChunk};
use crate::utils::{basename, friendly_size, hex_upper}; use crate::utils::{basename, friendly_size, hex_upper};
#[cfg(not(unix))]
use std::fs;
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileMeta { pub struct FileMeta {
pub basename: String, pub basename: String,
@@ -32,8 +37,23 @@ pub struct FileMeta {
} }
impl FileMeta { impl FileMeta {
#[cfg(not(unix))]
pub fn from_path_with_callback<F1, F2>( pub fn from_path_with_callback<F1, F2>(
path: &Path, path: &Path,
on_bytes_read: F1,
on_iop: F2,
) -> Result<Self>
where
F1: FnMut(u64),
F2: FnMut(),
{
let file = open_regular_file_nofollow(path)?;
Self::from_open_file_with_callback(path, file, on_bytes_read, on_iop)
}
pub(crate) fn from_open_file_with_callback<F1, F2>(
path: &Path,
mut file: File,
mut on_bytes_read: F1, mut on_bytes_read: F1,
mut on_iop: F2, mut on_iop: F2,
) -> Result<Self> ) -> Result<Self>
@@ -41,8 +61,9 @@ impl FileMeta {
F1: FnMut(u64), F1: FnMut(u64),
F2: FnMut(), F2: FnMut(),
{ {
let info = let info = file
fs::metadata(path).with_context(|| format!("无法读取文件信息: {}", path.display()))?; .metadata()
.with_context(|| format!("无法读取文件信息: {}", path.display()))?;
if !info.is_file() { if !info.is_file() {
return Err(anyhow!("{} 不是文件", path.display())); return Err(anyhow!("{} 不是文件", path.display()));
} }
@@ -60,9 +81,6 @@ impl FileMeta {
.map(|d| d.as_secs() as i64) .map(|d| d.as_secs() as i64)
.unwrap_or(0); .unwrap_or(0);
let mut file =
File::open(path).with_context(|| format!("无法打开文件: {}", path.display()))?;
let mut buffer = vec![0u8; DEFAULT_BUFFER_SIZE]; let mut buffer = vec![0u8; DEFAULT_BUFFER_SIZE];
let mut md5_hasher = Md5::new(); let mut md5_hasher = Md5::new();
let mut sha1_hasher = Sha1::new(); let mut sha1_hasher = Sha1::new();
@@ -124,8 +142,23 @@ impl FileMeta {
} }
} }
#[cfg(not(unix))]
pub fn calc_xxh128_with_callback<F1, F2>( pub fn calc_xxh128_with_callback<F1, F2>(
path: &Path, path: &Path,
on_bytes_read: F1,
on_iop: F2,
) -> Result<String>
where
F1: FnMut(u64),
F2: FnMut(),
{
let file = open_regular_file_nofollow(path)?;
calc_xxh128_from_file_with_callback(path, file, on_bytes_read, on_iop)
}
pub(crate) fn calc_xxh128_from_file_with_callback<F1, F2>(
path: &Path,
mut file: File,
mut on_bytes_read: F1, mut on_bytes_read: F1,
mut on_iop: F2, mut on_iop: F2,
) -> Result<String> ) -> Result<String>
@@ -133,7 +166,13 @@ where
F1: FnMut(u64), F1: FnMut(u64),
F2: FnMut(), F2: FnMut(),
{ {
let mut file = File::open(path).with_context(|| format!("无法打开文件: {}", path.display()))?; let info = file
.metadata()
.with_context(|| format!("无法读取文件信息: {}", path.display()))?;
if !info.is_file() {
return Err(anyhow!("{} 不是文件", path.display()));
}
let mut buffer = vec![0u8; DEFAULT_BUFFER_SIZE]; let mut buffer = vec![0u8; DEFAULT_BUFFER_SIZE];
let mut hasher = Xxh3::new(); let mut hasher = Xxh3::new();
@@ -149,3 +188,34 @@ where
Ok(hex_upper(hasher.digest128().to_be_bytes())) Ok(hex_upper(hasher.digest128().to_be_bytes()))
} }
pub(crate) fn open_regular_file_nofollow(path: &Path) -> Result<File> {
let file = open_file_nofollow(path)?;
let info = file
.metadata()
.with_context(|| format!("无法读取文件信息: {}", path.display()))?;
if !info.is_file() {
return Err(anyhow!("{} 不是文件", path.display()));
}
Ok(file)
}
#[cfg(unix)]
fn open_file_nofollow(path: &Path) -> Result<File> {
let mut options = OpenOptions::new();
options.read(true);
options.custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW);
options
.open(path)
.with_context(|| format!("无法打开文件: {}", path.display()))
}
#[cfg(not(unix))]
fn open_file_nofollow(path: &Path) -> Result<File> {
let info = fs::symlink_metadata(path)
.with_context(|| format!("无法读取文件信息: {}", path.display()))?;
if info.file_type().is_symlink() {
return Err(anyhow!("不支持扫描符号链接: {}", path.display()));
}
File::open(path).with_context(|| format!("无法打开文件: {}", path.display()))
}
+2 -1
View File
@@ -2,6 +2,7 @@ mod file;
mod progress; mod progress;
mod tree; mod tree;
pub use file::{calc_xxh128_with_callback, FileMeta}; pub use file::FileMeta;
pub(crate) use file::{calc_xxh128_from_file_with_callback, open_regular_file_nofollow};
pub use progress::ProgressTracker; pub use progress::ProgressTracker;
pub use tree::{scan_dir_xxh128, DirSnapshot}; pub use tree::{scan_dir_xxh128, DirSnapshot};
+473 -7
View File
@@ -1,16 +1,26 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::fs;
use std::fs::File;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::{anyhow, Context, Result}; use anyhow::Result;
#[cfg(not(unix))]
use anyhow::{anyhow, Context};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use super::file::{calc_xxh128_with_callback, FileMeta}; #[cfg(unix)]
use super::file::calc_xxh128_from_file_with_callback;
#[cfg(not(unix))]
use super::file::calc_xxh128_with_callback;
use super::file::FileMeta;
use super::progress::ProgressTracker; use super::progress::ProgressTracker;
use crate::constants::META_VERSION; use crate::constants::META_VERSION;
#[cfg(not(unix))]
use crate::utils::{basename, should_skip_dir, should_skip_file}; use crate::utils::{basename, should_skip_dir, should_skip_file};
#[cfg(not(unix))]
use std::fs;
#[cfg(not(unix))]
use std::fs::File;
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DirSnapshot { pub struct DirSnapshot {
pub dir_name: String, pub dir_name: String,
@@ -37,6 +47,12 @@ impl DirSnapshot {
Ok(serde_json::from_reader(reader)?) Ok(serde_json::from_reader(reader)?)
} }
#[cfg(unix)]
fn build_node(path: &Path, tracker: &ProgressTracker) -> Result<Self> {
unix_walk::build_node(path, tracker)
}
#[cfg(not(unix))]
fn build_node(path: &Path, tracker: &ProgressTracker) -> Result<Self> { fn build_node(path: &Path, tracker: &ProgressTracker) -> Result<Self> {
let dir_name = path let dir_name = path
.file_name() .file_name()
@@ -120,6 +136,7 @@ impl DirSnapshot {
/// 加载子目录的 meta.json 并通过 xxh128 快速校验。 /// 加载子目录的 meta.json 并通过 xxh128 快速校验。
/// 校验通过则返回已有的 DirSnapshot,否则返回 Err 终止流程。 /// 校验通过则返回已有的 DirSnapshot,否则返回 Err 终止流程。
#[cfg(not(unix))]
fn verify_and_load(path: &Path, tracker: &ProgressTracker) -> Result<Self> { fn verify_and_load(path: &Path, tracker: &ProgressTracker) -> Result<Self> {
let meta_path = path.join("meta.json"); let meta_path = path.join("meta.json");
let meta_file = let meta_file =
@@ -181,11 +198,20 @@ pub fn scan_dir_xxh128(path: &Path) -> Result<BTreeMap<PathBuf, String>> {
} }
fn count_files(path: &Path) -> Result<u64> { fn count_files(path: &Path) -> Result<u64> {
#[cfg(unix)]
{
unix_walk::count_files(path)
}
#[cfg(not(unix))]
{
let mut count = 0u64; let mut count = 0u64;
count_files_recursive(path, &mut count)?; count_files_recursive(path, &mut count)?;
Ok(count) Ok(count)
} }
}
#[cfg(not(unix))]
fn count_files_recursive(path: &Path, count: &mut u64) -> Result<()> { fn count_files_recursive(path: &Path, count: &mut u64) -> Result<()> {
let entries = fs::read_dir(path) let entries = fs::read_dir(path)
.with_context(|| format!("无法遍历目录: {}", path.display()))? .with_context(|| format!("无法遍历目录: {}", path.display()))?
@@ -209,12 +235,10 @@ fn count_files_recursive(path: &Path, count: &mut u64) -> Result<()> {
continue; continue;
} }
count_files_recursive(&full_path, count)?; count_files_recursive(&full_path, count)?;
} else { } else if !should_skip_file(&name) {
if !should_skip_file(&name) {
*count += 1; *count += 1;
} }
} }
}
Ok(()) Ok(())
} }
@@ -224,6 +248,13 @@ fn walk_dir_with_progress(
map: &mut BTreeMap<PathBuf, String>, map: &mut BTreeMap<PathBuf, String>,
tracker: &ProgressTracker, tracker: &ProgressTracker,
) -> Result<()> { ) -> Result<()> {
#[cfg(unix)]
{
unix_walk::walk_dir_with_progress(path, map, tracker)
}
#[cfg(not(unix))]
{
let mut entries = fs::read_dir(path) let mut entries = fs::read_dir(path)
.with_context(|| format!("无法遍历目录: {}", path.display()))? .with_context(|| format!("无法遍历目录: {}", path.display()))?
.collect::<Result<Vec<_>, _>>() .collect::<Result<Vec<_>, _>>()
@@ -267,3 +298,438 @@ fn walk_dir_with_progress(
Ok(()) Ok(())
} }
}
#[cfg(unix)]
mod unix_walk {
use std::collections::BTreeMap;
use std::ffi::{CStr, CString, OsStr, OsString};
use std::fs::File;
use std::io;
use std::mem::MaybeUninit;
use std::os::fd::{AsRawFd, FromRawFd, RawFd};
use std::os::unix::ffi::{OsStrExt, OsStringExt};
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use anyhow::{anyhow, Context, Result};
use super::{calc_xxh128_from_file_with_callback, DirSnapshot, FileMeta, ProgressTracker};
use crate::utils::{basename, should_skip_dir, should_skip_file};
struct DirHandle {
file: File,
}
struct DirEntryInfo {
name: OsString,
stat: libc::stat,
}
enum EntryKind {
Directory,
RegularFile,
Symlink,
Other,
}
impl DirHandle {
fn open_path(path: &Path) -> Result<Self> {
let c_path = cstring_from_path(path)?;
let fd = unsafe { libc::open(c_path.as_ptr(), dir_open_flags()) };
if fd == -1 {
return Err(io::Error::last_os_error())
.with_context(|| format!("无法打开目录: {}", path.display()));
}
Ok(Self {
file: unsafe { File::from_raw_fd(fd) },
})
}
fn raw_fd(&self) -> RawFd {
self.file.as_raw_fd()
}
fn entries(&self, path: &Path) -> Result<Vec<DirEntryInfo>> {
let dup_fd = unsafe { libc::dup(self.raw_fd()) };
if dup_fd == -1 {
return Err(io::Error::last_os_error())
.with_context(|| format!("无法遍历目录: {}", path.display()));
}
let dir = unsafe { libc::fdopendir(dup_fd) };
if dir.is_null() {
let err = io::Error::last_os_error();
unsafe {
libc::close(dup_fd);
}
return Err(err).with_context(|| format!("无法遍历目录: {}", path.display()));
}
let _stream = DirStream(dir);
let mut entries = Vec::new();
loop {
let entry = unsafe { libc::readdir(dir) };
if entry.is_null() {
break;
}
let name_bytes = unsafe { CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes();
if name_bytes == b"." || name_bytes == b".." {
continue;
}
let name = OsString::from_vec(name_bytes.to_vec());
let full_path = path.join(&name);
if let Some(stat) = self.stat_child(&name, &full_path)? {
entries.push(DirEntryInfo { name, stat });
}
}
entries.sort_unstable_by(|left, right| left.name.cmp(&right.name));
Ok(entries)
}
fn stat_child(&self, name: &OsStr, path: &Path) -> Result<Option<libc::stat>> {
let c_name = cstring_from_os_str(name, path)?;
let mut stat = MaybeUninit::<libc::stat>::uninit();
let code = unsafe {
libc::fstatat(
self.raw_fd(),
c_name.as_ptr(),
stat.as_mut_ptr(),
libc::AT_SYMLINK_NOFOLLOW,
)
};
if code == -1 {
let err = io::Error::last_os_error();
if err.kind() == io::ErrorKind::NotFound {
return Ok(None);
}
return Err(err).with_context(|| format!("无法读取类型: {}", path.display()));
}
Ok(Some(unsafe { stat.assume_init() }))
}
fn has_regular_child(&self, name: &OsStr, path: &Path) -> Result<bool> {
Ok(matches!(
self.stat_child(name, path)?
.map(|stat| kind_from_mode(stat.st_mode)),
Some(EntryKind::RegularFile)
))
}
fn open_child_dir(&self, entry: &DirEntryInfo, path: &Path) -> Result<Self> {
let c_name = cstring_from_os_str(&entry.name, path)?;
let fd = unsafe { libc::openat(self.raw_fd(), c_name.as_ptr(), dir_open_flags()) };
if fd == -1 {
return Err(io::Error::last_os_error())
.with_context(|| format!("无法打开目录: {}", path.display()));
}
let file = unsafe { File::from_raw_fd(fd) };
let info = file
.metadata()
.with_context(|| format!("无法读取目录信息: {}", path.display()))?;
if !stat_matches(&info, &entry.stat) {
return Err(anyhow!("扫描期间目录被替换: {}", path.display()));
}
Ok(Self { file })
}
fn open_child_file(&self, entry: &DirEntryInfo, path: &Path) -> Result<File> {
let c_name = cstring_from_os_str(&entry.name, path)?;
let fd = unsafe { libc::openat(self.raw_fd(), c_name.as_ptr(), file_open_flags()) };
if fd == -1 {
return Err(io::Error::last_os_error())
.with_context(|| format!("无法打开文件: {}", path.display()));
}
let file = unsafe { File::from_raw_fd(fd) };
let info = file
.metadata()
.with_context(|| format!("无法读取文件信息: {}", path.display()))?;
if !info.is_file() {
return Err(anyhow!("{} 不是文件", path.display()));
}
if !stat_matches(&info, &entry.stat) {
return Err(anyhow!("扫描期间文件被替换: {}", path.display()));
}
Ok(file)
}
}
struct DirStream(*mut libc::DIR);
impl Drop for DirStream {
fn drop(&mut self) {
unsafe {
libc::closedir(self.0);
}
}
}
pub(super) fn build_node(path: &Path, tracker: &ProgressTracker) -> Result<DirSnapshot> {
let dir = DirHandle::open_path(path)?;
build_node_at(path, &dir, tracker)
}
pub(super) fn count_files(path: &Path) -> Result<u64> {
let dir = DirHandle::open_path(path)?;
let mut count = 0u64;
count_files_at(path, &dir, &mut count)?;
Ok(count)
}
pub(super) fn walk_dir_with_progress(
path: &Path,
map: &mut BTreeMap<PathBuf, String>,
tracker: &ProgressTracker,
) -> Result<()> {
let dir = DirHandle::open_path(path)?;
walk_dir_with_progress_at(path, &dir, map, tracker)
}
fn build_node_at(
path: &Path,
dir: &DirHandle,
tracker: &ProgressTracker,
) -> Result<DirSnapshot> {
let dir_name = path
.file_name()
.map(basename)
.unwrap_or_else(|| path.to_string_lossy().to_string());
let mut dirs = Vec::new();
let mut files = Vec::new();
for entry in dir.entries(path)? {
let name = entry.name.to_string_lossy().to_string();
let full_path = path.join(&entry.name);
match kind_from_mode(entry.stat.st_mode) {
EntryKind::Symlink => continue,
EntryKind::Directory => {
if should_skip_dir(&name) {
continue;
}
let child = dir.open_child_dir(&entry, &full_path)?;
let child_meta_path = full_path.join("meta.json");
if child.has_regular_child(OsStr::new("meta.json"), &child_meta_path)? {
dirs.push(verify_and_load_at(&full_path, &child, tracker)?);
} else {
dirs.push(build_node_at(&full_path, &child, tracker)?);
}
}
EntryKind::RegularFile => {
if should_skip_file(&name) {
continue;
}
let file_size = stat_size(&entry.stat);
tracker.start_file(file_size, &name);
let file = dir.open_child_file(&entry, &full_path)?;
let on_bytes = tracker.bytes_callback();
let on_iop = tracker.iop_callback();
let meta =
FileMeta::from_open_file_with_callback(&full_path, file, on_bytes, on_iop)?;
files.push(meta);
tracker.finish_file();
}
EntryKind::Other => {
if !should_skip_file(&name) {
return Err(anyhow!("{} 不是文件", full_path.display()));
}
}
}
}
Ok(DirSnapshot {
dir_name,
dirs,
files,
v: None,
})
}
fn verify_and_load_at(
path: &Path,
dir: &DirHandle,
tracker: &ProgressTracker,
) -> Result<DirSnapshot> {
let meta_name = OsStr::new("meta.json");
let meta_path = path.join(meta_name);
let Some(meta_stat) = dir.stat_child(meta_name, &meta_path)? else {
return build_node_at(path, dir, tracker);
};
let meta_entry = DirEntryInfo {
name: meta_name.to_os_string(),
stat: meta_stat,
};
let meta_file = dir.open_child_file(&meta_entry, &meta_path)?;
let mut snapshot: DirSnapshot = serde_json::from_reader(meta_file)
.with_context(|| format!("无法解析: {}", meta_path.display()))?;
let mut stored = snapshot.collect_file_map(path);
let mut current = BTreeMap::new();
walk_dir_with_progress_at(path, dir, &mut current, tracker)?;
for (file_path, hash) in current {
if let Some(meta) = stored.remove(&file_path) {
if hash != meta.xxh128 {
return Err(anyhow!(
"校验失败: {}\n 期望: {}\n 当前: {}",
file_path.display(),
meta.xxh128,
hash
));
}
} else {
return Err(anyhow!("文件新增: {}", file_path.display()));
}
}
if let Some((missing_path, _)) = stored.into_iter().next() {
return Err(anyhow!("文件缺失: {}", missing_path.display()));
}
let msg = format!("✓ 校验通过: {}", path.display());
if let Some(multi) = tracker.multi() {
multi.suspend(|| {
eprintln!("{msg}");
});
} else {
eprintln!("{msg}");
}
snapshot.dir_name = path
.file_name()
.map(basename)
.unwrap_or_else(|| path.to_string_lossy().to_string());
snapshot.v = None;
Ok(snapshot)
}
fn count_files_at(path: &Path, dir: &DirHandle, count: &mut u64) -> Result<()> {
for entry in dir.entries(path)? {
let name = entry.name.to_string_lossy().to_string();
let full_path = path.join(&entry.name);
match kind_from_mode(entry.stat.st_mode) {
EntryKind::Symlink => continue,
EntryKind::Directory => {
if should_skip_dir(&name) {
continue;
}
let child = dir.open_child_dir(&entry, &full_path)?;
count_files_at(&full_path, &child, count)?;
}
EntryKind::RegularFile => {
if !should_skip_file(&name) {
*count += 1;
}
}
EntryKind::Other => {
if !should_skip_file(&name) {
return Err(anyhow!("{} 不是文件", full_path.display()));
}
}
}
}
Ok(())
}
fn walk_dir_with_progress_at(
path: &Path,
dir: &DirHandle,
map: &mut BTreeMap<PathBuf, String>,
tracker: &ProgressTracker,
) -> Result<()> {
for entry in dir.entries(path)? {
let name = entry.name.to_string_lossy().to_string();
let full_path = path.join(&entry.name);
match kind_from_mode(entry.stat.st_mode) {
EntryKind::Symlink => continue,
EntryKind::Directory => {
if should_skip_dir(&name) {
continue;
}
let child = dir.open_child_dir(&entry, &full_path)?;
walk_dir_with_progress_at(&full_path, &child, map, tracker)?;
}
EntryKind::RegularFile => {
if should_skip_file(&name) {
continue;
}
let file_size = stat_size(&entry.stat);
tracker.start_file(file_size, &name);
let file = dir.open_child_file(&entry, &full_path)?;
let on_bytes = tracker.bytes_callback();
let on_iop = tracker.iop_callback();
let hash =
calc_xxh128_from_file_with_callback(&full_path, file, on_bytes, on_iop)?;
map.insert(full_path, hash);
tracker.finish_file();
}
EntryKind::Other => {
if !should_skip_file(&name) {
return Err(anyhow!("{} 不是文件", full_path.display()));
}
}
}
}
Ok(())
}
fn kind_from_mode(mode: libc::mode_t) -> EntryKind {
match mode & libc::S_IFMT as libc::mode_t {
value if value == libc::S_IFDIR as libc::mode_t => EntryKind::Directory,
value if value == libc::S_IFREG as libc::mode_t => EntryKind::RegularFile,
value if value == libc::S_IFLNK as libc::mode_t => EntryKind::Symlink,
_ => EntryKind::Other,
}
}
fn stat_size(stat: &libc::stat) -> u64 {
if stat.st_size >= 0 {
stat.st_size as u64
} else {
0
}
}
fn stat_matches(info: &std::fs::Metadata, stat: &libc::stat) -> bool {
info.dev() == stat.st_dev as u64 && info.ino() == stat.st_ino
}
fn dir_open_flags() -> libc::c_int {
libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY | libc::O_NOFOLLOW
}
fn file_open_flags() -> libc::c_int {
libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW
}
fn cstring_from_path(path: &Path) -> Result<CString> {
CString::new(path.as_os_str().as_bytes())
.with_context(|| format!("路径包含 NUL 字节: {}", path.display()))
}
fn cstring_from_os_str(value: &OsStr, path: &Path) -> Result<CString> {
CString::new(value.as_bytes())
.with_context(|| format!("路径包含 NUL 字节: {}", path.display()))
}
}