tyler
/
proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

pxar: factor out PxarCreateOptions 

containing the CLI parameters that are mostly passed-through from the
client to our pxar archive creation wrapper in pxar::create

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
This commit is contained in:
Fabian Grünbichler 2021-01-25 14:42:54 +01:00 committed by Wolfgang Bumiller
parent e97025ab02
commit 77486a608e
6 changed files with 61 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/bin/proxmox-backup-client.rs
View File

@ -283,23 +283,15 @@ async fn backup_directory<P: AsRef<Path>>(
dir_path: P,
archive_name: &str,
chunk_size: Option<usize>,
device_set: Option<HashSet<u64>>,
verbose: bool,
skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
exclude_pattern: Vec<MatchEntry>,
entries_max: usize,
pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
upload_options: UploadOptions,
) -> Result<BackupStats, Error> {
let pxar_stream = PxarBackupStream::open(
dir_path.as_ref(),
device_set,
verbose,
skip_lost_and_found,
catalog,
exclude_pattern,
entries_max,
pxar_create_options,
)?;
let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
@ -1039,6 +1031,15 @@ async fn create_backup(
println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
device_set: devices.clone(),
patterns: pattern_list.clone(),
entries_max: entries_max as usize,
skip_lost_and_found,
verbose,
};
let upload_options = UploadOptions {
previous_manifest: previous_manifest.clone(),
compress: true,
@ -1051,12 +1052,8 @@ async fn create_backup(
&filename,
&target,
chunk_size_opt,
devices.clone(),
verbose,
skip_lost_and_found,
catalog.clone(),
pattern_list.clone(),
entries_max as usize,
pxar_options,
upload_options,
).await?;
manifest.add_file(target, stats.size, stats.csum, crypt_mode)?;

22
src/bin/pxar.rs
View File

@ -311,16 +311,16 @@ fn create_archive(
exclude: Option<Vec<String>>,
entries_max: isize,
) -> Result<(), Error> {
let pattern_list = {
let patterns = {
let input = exclude.unwrap_or_else(Vec::new);
let mut pattern_list = Vec::with_capacity(input.len());
let mut patterns = Vec::with_capacity(input.len());
for entry in input {
pattern_list.push(
patterns.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
.map_err(|err| format_err!("error in exclude pattern: {}", err))?,
);
}
pattern_list
patterns
};
let device_set = if all_file_systems {
@ -329,6 +329,15 @@ fn create_archive(
Some(HashSet::new())
};
let options = proxmox_backup::pxar::PxarCreateOptions {
entries_max: entries_max as usize,
device_set,
patterns,
verbose,
skip_lost_and_found: false,
};
let source = PathBuf::from(source);
let dir = nix::dir::Dir::open(
@ -368,18 +377,15 @@ fn create_archive(
proxmox_backup::pxar::create_archive(
dir,
writer,
pattern_list,
feature_flags,
device_set,
false,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
entries_max as usize,
None,
options,
)?;
Ok(())

28
src/client/pxar_backup_stream.rs
View File

@ -1,4 +1,3 @@
use std::collections::HashSet;
use std::io::Write;
//use std::os::unix::io::FromRawFd;
use std::path::Path;
@ -13,8 +12,6 @@ use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
use pathpatterns::MatchEntry;
use crate::backup::CatalogWriter;
/// Stream implementation to encode and upload .pxar archives.
@ -38,12 +35,8 @@ impl Drop for PxarBackupStream {
impl PxarBackupStream {
pub fn new<W: Write + Send + 'static>(
dir: Dir,
device_set: Option<HashSet<u64>>,
verbose: bool,
skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
patterns: Vec<MatchEntry>,
entries_max: usize,
options: crate::pxar::PxarCreateOptions,
) -> Result<Self, Error> {
let (tx, rx) = std::sync::mpsc::sync_channel(10);
@ -61,22 +54,21 @@ impl PxarBackupStream {
crate::tools::StdChannelWriter::new(tx),
);
let verbose = options.verbose;
let writer = pxar::encoder::sync::StandardWriter::new(writer);
if let Err(err) = crate::pxar::create_archive(
dir,
writer,
patterns,
crate::pxar::Flags::DEFAULT,
device_set,
skip_lost_and_found,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
entries_max,
Some(&mut *catalog_guard),
options,
) {
let mut error = error.lock().unwrap();
*error = Some(err.to_string());
@ -93,23 +85,15 @@ impl PxarBackupStream {
pub fn open<W: Write + Send + 'static>(
dirname: &Path,
device_set: Option<HashSet<u64>>,
verbose: bool,
skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
patterns: Vec<MatchEntry>,
entries_max: usize,
options: crate::pxar::PxarCreateOptions,
) -> Result<Self, Error> {
let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
Self::new(
dir,
device_set,
verbose,
skip_lost_and_found,
catalog,
patterns,
entries_max,
options,
)
}
}

28
src/pxar/create.rs
View File

@ -27,6 +27,22 @@ use crate::pxar::Flags;
use crate::pxar::tools::assert_single_path_component;
use crate::tools::{acl, fs, xattr, Fd};
/// Pxar options for creating a pxar archive/stream
#[derive(Default, Clone)]
pub struct PxarCreateOptions {
/// Device/mountpoint st_dev numbers that should be included. None for no limitation.
pub device_set: Option<HashSet<u64>>,
/// Exclusion patterns
pub patterns: Vec<MatchEntry>,
/// Maximum number of entries to hold in memory
pub entries_max: usize,
/// Skip lost+found directory
pub skip_lost_and_found: bool,
/// Verbose output
pub verbose: bool,
}
fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
let mut fs_stat = std::mem::MaybeUninit::uninit();
let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
@ -136,13 +152,10 @@ type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::Seq
pub fn create_archive<T, F>(
source_dir: Dir,
mut writer: T,
mut patterns: Vec<MatchEntry>,
feature_flags: Flags,
mut device_set: Option<HashSet<u64>>,
skip_lost_and_found: bool,
mut callback: F,
entry_limit: usize,
catalog: Option<&mut dyn BackupCatalogWriter>,
options: PxarCreateOptions,
) -> Result<(), Error>
where
T: pxar::encoder::SeqWrite,
@ -164,6 +177,7 @@ where
)
.map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?;
let mut device_set = options.device_set.clone();
if let Some(ref mut set) = device_set {
set.insert(stat.st_dev);
}
@ -171,7 +185,9 @@ where
let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite;
let mut encoder = Encoder::new(writer, &metadata)?;
if skip_lost_and_found {
let mut patterns = options.patterns.clone();
if options.skip_lost_and_found {
patterns.push(MatchEntry::parse_pattern(
"lost+found",
PatternFlag::PATH_NAME,
@ -188,7 +204,7 @@ where
catalog,
path: PathBuf::new(),
entry_counter: 0,
entry_limit,
entry_limit: options.entries_max,
current_st_dev: stat.st_dev,
device_set,
hardlinks: HashMap::new(),

2
src/pxar/mod.rs
View File

@ -58,7 +58,7 @@ pub(crate) mod tools;
mod flags;
pub use flags::Flags;
pub use create::create_archive;
pub use create::{create_archive, PxarCreateOptions};
pub use extract::{extract_archive, ErrorHandler};
/// The format requires to build sorted directory lookup tables in

10
tests/catar.rs
View File

@ -25,16 +25,18 @@ fn run_test(dir_name: &str) -> Result<(), Error> {
dir_name, nix::fcntl::OFlag::O_NOFOLLOW,
nix::sys::stat::Mode::empty())?;
let options = PxarCreateOptions {
entries_max: ENCODER_MAX_ENTRIES,
..PxarCreateOptions::default()
};
create_archive(
dir,
writer,
Vec::new(),
Flags::DEFAULT,
None,
false,
|_| Ok(()),
ENCODER_MAX_ENTRIES,
None,
options,
)?;
Command::new("cmp")