tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

switch to external pxar and fuse crates

Browse Source 
Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller 2020-03-23 15:03:18 +01:00
parent ab1092392f
commit c443f58b09
28 changed files with 4213 additions and 6191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/api2/backup.rs
View File

@ -628,9 +628,9 @@ fn dynamic_chunk_index(
let count = index.index_count();
for pos in 0..count {
let (start, end, digest) = index.chunk_info(pos)?;
let size = (end - start) as u32;
env.register_chunk(digest, size)?;
let info = index.chunk_info(pos)?;
let size = info.size() as u32;
env.register_chunk(info.digest, size)?;
}
let reader = DigestListEncoder::new(Box::new(index));

90
src/backup/catalog.rs
View File

@ -1,18 +1,18 @@
use anyhow::{bail, format_err, Error};
use std::fmt;
use std::ffi::{CStr, CString, OsStr};
use std::os::unix::ffi::OsStrExt;
use std::io::{Read, Write, Seek, SeekFrom};
use std::convert::TryFrom;
use std::ffi::{CStr, CString, OsStr};
use std::fmt;
use std::io::{Read, Write, Seek, SeekFrom};
use std::os::unix::ffi::OsStrExt;
use anyhow::{bail, format_err, Error};
use chrono::offset::{TimeZone, Local};
use proxmox::tools::io::ReadExt;
use pathpatterns::{MatchList, MatchType};
use proxmox::sys::error::io_err_other;
use proxmox::tools::io::ReadExt;
use crate::pxar::catalog::BackupCatalogWriter;
use crate::pxar::{MatchPattern, MatchPatternSlice, MatchType};
use crate::backup::file_formats::PROXMOX_CATALOG_FILE_MAGIC_1_0;
use crate::pxar::catalog::BackupCatalogWriter;
use crate::tools::runtime::block_on;
#[repr(u8)]
@ -63,7 +63,7 @@ pub struct DirEntry {
}
/// Used to specific additional attributes inside DirEntry
#[derive(Clone, PartialEq)]
#[derive(Clone, Debug, PartialEq)]
pub enum DirEntryAttribute {
Directory { start: u64 },
File { size: u64, mtime: u64 },
@ -106,6 +106,23 @@ impl DirEntry {
}
}
/// Get file mode bits for this entry to be used with the `MatchList` api.
pub fn get_file_mode(&self) -> Option<u32> {
Some(
match self.attr {
DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR,
DirEntryAttribute::File { .. } => pxar::mode::IFREG,
DirEntryAttribute::Symlink => pxar::mode::IFLNK,
DirEntryAttribute::Hardlink => return None,
DirEntryAttribute::BlockDevice => pxar::mode::IFBLK,
DirEntryAttribute::CharDevice => pxar::mode::IFCHR,
DirEntryAttribute::Fifo => pxar::mode::IFIFO,
DirEntryAttribute::Socket => pxar::mode::IFSOCK,
}
as u32
)
}
/// Check if DirEntry is a directory
pub fn is_directory(&self) -> bool {
match self.attr {
@ -476,7 +493,7 @@ impl <R: Read + Seek> CatalogReader<R> {
&mut self,
parent: &DirEntry,
filename: &[u8],
) -> Result<DirEntry, Error> {
) -> Result<Option<DirEntry>, Error> {
let start = match parent.attr {
DirEntryAttribute::Directory { start } => start,
@ -496,10 +513,7 @@ impl <R: Read + Seek> CatalogReader<R> {
Ok(false) // stop parsing
})?;
match item {
None => bail!("no such file"),
Some(entry) => Ok(entry),
}
Ok(item)
}
/// Read the raw directory info block from current reader position.
@ -555,38 +569,30 @@ impl <R: Read + Seek> CatalogReader<R> {
/// provided callback on them.
pub fn find(
&mut self,
mut entry: &mut Vec<DirEntry>,
pattern: &[MatchPatternSlice],
callback: &Box<fn(&[DirEntry])>,
parent: &DirEntry,
file_path: &mut Vec<u8>,
match_list: &impl MatchList, //&[MatchEntry],
callback: &mut dyn FnMut(&[u8]) -> Result<(), Error>,
) -> Result<(), Error> {
let parent = entry.last().unwrap();
if !parent.is_directory() {
return Ok(())
}
let file_len = file_path.len();
for e in self.read_dir(parent)? {
match MatchPatternSlice::match_filename_include(
&CString::new(e.name.clone())?,
e.is_directory(),
pattern,
)? {
(MatchType::Positive, _) => {
entry.push(e);
callback(&entry);
let pattern = MatchPattern::from_line(b"**/*").unwrap().unwrap();
let child_pattern = vec![pattern.as_slice()];
self.find(&mut entry, &child_pattern, callback)?;
entry.pop();
}
(MatchType::PartialPositive, child_pattern)
| (MatchType::PartialNegative, child_pattern) => {
entry.push(e);
self.find(&mut entry, &child_pattern, callback)?;
entry.pop();
}
_ => {}
let is_dir = e.is_directory();
file_path.truncate(file_len);
if !e.name.starts_with(b"/") {
file_path.reserve(e.name.len() + 1);
file_path.push(b'/');
}
file_path.extend(&e.name);
match match_list.matches(&file_path, e.get_file_mode()) {
Some(MatchType::Exclude) => continue,
Some(MatchType::Include) => callback(&file_path)?,
None => (),
}
if is_dir {
self.find(&e, file_path, match_list, callback)?;
}
}
file_path.truncate(file_len);
Ok(())
}

1902
src/backup/catalog_shell.rs
View File

File diff suppressed because it is too large Load Diff

55
src/backup/dynamic_index.rs
View File

@ -1,6 +1,7 @@
use std::convert::TryInto;
use std::fs::File;
use std::io::{BufWriter, Seek, SeekFrom, Write};
use std::ops::Range;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@ -13,6 +14,7 @@ use proxmox::tools::vec;
use super::chunk_stat::ChunkStat;
use super::chunk_store::ChunkStore;
use super::index::ChunkReadInfo;
use super::read_chunk::ReadChunk;
use super::Chunker;
use super::IndexFile;
@ -136,7 +138,7 @@ impl DynamicIndexReader {
}
#[allow(clippy::cast_ptr_alignment)]
pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> {
pub fn chunk_info(&self, pos: usize) -> Result<ChunkReadInfo, Error> {
if pos >= self.index_entries {
bail!("chunk index out of range");
}
@ -157,7 +159,10 @@ impl DynamicIndexReader {
);
}
Ok((start, end, unsafe { digest.assume_init() }))
Ok(ChunkReadInfo {
range: start..end,
digest: unsafe { digest.assume_init() },
})
}
#[inline]
@ -258,6 +263,25 @@ impl IndexFile for DynamicIndexReader {
}
}
struct CachedChunk {
range: Range<u64>,
data: Vec<u8>,
}
impl CachedChunk {
/// Perform sanity checks on the range and data size:
pub fn new(range: Range<u64>, data: Vec<u8>) -> Result<Self, Error> {
if data.len() as u64 != range.end - range.start {
bail!(
"read chunk with wrong size ({} != {})",
data.len(),
range.end - range.start,
);
}
Ok(Self { range, data })
}
}
pub struct BufferedDynamicReader<S> {
store: S,
index: DynamicIndexReader,
@ -266,7 +290,7 @@ pub struct BufferedDynamicReader<S> {
buffered_chunk_idx: usize,
buffered_chunk_start: u64,
read_offset: u64,
lru_cache: crate::tools::lru_cache::LruCache<usize, (u64, u64, Vec<u8>)>,
lru_cache: crate::tools::lru_cache::LruCache<usize, CachedChunk>,
}
struct ChunkCacher<'a, S> {
@ -274,10 +298,12 @@ struct ChunkCacher<'a, S> {
index: &'a DynamicIndexReader,
}
impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, (u64, u64, Vec<u8>)> for ChunkCacher<'a, S> {
fn fetch(&mut self, index: usize) -> Result<Option<(u64, u64, Vec<u8>)>, anyhow::Error> {
let (start, end, digest) = self.index.chunk_info(index)?;
self.store.read_chunk(&digest).and_then(|data| Ok(Some((start, end, data))))
impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, CachedChunk> for ChunkCacher<'a, S> {
fn fetch(&mut self, index: usize) -> Result<Option<CachedChunk>, Error> {
let info = self.index.chunk_info(index)?;
let range = info.range;
let data = self.store.read_chunk(&info.digest)?;
CachedChunk::new(range, data).map(Some)
}
}
@ -301,7 +327,8 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
}
fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
let (start, end, data) = self.lru_cache.access(
//let (start, end, data) = self.lru_cache.access(
let cached_chunk = self.lru_cache.access(
idx,
&mut ChunkCacher {
store: &mut self.store,
@ -309,21 +336,13 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
},
)?.ok_or_else(|| format_err!("chunk not found by cacher"))?;
if (*end - *start) != data.len() as u64 {
bail!(
"read chunk with wrong size ({} != {}",
(*end - *start),
data.len()
);
}
// fixme: avoid copy
self.read_buffer.clear();
self.read_buffer.extend_from_slice(&data);
self.read_buffer.extend_from_slice(&cached_chunk.data);
self.buffered_chunk_idx = idx;
self.buffered_chunk_start = *start;
self.buffered_chunk_start = cached_chunk.range.start;
//println!("BUFFER {} {}", self.buffered_chunk_start, end);
Ok(())
}

13
src/backup/index.rs
View File

@ -1,4 +1,5 @@
use std::collections::HashMap;
use std::ops::Range;
use std::pin::Pin;
use std::task::{Context, Poll};
@ -6,6 +7,18 @@ use bytes::{Bytes, BytesMut};
use anyhow::{format_err, Error};
use futures::*;
pub struct ChunkReadInfo {
pub range: Range<u64>,
pub digest: [u8; 32],
}
impl ChunkReadInfo {
#[inline]
pub fn size(&self) -> u64 {
self.range.end - self.range.start
}
}
/// Trait to get digest list from index files
///
/// To allow easy iteration over all used chunks.

159
src/bin/proxmox-backup-client.rs
View File

@ -1,13 +1,25 @@
use anyhow::{bail, format_err, Error};
use nix::unistd::{fork, ForkResult, pipe};
use std::os::unix::io::RawFd;
use chrono::{Local, DateTime, Utc, TimeZone};
use std::path::{Path, PathBuf};
use std::collections::{HashSet, HashMap};
use std::ffi::OsStr;
use std::io::{Write, Seek, SeekFrom};
use std::io::{self, Write, Seek, SeekFrom};
use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::io::RawFd;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use anyhow::{bail, format_err, Error};
use chrono::{Local, DateTime, Utc, TimeZone};
use futures::future::FutureExt;
use futures::select;
use futures::stream::{StreamExt, TryStreamExt};
use nix::unistd::{fork, ForkResult, pipe};
use serde_json::{json, Value};
use tokio::signal::unix::{signal, SignalKind};
use tokio::sync::mpsc;
use xdg::BaseDirectories;
use pathpatterns::{MatchEntry, MatchType, PatternFlag};
use proxmox::{sortable, identity};
use proxmox::tools::fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size};
use proxmox::sys::linux::tty;
@ -20,16 +32,7 @@ use proxmox_backup::tools;
use proxmox_backup::api2::types::*;
use proxmox_backup::client::*;
use proxmox_backup::backup::*;
use proxmox_backup::pxar::{ self, catalog::* };
use serde_json::{json, Value};
//use hyper::Body;
use std::sync::{Arc, Mutex};
//use regex::Regex;
use xdg::BaseDirectories;
use futures::*;
use tokio::sync::mpsc;
use proxmox_backup::pxar::catalog::*;
const ENV_VAR_PBS_FINGERPRINT: &str = "PBS_FINGERPRINT";
const ENV_VAR_PBS_PASSWORD: &str = "PBS_PASSWORD";
@ -243,7 +246,7 @@ async fn backup_directory<P: AsRef<Path>>(
skip_lost_and_found: bool,
crypt_config: Option<Arc<CryptConfig>>,
catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
exclude_pattern: Vec<pxar::MatchPattern>,
exclude_pattern: Vec<MatchEntry>,
entries_max: usize,
) -> Result<BackupStats, Error> {
@ -769,7 +772,7 @@ fn spawn_catalog_upload(
type: Integer,
description: "Max number of entries to hold in memory.",
optional: true,
default: pxar::ENCODER_MAX_ENTRIES as isize,
default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
},
"verbose": {
type: Boolean,
@ -812,17 +815,19 @@ async fn create_backup(
let include_dev = param["include-dev"].as_array();
let entries_max = param["entries-max"].as_u64().unwrap_or(pxar::ENCODER_MAX_ENTRIES as u64);
let entries_max = param["entries-max"].as_u64()
.unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
let empty = Vec::new();
let arg_pattern = param["exclude"].as_array().unwrap_or(&empty);
let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
let mut pattern_list = Vec::with_capacity(arg_pattern.len());
for s in arg_pattern {
let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
let p = pxar::MatchPattern::from_line(l.as_bytes())?
.ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
pattern_list.push(p);
let mut exclude_list = Vec::with_capacity(exclude_args.len());
for entry in exclude_args {
let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
exclude_list.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
.map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
);
}
let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
@ -966,7 +971,7 @@ async fn create_backup(
skip_lost_and_found,
crypt_config.clone(),
catalog.clone(),
pattern_list.clone(),
exclude_list.clone(),
entries_max as usize,
).await?;
manifest.add_file(target, stats.size, stats.csum)?;
@ -1246,18 +1251,19 @@ async fn restore(param: Value) -> Result<Value, Error> {
let mut reader = BufferedDynamicReader::new(index, chunk_reader);
if let Some(target) = target {
let feature_flags = pxar::flags::DEFAULT;
let mut decoder = pxar::SequentialDecoder::new(&mut reader, feature_flags);
decoder.set_callback(move |path| {
if verbose {
eprintln!("{:?}", path);
}
Ok(())
});
decoder.set_allow_existing_dirs(allow_existing_dirs);
decoder.restore(Path::new(target), &Vec::new())?;
proxmox_backup::pxar::extract_archive(
pxar::decoder::Decoder::from_std(reader)?,
Path::new(target),
&[],
proxmox_backup::pxar::flags::DEFAULT,
allow_existing_dirs,
|path| {
if verbose {
println!("{:?}", path);
}
},
)
.map_err(|err| format_err!("error extracting archive - {}", err))?;
} else {
let mut writer = std::fs::OpenOptions::new()
.write(true)
@ -1966,6 +1972,41 @@ fn mount(
}
}
use proxmox_backup::client::RemoteChunkReader;
/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
/// async use!
///
/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
/// so that we can properly access it from multiple threads simultaneously while not issuing
/// duplicate simultaneous reads over http.
struct BufferedDynamicReadAt {
inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
}
impl BufferedDynamicReadAt {
fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
Self {
inner: Mutex::new(inner),
}
}
}
impl pxar::accessor::ReadAt for BufferedDynamicReadAt {
fn poll_read_at(
self: Pin<&Self>,
_cx: &mut Context,
buf: &mut [u8],
offset: u64,
) -> Poll<io::Result<usize>> {
use std::io::Read;
tokio::task::block_in_place(move || {
let mut reader = self.inner.lock().unwrap();
reader.seek(SeekFrom::Start(offset))?;
Poll::Ready(Ok(reader.read(buf)?))
})
}
}
async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
let repo = extract_repository_from_value(&param)?;
let archive_name = tools::required_string_param(&param, "archive-name")?;
@ -2015,15 +2056,19 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
let most_used = index.find_most_used_chunks(8);
let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
let reader = BufferedDynamicReader::new(index, chunk_reader);
let decoder = pxar::Decoder::new(reader)?;
let archive_size = reader.archive_size();
let reader: proxmox_backup::pxar::fuse::Reader =
Arc::new(BufferedDynamicReadAt::new(reader));
let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
let options = OsStr::new("ro,default_permissions");
let mut session = pxar::fuse::Session::new(decoder, &options, pipe.is_none())
.map_err(|err| format_err!("pxar mount failed: {}", err))?;
// Mount the session but not call fuse deamonize as this will cause
// issues with the runtime after the fork
let deamonize = false;
session.mount(&Path::new(target), deamonize)?;
let session = proxmox_backup::pxar::fuse::Session::mount(
decoder,
&options,
false,
Path::new(target),
)
.map_err(|err| format_err!("pxar mount failed: {}", err))?;
if let Some(pipe) = pipe {
nix::unistd::chdir(Path::new("/")).unwrap();
@ -2045,8 +2090,13 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
nix::unistd::close(pipe).unwrap();
}
let multithreaded = true;
session.run_loop(multithreaded)?;
let mut interrupt = signal(SignalKind::interrupt())?;
select! {
res = session.fuse() => res?,
_ = interrupt.recv().fuse() => {
// exit on interrupted
}
}
} else {
bail!("unknown archive file extension (expected .pxar)");
}
@ -2129,11 +2179,10 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
let most_used = index.find_most_used_chunks(8);
let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
let reader = BufferedDynamicReader::new(index, chunk_reader);
let mut decoder = pxar::Decoder::new(reader)?;
decoder.set_callback(|path| {
println!("{:?}", path);
Ok(())
});
let archive_size = reader.archive_size();
let reader: proxmox_backup::pxar::fuse::Reader =
Arc::new(BufferedDynamicReadAt::new(reader));
let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
let tmpfile = client.download(CATALOG_NAME, tmpfile).await?;
let index = DynamicIndexReader::new(tmpfile)
@ -2161,10 +2210,10 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
catalog_reader,
&server_archive_name,
decoder,
)?;
).await?;
println!("Starting interactive shell");
state.shell()?;
state.shell().await?;
record_repository(&repo);

798
src/bin/pxar.rs
View File

@ -1,66 +1,21 @@
extern crate proxmox_backup;
use std::collections::HashSet;
use std::ffi::OsStr;
use std::fs::OpenOptions;
use std::os::unix::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use anyhow::{format_err, Error};
use futures::future::FutureExt;
use futures::select;
use tokio::signal::unix::{signal, SignalKind};
use pathpatterns::{MatchEntry, MatchType, PatternFlag};
use proxmox::{sortable, identity};
use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
use proxmox::api::schema::*;
use proxmox::api::cli::*;
use proxmox::api::api;
use proxmox_backup::tools;
use serde_json::{Value};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::fs::OpenOptions;
use std::ffi::OsStr;
use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::io::AsRawFd;
use std::collections::HashSet;
use proxmox_backup::pxar;
fn dump_archive_from_reader<R: std::io::Read>(
reader: &mut R,
feature_flags: u64,
verbose: bool,
) -> Result<(), Error> {
let mut decoder = pxar::SequentialDecoder::new(reader, feature_flags);
let stdout = std::io::stdout();
let mut out = stdout.lock();
let mut path = PathBuf::new();
decoder.dump_entry(&mut path, verbose, &mut out)?;
Ok(())
}
fn dump_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
let archive = tools::required_string_param(&param, "archive")?;
let verbose = param["verbose"].as_bool().unwrap_or(false);
let feature_flags = pxar::flags::DEFAULT;
if archive == "-" {
let stdin = std::io::stdin();
let mut reader = stdin.lock();
dump_archive_from_reader(&mut reader, feature_flags, verbose)?;
} else {
if verbose { println!("PXAR dump: {}", archive); }
let file = std::fs::File::open(archive)?;
let mut reader = std::io::BufReader::new(file);
dump_archive_from_reader(&mut reader, feature_flags, verbose)?;
}
Ok(Value::Null)
}
use proxmox_backup::pxar::{flags, fuse, format_single_line_entry, ENCODER_MAX_ENTRIES};
fn extract_archive_from_reader<R: std::io::Read>(
reader: &mut R,
@ -68,124 +23,283 @@ fn extract_archive_from_reader<R: std::io::Read>(
feature_flags: u64,
allow_existing_dirs: bool,
verbose: bool,
pattern: Option<Vec<pxar::MatchPattern>>
match_list: &[MatchEntry],
) -> Result<(), Error> {
let mut decoder = pxar::SequentialDecoder::new(reader, feature_flags);
decoder.set_callback(move |path| {
if verbose {
println!("{:?}", path);
}
Ok(())
});
decoder.set_allow_existing_dirs(allow_existing_dirs);
let pattern = pattern.unwrap_or_else(Vec::new);
decoder.restore(Path::new(target), &pattern)?;
Ok(())
proxmox_backup::pxar::extract_archive(
pxar::decoder::Decoder::from_std(reader)?,
Path::new(target),
&match_list,
feature_flags,
allow_existing_dirs,
|path| {
if verbose {
println!("{:?}", path);
}
},
)
}
#[api(
input: {
properties: {
archive: {
description: "Archive name.",
},
pattern: {
description: "List of paths or pattern matching files to restore",
type: Array,
items: {
type: String,
description: "Path or pattern matching files to restore.",
},
optional: true,
},
target: {
description: "Target directory",
optional: true,
},
verbose: {
description: "Verbose output.",
optional: true,
default: false,
},
"no-xattrs": {
description: "Ignore extended file attributes.",
optional: true,
default: false,
},
"no-fcaps": {
description: "Ignore file capabilities.",
optional: true,
default: false,
},
"no-acls": {
description: "Ignore access control list entries.",
optional: true,
default: false,
},
"allow-existing-dirs": {
description: "Allows directories to already exist on restore.",
optional: true,
default: false,
},
"files-from": {
description: "File containing match pattern for files to restore.",
optional: true,
},
"no-device-nodes": {
description: "Ignore device nodes.",
optional: true,
default: false,
},
"no-fifos": {
description: "Ignore fifos.",
optional: true,
default: false,
},
"no-sockets": {
description: "Ignore sockets.",
optional: true,
default: false,
},
},
},
)]
/// Extract an archive.
fn extract_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
let archive = tools::required_string_param(&param, "archive")?;
let target = param["target"].as_str().unwrap_or(".");
let verbose = param["verbose"].as_bool().unwrap_or(false);
let no_xattrs = param["no-xattrs"].as_bool().unwrap_or(false);
let no_fcaps = param["no-fcaps"].as_bool().unwrap_or(false);
let no_acls = param["no-acls"].as_bool().unwrap_or(false);
let no_device_nodes = param["no-device-nodes"].as_bool().unwrap_or(false);
let no_fifos = param["no-fifos"].as_bool().unwrap_or(false);
let no_sockets = param["no-sockets"].as_bool().unwrap_or(false);
let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
let files_from = param["files-from"].as_str();
let empty = Vec::new();
let arg_pattern = param["pattern"].as_array().unwrap_or(&empty);
let mut feature_flags = pxar::flags::DEFAULT;
archive: String,
pattern: Option<Vec<String>>,
target: Option<String>,
verbose: bool,
no_xattrs: bool,
no_fcaps: bool,
no_acls: bool,
allow_existing_dirs: bool,
files_from: Option<String>,
no_device_nodes: bool,
no_fifos: bool,
no_sockets: bool,
) -> Result<(), Error> {
let mut feature_flags = flags::DEFAULT;
if no_xattrs {
feature_flags ^= pxar::flags::WITH_XATTRS;
feature_flags ^= flags::WITH_XATTRS;
}
if no_fcaps {
feature_flags ^= pxar::flags::WITH_FCAPS;
feature_flags ^= flags::WITH_FCAPS;
}
if no_acls {
feature_flags ^= pxar::flags::WITH_ACL;
feature_flags ^= flags::WITH_ACL;
}
if no_device_nodes {
feature_flags ^= pxar::flags::WITH_DEVICE_NODES;
feature_flags ^= flags::WITH_DEVICE_NODES;
}
if no_fifos {
feature_flags ^= pxar::flags::WITH_FIFOS;
feature_flags ^= flags::WITH_FIFOS;
}
if no_sockets {
feature_flags ^= pxar::flags::WITH_SOCKETS;
feature_flags ^= flags::WITH_SOCKETS;
}
let mut pattern_list = Vec::new();
if let Some(filename) = files_from {
let dir = nix::dir::Dir::open("./", nix::fcntl::OFlag::O_RDONLY, nix::sys::stat::Mode::empty())?;
if let Some((mut pattern, _, _)) = pxar::MatchPattern::from_file(dir.as_raw_fd(), filename)? {
pattern_list.append(&mut pattern);
let pattern = pattern.unwrap_or_else(Vec::new);
let target = target.as_ref().map_or_else(|| ".", String::as_str);
let mut match_list = Vec::new();
if let Some(filename) = &files_from {
for line in proxmox_backup::tools::file_get_non_comment_lines(filename)? {
let line = line
.map_err(|err| format_err!("error reading {}: {}", filename, err))?;
match_list.push(
MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, MatchType::Include)
.map_err(|err| format_err!("bad pattern in file '{}': {}", filename, err))?,
);
}
}
for s in arg_pattern {
let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
let p = pxar::MatchPattern::from_line(l.as_bytes())?
.ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
pattern_list.push(p);
for entry in pattern {
match_list.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Include)
.map_err(|err| format_err!("error in pattern: {}", err))?,
);
}
let pattern = if pattern_list.is_empty() {
None
} else {
Some(pattern_list)
};
if archive == "-" {
let stdin = std::io::stdin();
let mut reader = stdin.lock();
extract_archive_from_reader(&mut reader, target, feature_flags, allow_existing_dirs, verbose, pattern)?;
extract_archive_from_reader(
&mut reader,
&target,
feature_flags,
allow_existing_dirs,
verbose,
&match_list,
)?;
} else {
if verbose { println!("PXAR extract: {}", archive); }
if verbose {
println!("PXAR extract: {}", archive);
}
let file = std::fs::File::open(archive)?;
let mut reader = std::io::BufReader::new(file);
extract_archive_from_reader(&mut reader, target, feature_flags, allow_existing_dirs, verbose, pattern)?;
extract_archive_from_reader(
&mut reader,
&target,
feature_flags,
allow_existing_dirs,
verbose,
&match_list,
)?;
}
Ok(Value::Null)
Ok(())
}
#[api(
input: {
properties: {
archive: {
description: "Archive name.",
},
source: {
description: "Source directory.",
},
verbose: {
description: "Verbose output.",
optional: true,
default: false,
},
"no-xattrs": {
description: "Ignore extended file attributes.",
optional: true,
default: false,
},
"no-fcaps": {
description: "Ignore file capabilities.",
optional: true,
default: false,
},
"no-acls": {
description: "Ignore access control list entries.",
optional: true,
default: false,
},
"all-file-systems": {
description: "Include mounted sudirs.",
optional: true,
default: false,
},
"no-device-nodes": {
description: "Ignore device nodes.",
optional: true,
default: false,
},
"no-fifos": {
description: "Ignore fifos.",
optional: true,
default: false,
},
"no-sockets": {
description: "Ignore sockets.",
optional: true,
default: false,
},
exclude: {
description: "List of paths or pattern matching files to exclude.",
optional: true,
type: Array,
items: {
description: "Path or pattern matching files to restore",
type: String,
},
},
"entries-max": {
description: "Max number of entries loaded at once into memory",
optional: true,
default: ENCODER_MAX_ENTRIES as isize,
minimum: 0,
maximum: std::isize::MAX,
},
},
},
)]
/// Create a new .pxar archive.
fn create_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
archive: String,
source: String,
verbose: bool,
no_xattrs: bool,
no_fcaps: bool,
no_acls: bool,
all_file_systems: bool,
no_device_nodes: bool,
no_fifos: bool,
no_sockets: bool,
exclude: Option<Vec<String>>,
entries_max: isize,
) -> Result<(), Error> {
let exclude_list = {
let input = exclude.unwrap_or_else(Vec::new);
let mut exclude = Vec::with_capacity(input.len());
for entry in input {
exclude.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
.map_err(|err| format_err!("error in exclude pattern: {}", err))?,
);
}
exclude
};
let archive = tools::required_string_param(&param, "archive")?;
let source = tools::required_string_param(&param, "source")?;
let verbose = param["verbose"].as_bool().unwrap_or(false);
let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
let no_xattrs = param["no-xattrs"].as_bool().unwrap_or(false);
let no_fcaps = param["no-fcaps"].as_bool().unwrap_or(false);
let no_acls = param["no-acls"].as_bool().unwrap_or(false);
let no_device_nodes = param["no-device-nodes"].as_bool().unwrap_or(false);
let no_fifos = param["no-fifos"].as_bool().unwrap_or(false);
let no_sockets = param["no-sockets"].as_bool().unwrap_or(false);
let empty = Vec::new();
let exclude_pattern = param["exclude"].as_array().unwrap_or(&empty);
let entries_max = param["entries-max"].as_u64().unwrap_or(pxar::ENCODER_MAX_ENTRIES as u64);
let devices = if all_file_systems { None } else { Some(HashSet::new()) };
let device_set = if all_file_systems {
None
} else {
Some(HashSet::new())
};
let source = PathBuf::from(source);
let mut dir = nix::dir::Dir::open(
&source, nix::fcntl::OFlag::O_NOFOLLOW, nix::sys::stat::Mode::empty())?;
let dir = nix::dir::Dir::open(
&source,
nix::fcntl::OFlag::O_NOFOLLOW,
nix::sys::stat::Mode::empty(),
)?;
let file = OpenOptions::new()
.create_new(true)
@ -193,332 +307,150 @@ fn create_archive(
.mode(0o640)
.open(archive)?;
let mut writer = std::io::BufWriter::with_capacity(1024*1024, file);
let mut feature_flags = pxar::flags::DEFAULT;
let writer = std::io::BufWriter::with_capacity(1024 * 1024, file);
let mut feature_flags = flags::DEFAULT;
if no_xattrs {
feature_flags ^= pxar::flags::WITH_XATTRS;
feature_flags ^= flags::WITH_XATTRS;
}
if no_fcaps {
feature_flags ^= pxar::flags::WITH_FCAPS;
feature_flags ^= flags::WITH_FCAPS;
}
if no_acls {
feature_flags ^= pxar::flags::WITH_ACL;
feature_flags ^= flags::WITH_ACL;
}
if no_device_nodes {
feature_flags ^= pxar::flags::WITH_DEVICE_NODES;
feature_flags ^= flags::WITH_DEVICE_NODES;
}
if no_fifos {
feature_flags ^= pxar::flags::WITH_FIFOS;
feature_flags ^= flags::WITH_FIFOS;
}
if no_sockets {
feature_flags ^= pxar::flags::WITH_SOCKETS;
feature_flags ^= flags::WITH_SOCKETS;
}
let mut pattern_list = Vec::new();
for s in exclude_pattern {
let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
let p = pxar::MatchPattern::from_line(l.as_bytes())?
.ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
pattern_list.push(p);
}
let catalog = None::<&mut pxar::catalog::DummyCatalogWriter>;
pxar::Encoder::encode(
source,
&mut dir,
&mut writer,
catalog,
devices,
verbose,
false,
let writer = pxar::encoder::sync::StandardWriter::new(writer);
proxmox_backup::pxar::create_archive(
dir,
writer,
exclude_list,
feature_flags,
pattern_list,
device_set,
true,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
entries_max as usize,
None,
)?;
writer.flush()?;
Ok(Value::Null)
Ok(())
}
#[api(
input: {
properties: {
archive: { description: "Archive name." },
mountpoint: { description: "Mountpoint for the file system." },
verbose: {
description: "Verbose output, running in the foreground (for debugging).",
optional: true,
default: false,
},
},
},
)]
/// Mount the archive to the provided mountpoint via FUSE.
fn mount_archive(
param: Value,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
let archive = tools::required_string_param(&param, "archive")?;
let mountpoint = tools::required_string_param(&param, "mountpoint")?;
let verbose = param["verbose"].as_bool().unwrap_or(false);
let no_mt = param["no-mt"].as_bool().unwrap_or(false);
let archive = Path::new(archive);
let mountpoint = Path::new(mountpoint);
async fn mount_archive(
archive: String,
mountpoint: String,
verbose: bool,
) -> Result<(), Error> {
let archive = Path::new(&archive);
let mountpoint = Path::new(&mountpoint);
let options = OsStr::new("ro,default_permissions");
let mut session = pxar::fuse::Session::from_path(&archive, &options, verbose)
.map_err(|err| format_err!("pxar mount failed: {}", err))?;
// Mount the session and deamonize if verbose is not set
session.mount(&mountpoint, !verbose)?;
session.run_loop(!no_mt)?;
Ok(Value::Null)
let session = fuse::Session::mount_path(&archive, &options, verbose, mountpoint)
.await
.map_err(|err| format_err!("pxar mount failed: {}", err))?;
let mut interrupt = signal(SignalKind::interrupt())?;
select! {
res = session.fuse() => res?,
_ = interrupt.recv().fuse() => {
if verbose {
eprintln!("interrupted");
}
}
}
Ok(())
}
#[sortable]
const API_METHOD_CREATE_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&create_archive),
&ObjectSchema::new(
"Create new .pxar archive.",
&sorted!([
(
"archive",
false,
&StringSchema::new("Archive name").schema()
),
(
"source",
false,
&StringSchema::new("Source directory.").schema()
),
(
"verbose",
true,
&BooleanSchema::new("Verbose output.")
.default(false)
.schema()
),
(
"no-xattrs",
true,
&BooleanSchema::new("Ignore extended file attributes.")
.default(false)
.schema()
),
(
"no-fcaps",
true,
&BooleanSchema::new("Ignore file capabilities.")
.default(false)
.schema()
),
(
"no-acls",
true,
&BooleanSchema::new("Ignore access control list entries.")
.default(false)
.schema()
),
(
"all-file-systems",
true,
&BooleanSchema::new("Include mounted sudirs.")
.default(false)
.schema()
),
(
"no-device-nodes",
true,
&BooleanSchema::new("Ignore device nodes.")
.default(false)
.schema()
),
(
"no-fifos",
true,
&BooleanSchema::new("Ignore fifos.")
.default(false)
.schema()
),
(
"no-sockets",
true,
&BooleanSchema::new("Ignore sockets.")
.default(false)
.schema()
),
(
"exclude",
true,
&ArraySchema::new(
"List of paths or pattern matching files to exclude.",
&StringSchema::new("Path or pattern matching files to restore.").schema()
).schema()
),
(
"entries-max",
true,
&IntegerSchema::new("Max number of entries loaded at once into memory")
.default(pxar::ENCODER_MAX_ENTRIES as isize)
.minimum(0)
.maximum(std::isize::MAX)
.schema()
),
]),
)
);
#[api(
input: {
properties: {
archive: {
description: "Archive name.",
},
verbose: {
description: "Verbose output.",
optional: true,
default: false,
},
},
},
)]
/// List the contents of an archive.
fn dump_archive(archive: String, verbose: bool) -> Result<(), Error> {
for entry in pxar::decoder::Decoder::open(archive)? {
let entry = entry?;
#[sortable]
const API_METHOD_EXTRACT_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&extract_archive),
&ObjectSchema::new(
"Extract an archive.",
&sorted!([
(
"archive",
false,
&StringSchema::new("Archive name.").schema()
),
(
"pattern",
true,
&ArraySchema::new(
"List of paths or pattern matching files to restore",
&StringSchema::new("Path or pattern matching files to restore.").schema()
).schema()
),
(
"target",
true,
&StringSchema::new("Target directory.").schema()
),
(
"verbose",
true,
&BooleanSchema::new("Verbose output.")
.default(false)
.schema()
),
(
"no-xattrs",
true,
&BooleanSchema::new("Ignore extended file attributes.")
.default(false)
.schema()
),
(
"no-fcaps",
true,
&BooleanSchema::new("Ignore file capabilities.")
.default(false)
.schema()
),
(
"no-acls",
true,
&BooleanSchema::new("Ignore access control list entries.")
.default(false)
.schema()
),
(
"allow-existing-dirs",
true,
&BooleanSchema::new("Allows directories to already exist on restore.")
.default(false)
.schema()
),
(
"files-from",
true,
&StringSchema::new("Match pattern for files to restore.").schema()
),
(
"no-device-nodes",
true,
&BooleanSchema::new("Ignore device nodes.")
.default(false)
.schema()
),
(
"no-fifos",
true,
&BooleanSchema::new("Ignore fifos.")
.default(false)
.schema()
),
(
"no-sockets",
true,
&BooleanSchema::new("Ignore sockets.")
.default(false)
.schema()
),
]),
)
);
#[sortable]
const API_METHOD_MOUNT_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&mount_archive),
&ObjectSchema::new(
"Mount the archive as filesystem via FUSE.",
&sorted!([
(
"archive",
false,
&StringSchema::new("Archive name.").schema()
),
(
"mountpoint",
false,
&StringSchema::new("Mountpoint for the filesystem root.").schema()
),
(
"verbose",
true,
&BooleanSchema::new("Verbose output, keeps process running in foreground (for debugging).")
.default(false)
.schema()
),
(
"no-mt",
true,
&BooleanSchema::new("Run in single threaded mode (for debugging).")
.default(false)
.schema()
),
]),
)
);
#[sortable]
const API_METHOD_DUMP_ARCHIVE: ApiMethod = ApiMethod::new(
&ApiHandler::Sync(&dump_archive),
&ObjectSchema::new(
"List the contents of an archive.",
&sorted!([
( "archive", false, &StringSchema::new("Archive name.").schema()),
( "verbose", true, &BooleanSchema::new("Verbose output.")
.default(false)
.schema()
),
])
)
);
if verbose {
println!("{}", format_single_line_entry(&entry));
} else {
println!("{:?}", entry.path());
}
}
Ok(())
}
fn main() {
let cmd_def = CliCommandMap::new()
.insert("create", CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
.arg_param(&["archive", "source"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("source", tools::complete_file_name)
.insert(
"create",
CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
.arg_param(&["archive", "source"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("source", tools::complete_file_name),
)
.insert("extract", CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
.arg_param(&["archive", "target"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("target", tools::complete_file_name)
.completion_cb("files-from", tools::complete_file_name)
)
.insert("mount", CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
.arg_param(&["archive", "mountpoint"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("mountpoint", tools::complete_file_name)
.insert(
"extract",
CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
.arg_param(&["archive", "target"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("target", tools::complete_file_name)
.completion_cb("files-from", tools::complete_file_name),
)
.insert("list", CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
.arg_param(&["archive"])
.completion_cb("archive", tools::complete_file_name)
.insert(
"mount",
CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
.arg_param(&["archive", "mountpoint"])
.completion_cb("archive", tools::complete_file_name)
.completion_cb("mountpoint", tools::complete_file_name),
)
.insert(
"list",
CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
.arg_param(&["archive"])
.completion_cb("archive", tools::complete_file_name),
);
let rpcenv = CliEnvironment::new();
run_cli_command(cmd_def, rpcenv, None);
run_cli_command(cmd_def, rpcenv, Some(|future| {
proxmox_backup::tools::runtime::main(future)
}));
}

7
src/client.rs
View File

@ -3,11 +3,11 @@
//! This library implements the client side to access the backups
//! server using https.
pub mod pipe_to_stream;
mod merge_known_chunks;
pub mod pipe_to_stream;
mod http_client;
pub use http_client::*;
pub use http_client::*;
mod task_log;
pub use task_log::*;
@ -24,9 +24,6 @@ pub use remote_chunk_reader::*;
mod pxar_backup_stream;
pub use pxar_backup_stream::*;
mod pxar_decode_writer;
pub use pxar_decode_writer::*;
mod backup_repo;
pub use backup_repo::*;

86
src/client/pxar_backup_stream.rs
View File

@ -9,12 +9,12 @@ use std::thread;
use anyhow::{format_err, Error};
use futures::stream::Stream;
use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
use nix::dir::Dir;
use crate::pxar;
use pathpatterns::MatchEntry;
use crate::backup::CatalogWriter;
/// Stream implementation to encode and upload .pxar archives.
@ -29,7 +29,6 @@ pub struct PxarBackupStream {
}
impl Drop for PxarBackupStream {
fn drop(&mut self) {
self.rx = None;
self.child.take().unwrap().join().unwrap();
@ -37,46 +36,49 @@ impl Drop for PxarBackupStream {
}
impl PxarBackupStream {
pub fn new<W: Write + Send + 'static>(
mut dir: Dir,
path: PathBuf,
dir: Dir,
_path: PathBuf,
device_set: Option<HashSet<u64>>,
verbose: bool,
_verbose: bool,
skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
exclude_pattern: Vec<pxar::MatchPattern>,
exclude_pattern: Vec<MatchEntry>,
entries_max: usize,
) -> Result<Self, Error> {
let (tx, rx) = std::sync::mpsc::sync_channel(10);
let buffer_size = 256*1024;
let buffer_size = 256 * 1024;
let error = Arc::new(Mutex::new(None));
let error2 = error.clone();
let child = std::thread::Builder::new()
.name("PxarBackupStream".to_string())
.spawn({
let error = Arc::clone(&error);
move || {
let mut catalog_guard = catalog.lock().unwrap();
let writer = std::io::BufWriter::with_capacity(
buffer_size,
crate::tools::StdChannelWriter::new(tx),
);
let catalog = catalog.clone();
let child = std::thread::Builder::new().name("PxarBackupStream".to_string()).spawn(move || {
let mut guard = catalog.lock().unwrap();
let mut writer = std::io::BufWriter::with_capacity(buffer_size, crate::tools::StdChannelWriter::new(tx));
if let Err(err) = pxar::Encoder::encode(
path,
&mut dir,
&mut writer,
Some(&mut *guard),
device_set,
verbose,
skip_lost_and_found,
pxar::flags::DEFAULT,
exclude_pattern,
entries_max,
) {
let mut error = error2.lock().unwrap();
*error = Some(err.to_string());
}
})?;
let writer = pxar::encoder::sync::StandardWriter::new(writer);
if let Err(err) = crate::pxar::create_archive(
dir,
writer,
exclude_pattern,
crate::pxar::flags::DEFAULT,
device_set,
skip_lost_and_found,
|_| Ok(()),
entries_max,
Some(&mut *catalog_guard),
) {
let mut error = error.lock().unwrap();
*error = Some(err.to_string());
}
}
})?;
Ok(Self {
rx: Some(rx),
@ -91,23 +93,31 @@ impl PxarBackupStream {
verbose: bool,
skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
exclude_pattern: Vec<pxar::MatchPattern>,
exclude_pattern: Vec<MatchEntry>,
entries_max: usize,
) -> Result<Self, Error> {
let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
let path = std::path::PathBuf::from(dirname);
Self::new(dir, path, device_set, verbose, skip_lost_and_found, catalog, exclude_pattern, entries_max)
Self::new(
dir,
path,
device_set,
verbose,
skip_lost_and_found,
catalog,
exclude_pattern,
entries_max,
)
}
}
impl Stream for PxarBackupStream {
type Item = Result<Vec<u8>, Error>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Option<Self::Item>> {
{ // limit lock scope
{
// limit lock scope
let error = self.error.lock().unwrap();
if let Some(ref msg) = *error {
return Poll::Ready(Some(Err(format_err!("{}", msg))));

70
src/client/pxar_decode_writer.rs
View File

@ -1,70 +0,0 @@
use anyhow::{Error};
use std::thread;
use std::os::unix::io::FromRawFd;
use std::path::{Path, PathBuf};
use std::io::Write;
use crate::pxar;
/// Writer implementation to deccode a .pxar archive (download).
pub struct PxarDecodeWriter {
pipe: Option<std::fs::File>,
child: Option<thread::JoinHandle<()>>,
}
impl Drop for PxarDecodeWriter {
fn drop(&mut self) {
drop(self.pipe.take());
self.child.take().unwrap().join().unwrap();
}
}
impl PxarDecodeWriter {
pub fn new(base: &Path, verbose: bool) -> Result<Self, Error> {
let (rx, tx) = nix::unistd::pipe()?;
let base = PathBuf::from(base);
let child = thread::spawn(move|| {
let mut reader = unsafe { std::fs::File::from_raw_fd(rx) };
let mut decoder = pxar::SequentialDecoder::new(&mut reader, pxar::flags::DEFAULT);
decoder.set_callback(move |path| {
if verbose {
println!("{:?}", path);
}
Ok(())
});
if let Err(err) = decoder.restore(&base, &Vec::new()) {
eprintln!("pxar decode failed - {}", err);
}
});
let pipe = unsafe { std::fs::File::from_raw_fd(tx) };
Ok(Self { pipe: Some(pipe), child: Some(child) })
}
}
impl Write for PxarDecodeWriter {
fn write(&mut self, buffer: &[u8]) -> Result<usize, std::io::Error> {
let pipe = match self.pipe {
Some(ref mut pipe) => pipe,
None => unreachable!(),
};
pipe.write(buffer)
}
fn flush(&mut self) -> Result<(), std::io::Error> {
let pipe = match self.pipe {
Some(ref mut pipe) => pipe,
None => unreachable!(),
};
pipe.flush()
}
}

44
src/pxar.rs
View File

@ -47,33 +47,21 @@
//! (user, group, acl, ...) because this is already defined by the
//! linked `ENTRY`.
mod binary_search_tree;
pub use binary_search_tree::*;
pub mod flags;
pub use flags::*;
mod format_definition;
pub use format_definition::*;
mod encoder;
pub use encoder::*;
mod sequential_decoder;
pub use sequential_decoder::*;
mod decoder;
pub use decoder::*;
mod match_pattern;
pub use match_pattern::*;
mod dir_stack;
pub use dir_stack::*;
pub mod fuse;
pub use fuse::*;
pub mod catalog;
pub(crate) mod create;
pub(crate) mod dir_stack;
pub(crate) mod extract;
pub(crate) mod metadata;
pub mod flags;
pub mod fuse;
pub(crate) mod tools;
mod helper;
pub use create::create_archive;
pub use extract::extract_archive;
/// The format requires to build sorted directory lookup tables in
/// memory, so we restrict the number of allowed entries to limit
/// maximum memory usage.
pub const ENCODER_MAX_ENTRIES: usize = 1024 * 1024;
pub use tools::{format_multi_line_entry, format_single_line_entry};

229
src/pxar/binary_search_tree.rs
View File

@ -1,229 +0,0 @@
//! Helpers to generate a binary search tree stored in an array from a
//! sorted array.
//!
//! Specifically, for any given sorted array 'input' permute the
//! array so that the following rule holds:
//!
//! For each array item with index i, the item at 2i+1 is smaller and
//! the item 2i+2 is larger.
//!
//! This structure permits efficient (meaning: O(log(n)) binary
//! searches: start with item i=0 (i.e. the root of the BST), compare
//! the value with the searched item, if smaller proceed at item
//! 2i+1, if larger proceed at item 2i+2, and repeat, until either
//! the item is found, or the indexes grow beyond the array size,
//! which means the entry does not exist.
//!
//! Effectively this implements bisection, but instead of jumping
//! around wildly in the array during a single search we only search
//! with strictly monotonically increasing indexes.
//!
//! Algorithm is from casync (camakebst.c), simplified and optimized
//! for rust. Permutation function originally by L. Bressel, 2017. We
//! pass permutation info to user provided callback, which actually
//! implements the data copy.
//!
//! The Wikipedia Artikel for [Binary
//! Heap](https://en.wikipedia.org/wiki/Binary_heap) gives a short
//! intro howto store binary trees using an array.
use std::cmp::Ordering;
#[allow(clippy::many_single_char_names)]
fn copy_binary_search_tree_inner<F: FnMut(usize, usize)>(
copy_func: &mut F,
// we work on input array input[o..o+n]
n: usize,
o: usize,
e: usize,
i: usize,
) {
let p = 1 << e;
let t = p + (p>>1) - 1;
let m = if n > t {
// |...........p.............t....n........(2p)|
p - 1
} else {
// |...........p.....n.......t.............(2p)|
p - 1 - (t-n)
};
(copy_func)(o+m, i);
if m > 0 {
copy_binary_search_tree_inner(copy_func, m, o, e-1, i*2+1);
}
if (m + 1) < n {
copy_binary_search_tree_inner(copy_func, n-m-1, o+m+1, e-1, i*2+2);
}
}
/// This function calls the provided `copy_func()` with the permutation
/// info.
///
/// ```
/// # use proxmox_backup::pxar::copy_binary_search_tree;
/// copy_binary_search_tree(5, |src, dest| {
/// println!("Copy {} to {}", src, dest);
/// });
/// ```
///
/// This will produce the following output:
///
/// ```no-compile
/// Copy 3 to 0
/// Copy 1 to 1
/// Copy 0 to 3
/// Copy 2 to 4
/// Copy 4 to 2
/// ```
///
/// So this generates the following permutation: `[3,1,4,0,2]`.
pub fn copy_binary_search_tree<F: FnMut(usize, usize)>(
n: usize,
mut copy_func: F,
) {
if n == 0 { return };
let e = (64 - n.leading_zeros() - 1) as usize; // fast log2(n)
copy_binary_search_tree_inner(&mut copy_func, n, 0, e, 0);
}
/// This function searches for the index where the comparison by the provided
/// `compare()` function returns `Ordering::Equal`.
/// The order of the comparison matters (noncommutative) and should be search
/// value compared to value at given index as shown in the examples.
/// The parameter `skip_multiples` defines the number of matches to ignore while
/// searching before returning the index in order to lookup duplicate entries in
/// the tree.
///
/// ```
/// # use proxmox_backup::pxar::{copy_binary_search_tree, search_binary_tree_by};
/// let mut vals = vec![0,1,2,2,2,3,4,5,6,6,7,8,8,8];
///
/// let clone = vals.clone();
/// copy_binary_search_tree(vals.len(), |s, d| {
/// vals[d] = clone[s];
/// });
/// let should_be = vec![5,2,8,1,3,6,8,0,2,2,4,6,7,8];
/// assert_eq!(vals, should_be);
///
/// let find = 8;
/// let skip_multiples = 0;
/// let idx = search_binary_tree_by(0, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
/// assert_eq!(idx, Some(2));
///
/// let find = 8;
/// let skip_multiples = 1;
/// let idx = search_binary_tree_by(2, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
/// assert_eq!(idx, Some(6));
///
/// let find = 8;
/// let skip_multiples = 1;
/// let idx = search_binary_tree_by(6, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
/// assert_eq!(idx, Some(13));
///
/// let find = 5;
/// let skip_multiples = 1;
/// let idx = search_binary_tree_by(0, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
/// assert!(idx.is_none());
///
/// let find = 5;
/// let skip_multiples = 0;
/// // if start index is equal to the array length, `None` is returned.
/// let idx = search_binary_tree_by(vals.len(), vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
/// assert!(idx.is_none());
///
/// let find = 5;
/// let skip_multiples = 0;
/// // if start index is larger than length, `None` is returned.
/// let idx = search_binary_tree_by(vals.len() + 1, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
/// assert!(idx.is_none());
/// ```
pub fn search_binary_tree_by<F: Copy + Fn(usize) -> Ordering>(
start: usize,
size: usize,
skip_multiples: usize,
compare: F
) -> Option<usize> {
if start >= size {
return None;
}
let mut skip = skip_multiples;
let cmp = compare(start);
if cmp == Ordering::Equal {
if skip == 0 {
// Found matching hash and want this one
return Some(start);
}
// Found matching hash, but we should skip the first `skip_multiple`,
// so continue search with reduced skip count.
skip -= 1;
}
if cmp == Ordering::Less || cmp == Ordering::Equal {
let res = search_binary_tree_by(2 * start + 1, size, skip, compare);
if res.is_some() {
return res;
}
}
if cmp == Ordering::Greater || cmp == Ordering::Equal {
let res = search_binary_tree_by(2 * start + 2, size, skip, compare);
if res.is_some() {
return res;
}
}
None
}
#[test]
fn test_binary_search_tree() {
fn run_test(len: usize) -> Vec<usize> {
const MARKER: usize = 0xfffffff;
let mut output = vec![];
for _i in 0..len { output.push(MARKER); }
copy_binary_search_tree(len, |s, d| {
assert!(output[d] == MARKER);
output[d] = s;
});
if len < 32 { println!("GOT:{}:{:?}", len, output); }
for i in 0..len {
assert!(output[i] != MARKER);
}
output
}
assert!(run_test(0).len() == 0);
assert!(run_test(1) == [0]);
assert!(run_test(2) == [1,0]);
assert!(run_test(3) == [1,0,2]);
assert!(run_test(4) == [2,1,3,0]);
assert!(run_test(5) == [3,1,4,0,2]);
assert!(run_test(6) == [3,1,5,0,2,4]);
assert!(run_test(7) == [3,1,5,0,2,4,6]);
assert!(run_test(8) == [4,2,6,1,3,5,7,0]);
assert!(run_test(9) == [5,3,7,1,4,6,8,0,2]);
assert!(run_test(10) == [6,3,8,1,5,7,9,0,2,4]);
assert!(run_test(11) == [7,3,9,1,5,8,10,0,2,4,6]);
assert!(run_test(12) == [7,3,10,1,5,9,11,0,2,4,6,8]);
assert!(run_test(13) == [7,3,11,1,5,9,12,0,2,4,6,8,10]);
assert!(run_test(14) == [7,3,11,1,5,9,13,0,2,4,6,8,10,12]);
assert!(run_test(15) == [7,3,11,1,5,9,13,0,2,4,6,8,10,12,14]);
assert!(run_test(16) == [8,4,12,2,6,10,14,1,3,5,7,9,11,13,15,0]);
assert!(run_test(17) == [9,5,13,3,7,11,15,1,4,6,8,10,12,14,16,0,2]);
for len in 18..1000 {
run_test(len);
}
}

769
src/pxar/create.rs Normal file
View File

@ -0,0 +1,769 @@
use std::collections::{HashSet, HashMap};
use std::convert::TryFrom;
use std::ffi::{CStr, CString, OsStr};
use std::fmt;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
use std::path::{Path, PathBuf};
use anyhow::{bail, format_err, Error};
use nix::dir::Dir;
use nix::errno::Errno;
use nix::fcntl::OFlag;
use nix::sys::stat::{FileStat, Mode};
use pathpatterns::{MatchEntry, MatchList, MatchType, PatternFlag};
use pxar::Metadata;
use pxar::encoder::LinkOffset;
use proxmox::sys::error::SysError;
use proxmox::tools::fd::RawFdNum;
use crate::pxar::catalog::BackupCatalogWriter;
use crate::pxar::flags;
use crate::pxar::tools::assert_relative_path;
use crate::tools::{acl, fs, xattr, Fd};
fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
let mut fs_stat = std::mem::MaybeUninit::uninit();
let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
Errno::result(res)?;
let fs_stat = unsafe { fs_stat.assume_init() };
Ok(fs_stat.f_type)
}
pub fn is_virtual_file_system(magic: i64) -> bool {
use proxmox::sys::linux::magic::*;
match magic {
BINFMTFS_MAGIC |
CGROUP2_SUPER_MAGIC |
CGROUP_SUPER_MAGIC |
CONFIGFS_MAGIC |
DEBUGFS_MAGIC |
DEVPTS_SUPER_MAGIC |
EFIVARFS_MAGIC |
FUSE_CTL_SUPER_MAGIC |
HUGETLBFS_MAGIC |
MQUEUE_MAGIC |
NFSD_MAGIC |
PROC_SUPER_MAGIC |
PSTOREFS_MAGIC |
RPCAUTH_GSSMAGIC |
SECURITYFS_MAGIC |
SELINUX_MAGIC |
SMACK_MAGIC |
SYSFS_MAGIC => true,
_ => false
}
}
#[derive(Debug)]
struct ArchiveError {
path: PathBuf,
error: Error,
}
impl ArchiveError {
fn new(path: PathBuf, error: Error) -> Self {
Self { path, error }
}
}
impl std::error::Error for ArchiveError {}
impl fmt::Display for ArchiveError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "error at {:?}: {}", self.path, self.error)
}
}
#[derive(Eq, PartialEq, Hash)]
struct HardLinkInfo {
st_dev: u64,
st_ino: u64,
}
struct Archiver<'a, 'b> {
/// FIXME: use bitflags!() for feature_flags
feature_flags: u64,
fs_feature_flags: u64,
fs_magic: i64,
excludes: &'a [MatchEntry],
callback: &'a mut dyn FnMut(&Path) -> Result<(), Error>,
catalog: Option<&'b mut dyn BackupCatalogWriter>,
path: PathBuf,
entry_counter: usize,
entry_limit: usize,
current_st_dev: libc::dev_t,
device_set: Option<HashSet<u64>>,
hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
}
type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::SeqWrite>;
pub fn create_archive<T, F>(
source_dir: Dir,
mut writer: T,
mut excludes: Vec<MatchEntry>,
feature_flags: u64,
mut device_set: Option<HashSet<u64>>,
skip_lost_and_found: bool,
mut callback: F,
entry_limit: usize,
catalog: Option<&mut dyn BackupCatalogWriter>,
) -> Result<(), Error>
where
T: pxar::encoder::SeqWrite,
F: FnMut(&Path) -> Result<(), Error>,
{
let fs_magic = detect_fs_type(source_dir.as_raw_fd())?;
if is_virtual_file_system(fs_magic) {
bail!("refusing to backup a virtual file system");
}
let fs_feature_flags = flags::feature_flags_from_magic(fs_magic);
let stat = nix::sys::stat::fstat(source_dir.as_raw_fd())?;
let metadata = get_metadata(
source_dir.as_raw_fd(),
&stat,
feature_flags & fs_feature_flags,
fs_magic,
)
.map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?;
if let Some(ref mut set) = device_set {
set.insert(stat.st_dev);
}
let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite;
let mut encoder = Encoder::new(writer, &metadata)?;
if skip_lost_and_found {
excludes.push(MatchEntry::parse_pattern(
"**/lost+found",
PatternFlag::PATH_NAME,
MatchType::Exclude,
)?);
}
let mut archiver = Archiver {
feature_flags,
fs_feature_flags,
fs_magic,
callback: &mut callback,
excludes: &excludes,
catalog,
path: PathBuf::new(),
entry_counter: 0,
entry_limit,
current_st_dev: stat.st_dev,
device_set,
hardlinks: HashMap::new(),
};
archiver.archive_dir_contents(&mut encoder, source_dir)?;
encoder.finish()?;
Ok(())
}
struct FileListEntry {
name: CString,
path: PathBuf,
stat: FileStat,
}
impl<'a, 'b> Archiver<'a, 'b> {
fn flags(&self) -> u64 {
self.feature_flags & self.fs_feature_flags
}
fn wrap_err(&self, err: Error) -> Error {
if err.downcast_ref::<ArchiveError>().is_some() {
err
} else {
ArchiveError::new(self.path.clone(), err).into()
}
}
fn archive_dir_contents(&mut self, encoder: &mut Encoder, mut dir: Dir) -> Result<(), Error> {
let entry_counter = self.entry_counter;
let file_list = self.generate_directory_file_list(&mut dir)?;
let dir_fd = dir.as_raw_fd();
let old_path = std::mem::take(&mut self.path);
for file_entry in file_list {
(self.callback)(Path::new(OsStr::from_bytes(file_entry.name.to_bytes())))?;
self.path = file_entry.path;
self.add_entry(encoder, dir_fd, &file_entry.name, &file_entry.stat)
.map_err(|err| self.wrap_err(err))?;
}
self.path = old_path;
self.entry_counter = entry_counter;
Ok(())
}
fn generate_directory_file_list(&mut self, dir: &mut Dir) -> Result<Vec<FileListEntry>, Error> {
let dir_fd = dir.as_raw_fd();
let mut file_list = Vec::new();
for file in dir.iter() {
let file = file?;
let file_name = file.file_name().to_owned();
let file_name_bytes = file_name.to_bytes();
if file_name_bytes == b"." || file_name_bytes == b".." {
continue;
}
// FIXME: deal with `.pxarexclude-cli`
if file_name_bytes == b".pxarexclude" {
// FIXME: handle this file!
continue;
}
let os_file_name = OsStr::from_bytes(file_name_bytes);
assert_relative_path(os_file_name)?;
let full_path = self.path.join(os_file_name);
let stat = match nix::sys::stat::fstatat(
dir_fd,
file_name.as_c_str(),
nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
) {
Ok(stat) => stat,
Err(ref err) if err.not_found() => continue,
Err(err) => bail!("stat failed on {:?}: {}", full_path, err),
};
if self
.excludes
.matches(full_path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
== Some(MatchType::Exclude)
{
continue;
}
self.entry_counter += 1;
if self.entry_counter > self.entry_limit {
bail!("exceeded allowed number of file entries (> {})",self.entry_limit);
}
file_list.push(FileListEntry {
name: file_name,
path: full_path,
stat
});
}
file_list.sort_unstable_by(|a, b| a.name.cmp(&b.name));
Ok(file_list)
}
fn add_entry(
&mut self,
encoder: &mut Encoder,
parent: RawFd,
c_file_name: &CStr,
stat: &FileStat,
) -> Result<(), Error> {
use pxar::format::mode;
let file_mode = stat.st_mode & libc::S_IFMT;
let open_mode = if !(file_mode == libc::S_IFREG || file_mode == libc::S_IFDIR) {
OFlag::O_PATH
} else {
OFlag::empty()
};
let fd = Fd::openat(
&unsafe { RawFdNum::from_raw_fd(parent) },
c_file_name,
open_mode | OFlag::O_RDONLY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC | OFlag::O_NOCTTY,
Mode::empty(),
)?;
let metadata = get_metadata(fd.as_raw_fd(), &stat, self.flags(), self.fs_magic)?;
if self
.excludes
.matches(self.path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
== Some(MatchType::Exclude)
{
return Ok(());
}
let file_name: &Path = OsStr::from_bytes(c_file_name.to_bytes()).as_ref();
match metadata.file_type() {
mode::IFREG => {
let link_info = HardLinkInfo {
st_dev: stat.st_dev,
st_ino: stat.st_ino,
};
if stat.st_nlink > 1 {
if let Some((path, offset)) = self.hardlinks.get(&link_info) {
if let Some(ref mut catalog) = self.catalog {
catalog.add_hardlink(c_file_name)?;
}
encoder.add_hardlink(file_name, path, *offset)?;
return Ok(());
}
}
let file_size = stat.st_size as u64;
if let Some(ref mut catalog) = self.catalog {
catalog.add_file(c_file_name, file_size, metadata.stat.mtime)?;
}
let offset: LinkOffset =
self.add_regular_file(encoder, fd, file_name, &metadata, file_size)?;
if stat.st_nlink > 1 {
self.hardlinks.insert(link_info, (self.path.clone(), offset));
}
Ok(())
}
mode::IFDIR => {
let dir = Dir::from_fd(fd.into_raw_fd())?;
self.add_directory(encoder, dir, c_file_name, &metadata, stat)
}
mode::IFSOCK => {
if let Some(ref mut catalog) = self.catalog {
catalog.add_socket(c_file_name)?;
}
Ok(encoder.add_socket(&metadata, file_name)?)
}
mode::IFIFO => {
if let Some(ref mut catalog) = self.catalog {
catalog.add_fifo(c_file_name)?;
}
Ok(encoder.add_fifo(&metadata, file_name)?)
}
mode::IFLNK => {
if let Some(ref mut catalog) = self.catalog {
catalog.add_symlink(c_file_name)?;
}
self.add_symlink(encoder, fd, file_name, &metadata)
}
mode::IFBLK => {
if let Some(ref mut catalog) = self.catalog {
catalog.add_block_device(c_file_name)?;
}
self.add_device(encoder, file_name, &metadata, &stat)
}
mode::IFCHR => {
if let Some(ref mut catalog) = self.catalog {
catalog.add_char_device(c_file_name)?;
}
self.add_device(encoder, file_name, &metadata, &stat)
}
other => bail!(
"encountered unknown file type: 0x{:x} (0o{:o})",
other,
other
),
}
}
fn add_directory(
&mut self,
encoder: &mut Encoder,
dir: Dir,
dir_name: &CStr,
metadata: &Metadata,
stat: &FileStat,
) -> Result<(), Error> {
let dir_name = OsStr::from_bytes(dir_name.to_bytes());
let mut encoder = encoder.create_directory(dir_name, &metadata)?;
let old_fs_magic = self.fs_magic;
let old_fs_feature_flags = self.fs_feature_flags;
let old_st_dev = self.current_st_dev;
let mut skip_contents = false;
if old_st_dev != stat.st_dev {
self.fs_magic = detect_fs_type(dir.as_raw_fd())?;
self.fs_feature_flags = flags::feature_flags_from_magic(self.fs_magic);
self.current_st_dev = stat.st_dev;
if is_virtual_file_system(self.fs_magic) {
skip_contents = true;
} else if let Some(set) = &self.device_set {
skip_contents = !set.contains(&stat.st_dev);
}
}
let result = if skip_contents {
Ok(())
} else {
self.archive_dir_contents(&mut encoder, dir)
};
self.fs_magic = old_fs_magic;
self.fs_feature_flags = old_fs_feature_flags;
self.current_st_dev = old_st_dev;
encoder.finish()?;
result
}
fn add_regular_file(
&mut self,
encoder: &mut Encoder,
fd: Fd,
file_name: &Path,
metadata: &Metadata,
file_size: u64,
) -> Result<LinkOffset, Error> {
let mut file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
let offset = encoder.add_file(metadata, file_name, file_size, &mut file)?;
Ok(offset)
}
fn add_symlink(
&mut self,
encoder: &mut Encoder,
fd: Fd,
file_name: &Path,
metadata: &Metadata,
) -> Result<(), Error> {
let dest = nix::fcntl::readlinkat(fd.as_raw_fd(), &b""[..])?;
encoder.add_symlink(metadata, file_name, dest)?;
Ok(())
}
fn add_device(
&mut self,
encoder: &mut Encoder,
file_name: &Path,
metadata: &Metadata,
stat: &FileStat,
) -> Result<(), Error> {
Ok(encoder.add_device(
metadata,
file_name,
pxar::format::Device::from_dev_t(stat.st_rdev),
)?)
}
}
fn get_metadata(fd: RawFd, stat: &FileStat, flags: u64, fs_magic: i64) -> Result<Metadata, Error> {
// required for some of these
let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
let mtime = u64::try_from(stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec)
.map_err(|_| format_err!("file with negative mtime"))?;
let mut meta = Metadata {
stat: pxar::Stat {
mode: u64::from(stat.st_mode),
flags: 0,
uid: stat.st_uid,
gid: stat.st_gid,
mtime,
},
..Default::default()
};
get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags)?;
get_chattr(&mut meta, fd)?;
get_fat_attr(&mut meta, fd, fs_magic)?;
get_quota_project_id(&mut meta, fd, flags, fs_magic)?;
Ok(meta)
}
fn errno_is_unsupported(errno: Errno) -> bool {
match errno {
Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
_ => false,
}
}
fn get_fcaps(meta: &mut Metadata, fd: RawFd, flags: u64) -> Result<(), Error> {
if 0 == (flags & flags::WITH_FCAPS) {
return Ok(());
}
match xattr::fgetxattr(fd, xattr::xattr_name_fcaps()) {
Ok(data) => {
meta.fcaps = Some(pxar::format::FCaps { data });
Ok(())
}
Err(Errno::ENODATA) => Ok(()),
Err(Errno::EOPNOTSUPP) => Ok(()),
Err(Errno::EBADF) => Ok(()), // symlinks
Err(err) => bail!("failed to read file capabilities: {}", err),
}
}
fn get_xattr_fcaps_acl(
meta: &mut Metadata,
fd: RawFd,
proc_path: &Path,
flags: u64,
) -> Result<(), Error> {
if 0 == (flags & flags::WITH_XATTRS) {
return Ok(());
}
let xattrs = match xattr::flistxattr(fd) {
Ok(names) => names,
Err(Errno::EOPNOTSUPP) => return Ok(()),
Err(Errno::EBADF) => return Ok(()), // symlinks
Err(err) => bail!("failed to read xattrs: {}", err),
};
for attr in &xattrs {
if xattr::is_security_capability(&attr) {
get_fcaps(meta, fd, flags)?;
continue;
}
if xattr::is_acl(&attr) {
get_acl(meta, proc_path, flags)?;
continue;
}
if !xattr::is_valid_xattr_name(&attr) {
continue;
}
match xattr::fgetxattr(fd, attr) {
Ok(data) => meta
.xattrs
.push(pxar::format::XAttr::new(attr.to_bytes(), data)),
Err(Errno::ENODATA) => (), // it got removed while we were iterating...
Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this
Err(Errno::EBADF) => (), // symlinks, shouldn't be able to reach this either
Err(err) => bail!("error reading extended attribute {:?}: {}", attr, err),
}
}
Ok(())
}
fn get_chattr(metadata: &mut Metadata, fd: RawFd) -> Result<(), Error> {
let mut attr: usize = 0;
match unsafe { fs::read_attr_fd(fd, &mut attr) } {
Ok(_) => (),
Err(nix::Error::Sys(errno)) if errno_is_unsupported(errno) => {
return Ok(());
}
Err(err) => bail!("failed to read file attributes: {}", err),
}
metadata.stat.flags |= flags::feature_flags_from_chattr(attr as u32);
Ok(())
}
fn get_fat_attr(metadata: &mut Metadata, fd: RawFd, fs_magic: i64) -> Result<(), Error> {
use proxmox::sys::linux::magic::*;
if fs_magic != MSDOS_SUPER_MAGIC && fs_magic != FUSE_SUPER_MAGIC {
return Ok(());
}
let mut attr: u32 = 0;
match unsafe { fs::read_fat_attr_fd(fd, &mut attr) } {
Ok(_) => (),
Err(nix::Error::Sys(errno)) if errno_is_unsupported(errno) => {
return Ok(());
}
Err(err) => bail!("failed to read fat attributes: {}", err),
}
metadata.stat.flags |= flags::feature_flags_from_fat_attr(attr);
Ok(())
}
/// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
fn get_quota_project_id(
metadata: &mut Metadata,
fd: RawFd,
flags: u64,
magic: i64,
) -> Result<(), Error> {
if !(metadata.is_dir() || metadata.is_regular_file()) {
return Ok(());
}
if 0 == (flags & flags::WITH_QUOTA_PROJID) {
return Ok(());
}
use proxmox::sys::linux::magic::*;
match magic {
EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => (),
_ => return Ok(()),
}
let mut fsxattr = fs::FSXAttr::default();
let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
// On some FUSE filesystems it can happen that ioctl is not supported.
// For these cases projid is set to 0 while the error is ignored.
if let Err(err) = res {
let errno = err
.as_errno()
.ok_or_else(|| format_err!("error while reading quota project id"))?;
if errno_is_unsupported(errno) {
return Ok(());
} else {
bail!("error while reading quota project id ({})", errno);
}
}
let projid = fsxattr.fsx_projid as u64;
if projid != 0 {
metadata.quota_project_id = Some(pxar::format::QuotaProjectId { projid });
}
Ok(())
}
fn get_acl(metadata: &mut Metadata, proc_path: &Path, flags: u64) -> Result<(), Error> {
if 0 == (flags & flags::WITH_ACL) {
return Ok(());
}
if metadata.is_symlink() {
return Ok(());
}
get_acl_do(metadata, proc_path, acl::ACL_TYPE_ACCESS)?;
if metadata.is_dir() {
get_acl_do(metadata, proc_path, acl::ACL_TYPE_DEFAULT)?;
}
Ok(())
}
fn get_acl_do(
metadata: &mut Metadata,
proc_path: &Path,
acl_type: acl::ACLType,
) -> Result<(), Error> {
// In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
// to create a path for acl_get_file(). acl_get_fd() only allows to get
// ACL_TYPE_ACCESS attributes.
let acl = match acl::ACL::get_file(&proc_path, acl_type) {
Ok(acl) => acl,
// Don't bail if underlying endpoint does not support acls
Err(Errno::EOPNOTSUPP) => return Ok(()),
// Don't bail if the endpoint cannot carry acls
Err(Errno::EBADF) => return Ok(()),
// Don't bail if there is no data
Err(Errno::ENODATA) => return Ok(()),
Err(err) => bail!("error while reading ACL - {}", err),
};
process_acl(metadata, acl, acl_type)
}
fn process_acl(
metadata: &mut Metadata,
acl: acl::ACL,
acl_type: acl::ACLType,
) -> Result<(), Error> {
use pxar::format::acl as pxar_acl;
use pxar::format::acl::{Group, GroupObject, Permissions, User};
let mut acl_user = Vec::new();
let mut acl_group = Vec::new();
let mut acl_group_obj = None;
let mut acl_default = None;
let mut user_obj_permissions = None;
let mut group_obj_permissions = None;
let mut other_permissions = None;
let mut mask_permissions = None;
for entry in &mut acl.entries() {
let tag = entry.get_tag_type()?;
let permissions = entry.get_permissions()?;
match tag {
acl::ACL_USER_OBJ => user_obj_permissions = Some(Permissions(permissions)),
acl::ACL_GROUP_OBJ => group_obj_permissions = Some(Permissions(permissions)),
acl::ACL_OTHER => other_permissions = Some(Permissions(permissions)),
acl::ACL_MASK => mask_permissions = Some(Permissions(permissions)),
acl::ACL_USER => {
acl_user.push(User {
uid: entry.get_qualifier()?,
permissions: Permissions(permissions),
});
}
acl::ACL_GROUP => {
acl_group.push(Group {
gid: entry.get_qualifier()?,
permissions: Permissions(permissions),
});
}
_ => bail!("Unexpected ACL tag encountered!"),
}
}
acl_user.sort();
acl_group.sort();
match acl_type {
acl::ACL_TYPE_ACCESS => {
// The mask permissions are mapped to the stat group permissions
// in case that the ACL group permissions were set.
// Only in that case we need to store the group permissions,
// in the other cases they are identical to the stat group permissions.
if let (Some(gop), true) = (group_obj_permissions, mask_permissions.is_some()) {
acl_group_obj = Some(GroupObject { permissions: gop });
}
metadata.acl.users = acl_user;
metadata.acl.groups = acl_group;
}
acl::ACL_TYPE_DEFAULT => {
if user_obj_permissions != None
|| group_obj_permissions != None
|| other_permissions != None
|| mask_permissions != None
{
acl_default = Some(pxar_acl::Default {
// The value is set to UINT64_MAX as placeholder if one
// of the permissions is not set
user_obj_permissions: user_obj_permissions.unwrap_or(Permissions::NO_MASK),
group_obj_permissions: group_obj_permissions.unwrap_or(Permissions::NO_MASK),
other_permissions: other_permissions.unwrap_or(Permissions::NO_MASK),
mask_permissions: mask_permissions.unwrap_or(Permissions::NO_MASK),
});
}
metadata.acl.default_users = acl_user;
metadata.acl.default_groups = acl_group;
}
_ => bail!("Unexpected ACL type encountered"),
}
metadata.acl.group_obj = acl_group_obj;
metadata.acl.default = acl_default;
Ok(())
}

365
src/pxar/decoder.rs
View File

@ -1,365 +0,0 @@
//! *pxar* format decoder for seekable files
//!
//! This module contain the code to decode *pxar* archive files.
use std::convert::TryFrom;
use std::ffi::{OsString, OsStr};
use std::io::{Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use std::os::unix::ffi::OsStrExt;
use anyhow::{bail, format_err, Error};
use libc;
use super::binary_search_tree::search_binary_tree_by;
use super::format_definition::*;
use super::sequential_decoder::SequentialDecoder;
use super::match_pattern::MatchPattern;
use proxmox::tools::io::ReadExt;
pub struct DirectoryEntry {
/// Points to the `PxarEntry` of the directory
start: u64,
/// Points past the goodbye table tail
end: u64,
/// Filename of entry
pub filename: OsString,
/// Entry (mode, permissions)
pub entry: PxarEntry,
/// Extended attributes
pub xattr: PxarAttributes,
/// Payload size
pub size: u64,
/// Target path for symbolic links
pub target: Option<PathBuf>,
/// Start offset of the payload if present.
pub payload_offset: Option<u64>,
}
/// Trait to create ReadSeek Decoder trait objects.
trait ReadSeek: Read + Seek {}
impl <R: Read + Seek> ReadSeek for R {}
// This one needs Read+Seek
pub struct Decoder {
inner: SequentialDecoder<Box<dyn ReadSeek + Send>>,
root_start: u64,
root_end: u64,
}
const HEADER_SIZE: u64 = std::mem::size_of::<PxarHeader>() as u64;
const GOODBYE_ITEM_SIZE: u64 = std::mem::size_of::<PxarGoodbyeItem>() as u64;
impl Decoder {
pub fn new<R: Read + Seek + Send + 'static>(mut reader: R) -> Result<Self, Error> {
let root_end = reader.seek(SeekFrom::End(0))?;
let boxed_reader: Box<dyn ReadSeek + 'static + Send> = Box::new(reader);
let inner = SequentialDecoder::new(boxed_reader, super::flags::DEFAULT);
Ok(Self { inner, root_start: 0, root_end })
}
pub fn set_callback<F: Fn(&Path) -> Result<(), Error> + Send + 'static>(&mut self, callback: F ) {
self.inner.set_callback(callback);
}
pub fn root(&mut self) -> Result<DirectoryEntry, Error> {
self.seek(SeekFrom::Start(0))?;
let header: PxarHeader = self.inner.read_item()?;
check_ca_header::<PxarEntry>(&header, PXAR_ENTRY)?;
let entry: PxarEntry = self.inner.read_item()?;
let (header, xattr) = self.inner.read_attributes()?;
let (size, payload_offset) = match header.htype {
PXAR_PAYLOAD => (header.size - HEADER_SIZE, Some(self.seek(SeekFrom::Current(0))?)),
_ => (0, None),
};
Ok(DirectoryEntry {
start: self.root_start,
end: self.root_end,
filename: OsString::new(), // Empty
entry,
xattr,
size,
target: None,
payload_offset,
})
}
fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
let pos = self.inner.get_reader_mut().seek(pos)?;
Ok(pos)
}
pub(crate) fn root_end_offset(&self) -> u64 {
self.root_end
}
/// Restore the subarchive starting at `dir` to the provided target `path`.
///
/// Only restore the content matched by the MatchPattern `pattern`.
/// An empty Vec `pattern` means restore all.
pub fn restore(&mut self, dir: &DirectoryEntry, path: &Path, pattern: &Vec<MatchPattern>) -> Result<(), Error> {
let start = dir.start;
self.seek(SeekFrom::Start(start))?;
self.inner.restore(path, pattern)?;
Ok(())
}
pub(crate) fn read_directory_entry(
&mut self,
start: u64,
end: u64,
) -> Result<DirectoryEntry, Error> {
self.seek(SeekFrom::Start(start))?;
let head: PxarHeader = self.inner.read_item()?;
if head.htype != PXAR_FILENAME {
bail!("wrong filename header type for object [{}..{}]", start, end);
}
let entry_start = start + head.size;
let filename = self.inner.read_filename(head.size)?;
let head: PxarHeader = self.inner.read_item()?;
if head.htype == PXAR_FORMAT_HARDLINK {
let (_, offset) = self.inner.read_hardlink(head.size)?;
// TODO: Howto find correct end offset for hardlink target?
// This is a bit tricky since we cannot find correct end in an efficient
// way, on the other hand it doesn't really matter (for now) since target
// is never a directory and end is not used in such cases.
return self.read_directory_entry(start - offset, end);
}
check_ca_header::<PxarEntry>(&head, PXAR_ENTRY)?;
let entry: PxarEntry = self.inner.read_item()?;
let (header, xattr) = self.inner.read_attributes()?;
let (size, payload_offset, target) = match header.htype {
PXAR_PAYLOAD =>
(header.size - HEADER_SIZE, Some(self.seek(SeekFrom::Current(0))?), None),
PXAR_SYMLINK =>
(header.size - HEADER_SIZE, None, Some(self.inner.read_link(header.size)?)),
_ => (0, None, None),
};
Ok(DirectoryEntry {
start: entry_start,
end,
filename,
entry,
xattr,
size,
target,
payload_offset,
})
}
/// Return the goodbye table based on the provided end offset.
///
/// Get the goodbye table entries and the start and end offsets of the
/// items they reference.
/// If the start offset is provided, we use that to check the consistency of
/// the data, else the start offset calculated based on the goodbye tail is
/// used.
pub(crate) fn goodbye_table(
&mut self,
start: Option<u64>,
end: u64,
) -> Result<Vec<(PxarGoodbyeItem, u64, u64)>, Error> {
self.seek(SeekFrom::Start(end - GOODBYE_ITEM_SIZE))?;
let tail: PxarGoodbyeItem = self.inner.read_item()?;
if tail.hash != PXAR_GOODBYE_TAIL_MARKER {
bail!("missing goodbye tail marker for object at offset {}", end);
}
// If the start offset was provided, we use and check based on that.
// If not, we rely on the offset calculated from the goodbye table entry.
let start = start.unwrap_or(end - tail.offset - tail.size);
let goodbye_table_size = tail.size;
if goodbye_table_size < (HEADER_SIZE + GOODBYE_ITEM_SIZE) {
bail!("short goodbye table size for object [{}..{}]", start, end);
}
let goodbye_inner_size = goodbye_table_size - HEADER_SIZE - GOODBYE_ITEM_SIZE;
if (goodbye_inner_size % GOODBYE_ITEM_SIZE) != 0 {
bail!(
"wrong goodbye inner table size for entry [{}..{}]",
start,
end
);
}
let goodbye_start = end - goodbye_table_size;
if tail.offset != (goodbye_start - start) {
bail!(
"wrong offset in goodbye tail marker for entry [{}..{}]",
start,
end
);
}
self.seek(SeekFrom::Start(goodbye_start))?;
let head: PxarHeader = self.inner.read_item()?;
if head.htype != PXAR_GOODBYE {
bail!(
"wrong goodbye table header type for entry [{}..{}]",
start,
end
);
}
if head.size != goodbye_table_size {
bail!("wrong goodbye table size for entry [{}..{}]", start, end);
}
let mut gb_entries = Vec::new();
for i in 0..goodbye_inner_size / GOODBYE_ITEM_SIZE {
let item: PxarGoodbyeItem = self.inner.read_item()?;
if item.offset > (goodbye_start - start) {
bail!(
"goodbye entry {} offset out of range [{}..{}] {} {} {}",
i,
start,
end,
item.offset,
goodbye_start,
start
);
}
let item_start = goodbye_start - item.offset;
let item_end = item_start + item.size;
if item_end > goodbye_start {
bail!("goodbye entry {} end out of range [{}..{}]", i, start, end);
}
gb_entries.push((item, item_start, item_end));
}
Ok(gb_entries)
}
pub fn list_dir(&mut self, dir: &DirectoryEntry) -> Result<Vec<DirectoryEntry>, Error> {
let start = dir.start;
let end = dir.end;
//println!("list_dir1: {} {}", start, end);
if (end - start) < (HEADER_SIZE + GOODBYE_ITEM_SIZE) {
bail!("detected short object [{}..{}]", start, end);
}
let mut result = vec![];
let goodbye_table = self.goodbye_table(Some(start), end)?;
for (_, item_start, item_end) in goodbye_table {
let entry = self.read_directory_entry(item_start, item_end)?;
//println!("ENTRY: {} {} {:?}", item_start, item_end, entry.filename);
result.push(entry);
}
Ok(result)
}
pub fn print_filenames<W: std::io::Write>(
&mut self,
output: &mut W,
prefix: &mut PathBuf,
dir: &DirectoryEntry,
) -> Result<(), Error> {
let mut list = self.list_dir(dir)?;
list.sort_unstable_by(|a, b| a.filename.cmp(&b.filename));
for item in &list {
prefix.push(item.filename.clone());
let mode = item.entry.mode as u32;
let ifmt = mode & libc::S_IFMT;
writeln!(output, "{:?}", prefix)?;
match ifmt {
libc::S_IFDIR => self.print_filenames(output, prefix, item)?,
libc::S_IFREG | libc::S_IFLNK | libc::S_IFBLK | libc::S_IFCHR => {}
_ => bail!("unknown item mode/type for {:?}", prefix),
}
prefix.pop();
}
Ok(())
}
/// Lookup the item identified by `filename` in the provided `DirectoryEntry`.
///
/// Calculates the hash of the filename and searches for matching entries in
/// the goodbye table of the provided `DirectoryEntry`.
/// If found, also the filename is compared to avoid hash collision.
/// If the filename does not match, the search resumes with the next entry in
/// the goodbye table.
/// If there is no entry with matching `filename`, `Ok(None)` is returned.
pub fn lookup(
&mut self,
dir: &DirectoryEntry,
filename: &OsStr,
) -> Result<Option<DirectoryEntry>, Error> {
let gbt = self.goodbye_table(Some(dir.start), dir.end)?;
let hash = compute_goodbye_hash(filename.as_bytes());
let mut start_idx = 0;
let mut skip_multiple = 0;
loop {
// Search for the next goodbye entry with matching hash.
let idx = search_binary_tree_by(
start_idx,
gbt.len(),
skip_multiple,
|idx| hash.cmp(&gbt[idx].0.hash),
);
let (_item, start, end) = match idx {
Some(idx) => &gbt[idx],
None => return Ok(None),
};
let entry = self.read_directory_entry(*start, *end)?;
// Possible hash collision, need to check if the found entry is indeed
// the filename to lookup.
if entry.filename == filename {
return Ok(Some(entry));
}
// Hash collision, check the next entry in the goodbye table by starting
// from given index but skipping one more match (so hash at index itself).
start_idx = idx.unwrap();
skip_multiple = 1;
}
}
/// Read the payload of the file given by `entry`.
///
/// This will read a files payload as raw bytes starting from `offset` after
/// the payload marker, reading `size` bytes.
/// If the payload from `offset` to EOF is smaller than `size` bytes, the
/// buffer with reduced size is returned.
/// If `offset` is larger than the payload size of the `DirectoryEntry`, an
/// empty buffer is returned.
pub fn read(&mut self, entry: &DirectoryEntry, size: usize, offset: u64) -> Result<Vec<u8>, Error> {
let start_offset = entry.payload_offset
.ok_or_else(|| format_err!("entry has no payload offset"))?;
if offset >= entry.size {
return Ok(Vec::new());
}
let len = if u64::try_from(size)? > entry.size {
usize::try_from(entry.size)?
} else {
size
};
self.seek(SeekFrom::Start(start_offset + offset))?;
let data = self.inner.get_reader_mut().read_exact_allocated(len)?;
Ok(data)
}
}

173
src/pxar/dir_stack.rs
View File

@ -1,118 +1,141 @@
use std::ffi::{OsStr, OsString};
use std::ffi::OsString;
use std::os::unix::io::{AsRawFd, RawFd};
use std::path::PathBuf;
use anyhow::{format_err, Error};
use nix::errno::Errno;
use anyhow::{bail, format_err, Error};
use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
use nix::NixPath;
use nix::sys::stat::{mkdirat, Mode};
use super::format_definition::{PxarAttributes, PxarEntry};
use proxmox::sys::error::SysError;
use pxar::Metadata;
use crate::pxar::tools::{assert_relative_path, perms_from_metadata};
pub struct PxarDir {
pub filename: OsString,
pub entry: PxarEntry,
pub attr: PxarAttributes,
pub dir: Option<nix::dir::Dir>,
}
pub struct PxarDirStack {
root: RawFd,
data: Vec<PxarDir>,
file_name: OsString,
metadata: Metadata,
dir: Option<Dir>,
}
impl PxarDir {
pub fn new(filename: &OsStr, entry: PxarEntry, attr: PxarAttributes) -> Self {
pub fn new(file_name: OsString, metadata: Metadata) -> Self {
Self {
filename: filename.to_os_string(),
entry,
attr,
file_name,
metadata,
dir: None,
}
}
fn create_dir(&self, parent: RawFd, create_new: bool) -> Result<nix::dir::Dir, nix::Error> {
let res = self
.filename
.with_nix_path(|cstr| unsafe { libc::mkdirat(parent, cstr.as_ptr(), libc::S_IRWXU) })?;
pub fn with_dir(dir: Dir, metadata: Metadata) -> Self {
Self {
file_name: OsString::from("."),
metadata,
dir: Some(dir),
}
}
match Errno::result(res) {
Ok(_) => {}
fn create_dir(&mut self, parent: RawFd, allow_existing_dirs: bool) -> Result<RawFd, Error> {
match mkdirat(
parent,
self.file_name.as_os_str(),
perms_from_metadata(&self.metadata)?,
) {
Ok(()) => (),
Err(err) => {
if err == nix::Error::Sys(nix::errno::Errno::EEXIST) {
if create_new {
return Err(err);
}
} else {
return Err(err);
if !(allow_existing_dirs && err.already_exists()) {
return Err(err.into());
}
}
}
let dir = nix::dir::Dir::openat(
self.open_dir(parent)
}
fn open_dir(&mut self, parent: RawFd) -> Result<RawFd, Error> {
let dir = Dir::openat(
parent,
self.filename.as_os_str(),
self.file_name.as_os_str(),
OFlag::O_DIRECTORY,
Mode::empty(),
)?;
Ok(dir)
let fd = dir.as_raw_fd();
self.dir = Some(dir);
Ok(fd)
}
pub fn try_as_raw_fd(&self) -> Option<RawFd> {
self.dir.as_ref().map(AsRawFd::as_raw_fd)
}
pub fn metadata(&self) -> &Metadata {
&self.metadata
}
}
pub struct PxarDirStack {
dirs: Vec<PxarDir>,
path: PathBuf,
created: usize,
}
impl PxarDirStack {
pub fn new(parent: RawFd) -> Self {
pub fn new(root: Dir, metadata: Metadata) -> Self {
Self {
root: parent,
data: Vec::new(),
dirs: vec![PxarDir::with_dir(root, metadata)],
path: PathBuf::from("/"),
created: 1, // the root directory exists
}
}
pub fn push(&mut self, dir: PxarDir) {
self.data.push(dir);
pub fn is_empty(&self) -> bool {
self.dirs.is_empty()
}
pub fn pop(&mut self) -> Option<PxarDir> {
self.data.pop()
pub fn push(&mut self, file_name: OsString, metadata: Metadata) -> Result<(), Error> {
assert_relative_path(&file_name)?;
self.path.push(&file_name);
self.dirs.push(PxarDir::new(file_name, metadata));
Ok(())
}
pub fn as_path_buf(&self) -> PathBuf {
let path: PathBuf = self.data.iter().map(|d| d.filename.clone()).collect();
path
}
pub fn last(&self) -> Option<&PxarDir> {
self.data.last()
}
pub fn last_mut(&mut self) -> Option<&mut PxarDir> {
self.data.last_mut()
}
pub fn last_dir_fd(&self) -> Option<RawFd> {
let last_dir = self.data.last()?;
match &last_dir.dir {
Some(d) => Some(d.as_raw_fd()),
None => None,
}
}
pub fn create_all_dirs(&mut self, create_new: bool) -> Result<RawFd, Error> {
let mut current_fd = self.root;
for d in &mut self.data {
match &d.dir {
Some(dir) => current_fd = dir.as_raw_fd(),
None => {
let dir = d
.create_dir(current_fd, create_new)
.map_err(|err| format_err!("create dir failed - {}", err))?;
current_fd = dir.as_raw_fd();
d.dir = Some(dir);
}
pub fn pop(&mut self) -> Result<Option<PxarDir>, Error> {
let out = self.dirs.pop();
if !self.path.pop() {
if self.path.as_os_str() == "/" {
// we just finished the root directory, make sure this can only happen once:
self.path = PathBuf::new();
} else {
bail!("lost track of path");
}
}
self.created = self.created.min(self.dirs.len());
Ok(out)
}
Ok(current_fd)
pub fn last_dir_fd(&mut self, allow_existing_dirs: bool) -> Result<RawFd, Error> {
// should not be possible given the way we use it:
assert!(!self.dirs.is_empty(), "PxarDirStack underrun");
let mut fd = self.dirs[self.created - 1]
.try_as_raw_fd()
.ok_or_else(|| format_err!("lost track of directory file descriptors"))?;
while self.created < self.dirs.len() {
fd = self.dirs[self.created].create_dir(fd, allow_existing_dirs)?;
self.created += 1;
}
Ok(fd)
}
pub fn root_dir_fd(&self) -> Result<RawFd, Error> {
// should not be possible given the way we use it:
assert!(!self.dirs.is_empty(), "PxarDirStack underrun");
self.dirs[0]
.try_as_raw_fd()
.ok_or_else(|| format_err!("lost track of directory file descriptors"))
}
}

1332
src/pxar/encoder.rs
View File

File diff suppressed because it is too large Load Diff

306
src/pxar/extract.rs Normal file
View File

@ -0,0 +1,306 @@
//! Code for extraction of pxar contents onto the file system.
use std::convert::TryFrom;
use std::ffi::{CStr, CString, OsStr};
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::path::Path;
use anyhow::{bail, format_err, Error};
use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
use pathpatterns::{MatchEntry, MatchList, MatchType};
use pxar::format::Device;
use pxar::Metadata;
use proxmox::c_result;
use proxmox::tools::fs::{create_path, CreateOptions};
use crate::pxar::dir_stack::PxarDirStack;
use crate::pxar::flags;
use crate::pxar::metadata;
struct Extractor<'a> {
/// FIXME: use bitflags!() for feature_flags
feature_flags: u64,
allow_existing_dirs: bool,
callback: &'a mut dyn FnMut(&Path),
dir_stack: PxarDirStack,
}
impl<'a> Extractor<'a> {
fn with_flag(&self, flag: u64) -> bool {
flag == (self.feature_flags & flag)
}
}
pub fn extract_archive<T, F>(
mut decoder: pxar::decoder::Decoder<T>,
destination: &Path,
match_list: &[MatchEntry],
feature_flags: u64,
allow_existing_dirs: bool,
mut callback: F,
) -> Result<(), Error>
where
T: pxar::decoder::SeqRead,
F: FnMut(&Path),
{
// we use this to keep track of our directory-traversal
decoder.enable_goodbye_entries(true);
let root = decoder
.next()
.ok_or_else(|| format_err!("found empty pxar archive"))?
.map_err(|err| format_err!("error reading pxar archive: {}", err))?;
if !root.is_dir() {
bail!("pxar archive does not start with a directory entry!");
}
create_path(
&destination,
None,
Some(CreateOptions::new().perm(Mode::from_bits_truncate(0o700))),
)
.map_err(|err| format_err!("error creating directory {:?}: {}", destination, err))?;
let dir = Dir::open(
destination,
OFlag::O_DIRECTORY | OFlag::O_CLOEXEC,
Mode::empty(),
)
.map_err(|err| format_err!("unable to open target directory {:?}: {}", destination, err,))?;
let mut extractor = Extractor {
feature_flags,
allow_existing_dirs,
callback: &mut callback,
dir_stack: PxarDirStack::new(dir, root.metadata().clone()),
};
let mut match_stack = Vec::new();
let mut current_match = true;
while let Some(entry) = decoder.next() {
use pxar::EntryKind;
let entry = entry.map_err(|err| format_err!("error reading pxar archive: {}", err))?;
let file_name_os = entry.file_name();
// safety check: a file entry in an archive must never contain slashes:
if file_name_os.as_bytes().contains(&b'/') {
bail!("archive file entry contains slashes, which is invalid and a security concern");
}
let file_name = CString::new(file_name_os.as_bytes())
.map_err(|_| format_err!("encountered file name with null-bytes"))?;
let metadata = entry.metadata();
let match_result = match_list.matches(
entry.path().as_os_str().as_bytes(),
Some(metadata.file_type() as u32),
);
let did_match = match match_result {
Some(MatchType::Include) => true,
Some(MatchType::Exclude) => false,
None => current_match,
};
match (did_match, entry.kind()) {
(_, EntryKind::Directory) => {
extractor.callback(entry.path());
extractor
.dir_stack
.push(file_name_os.to_owned(), metadata.clone())?;
if current_match && match_result != Some(MatchType::Exclude) {
// We're currently in a positive match and this directory does not match an
// exclude entry, so make sure it is created:
let _ = extractor
.dir_stack
.last_dir_fd(extractor.allow_existing_dirs)
.map_err(|err| {
format_err!("error creating entry {:?}: {}", file_name_os, err)
})?;
}
// We're starting a new directory, push our old matching state and replace it with
// our new one:
match_stack.push(current_match);
current_match = did_match;
Ok(())
}
(_, EntryKind::GoodbyeTable) => {
// go up a directory
let dir = extractor
.dir_stack
.pop()
.map_err(|err| format_err!("unexpected end of directory entry: {}", err))?
.ok_or_else(|| format_err!("broken pxar archive (directory stack underrun)"))?;
// We left a directory, also get back our previous matching state. This is in sync
// with `dir_stack` so this should never be empty except for the final goodbye
// table, in which case we get back to the default of `true`.
current_match = match_stack.pop().unwrap_or(true);
if let Some(fd) = dir.try_as_raw_fd() {
metadata::apply(extractor.feature_flags, dir.metadata(), fd, &file_name)
} else {
Ok(())
}
}
(true, EntryKind::Symlink(link)) => {
extractor.callback(entry.path());
extractor.extract_symlink(&file_name, metadata, link.as_ref())
}
(true, EntryKind::Hardlink(link)) => {
extractor.callback(entry.path());
extractor.extract_hardlink(&file_name, metadata, link.as_os_str())
}
(true, EntryKind::Device(dev)) => {
if extractor.with_flag(flags::WITH_DEVICE_NODES) {
extractor.callback(entry.path());
extractor.extract_device(&file_name, metadata, dev)
} else {
Ok(())
}
}
(true, EntryKind::Fifo) => {
if extractor.with_flag(flags::WITH_FIFOS) {
extractor.callback(entry.path());
extractor.extract_special(&file_name, metadata, 0)
} else {
Ok(())
}
}
(true, EntryKind::Socket) => {
if extractor.with_flag(flags::WITH_SOCKETS) {
extractor.callback(entry.path());
extractor.extract_special(&file_name, metadata, 0)
} else {
Ok(())
}
}
(true, EntryKind::File { size, .. }) => extractor.extract_file(
&file_name,
metadata,
*size,
&mut decoder.contents().ok_or_else(|| {
format_err!("found regular file entry without contents in archive")
})?,
),
(false, _) => Ok(()), // skip this
}
.map_err(|err| format_err!("error at entry {:?}: {}", file_name_os, err))?;
}
if !extractor.dir_stack.is_empty() {
bail!("unexpected eof while decoding pxar archive");
}
Ok(())
}
impl<'a> Extractor<'a> {
fn parent_fd(&mut self) -> Result<RawFd, Error> {
self.dir_stack.last_dir_fd(self.allow_existing_dirs)
}
fn callback(&mut self, path: &Path) {
(self.callback)(path)
}
fn extract_symlink(
&mut self,
file_name: &CStr,
metadata: &Metadata,
link: &OsStr,
) -> Result<(), Error> {
let parent = self.parent_fd()?;
nix::unistd::symlinkat(link, Some(parent), file_name)?;
metadata::apply_at(self.feature_flags, metadata, parent, file_name)
}
fn extract_hardlink(
&mut self,
file_name: &CStr,
_metadata: &Metadata, // for now we don't use this because hardlinks don't need it...
link: &OsStr,
) -> Result<(), Error> {
crate::pxar::tools::assert_relative_path(link)?;
let parent = self.parent_fd()?;
let root = self.dir_stack.root_dir_fd()?;
let target = CString::new(link.as_bytes())?;
nix::unistd::linkat(
Some(root),
target.as_c_str(),
Some(parent),
file_name,
nix::unistd::LinkatFlags::NoSymlinkFollow,
)?;
Ok(())
}
fn extract_device(
&mut self,
file_name: &CStr,
metadata: &Metadata,
device: &Device,
) -> Result<(), Error> {
self.extract_special(file_name, metadata, device.to_dev_t())
}
fn extract_special(
&mut self,
file_name: &CStr,
metadata: &Metadata,
device: libc::dev_t,
) -> Result<(), Error> {
let mode = metadata.stat.mode;
let mode = u32::try_from(mode).map_err(|_| {
format_err!(
"device node's mode contains illegal bits: 0x{:x} (0o{:o})",
mode,
mode,
)
})?;
let parent = self.parent_fd()?;
unsafe { c_result!(libc::mknodat(parent, file_name.as_ptr(), mode, device)) }
.map_err(|err| format_err!("failed to create device node: {}", err))?;
metadata::apply_at(self.feature_flags, metadata, parent, file_name)
}
fn extract_file(
&mut self,
file_name: &CStr,
metadata: &Metadata,
size: u64,
contents: &mut dyn io::Read,
) -> Result<(), Error> {
let parent = self.parent_fd()?;
let mut file = unsafe {
std::fs::File::from_raw_fd(nix::fcntl::openat(
parent,
file_name,
OFlag::O_CREAT | OFlag::O_WRONLY | OFlag::O_CLOEXEC,
Mode::from_bits(0o600).unwrap(),
)?)
};
let extracted = io::copy(&mut *contents, &mut file)?;
if size != extracted {
bail!("extracted {} bytes of a file of {} bytes", extracted, size);
}
metadata::apply(self.feature_flags, metadata, file.as_raw_fd(), file_name)
}
}

2
src/pxar/flags.rs
View File

@ -3,6 +3,8 @@
//! Flags for known supported features for a given filesystem can be derived
//! from the superblocks magic number.
// FIXME: use bitflags!() here!
/// FAT-style 2s time granularity
pub const WITH_2SEC_TIME: u64 = 0x40;
/// Preserve read only flag of files

263
src/pxar/format_definition.rs
View File

@ -1,263 +0,0 @@
//! *pxar* binary format definition
//!
//! Please note the all values are stored in little endian ordering.
//!
//! The Archive contains a list of items. Each item starts with a
//! `PxarHeader`, followed by the item data.
use std::cmp::Ordering;
use endian_trait::Endian;
use anyhow::{bail, Error};
use siphasher::sip::SipHasher24;
/// Header types identifying items stored in the archive
pub const PXAR_ENTRY: u64 = 0x1396fabcea5bbb51;
pub const PXAR_FILENAME: u64 = 0x6dbb6ebcb3161f0b;
pub const PXAR_SYMLINK: u64 = 0x664a6fb6830e0d6c;
pub const PXAR_DEVICE: u64 = 0xac3dace369dfe643;
pub const PXAR_XATTR: u64 = 0xb8157091f80bc486;
pub const PXAR_ACL_USER: u64 = 0x297dc88b2ef12faf;
pub const PXAR_ACL_GROUP: u64 = 0x36f2acb56cb3dd0b;
pub const PXAR_ACL_GROUP_OBJ: u64 = 0x23047110441f38f3;
pub const PXAR_ACL_DEFAULT: u64 = 0xfe3eeda6823c8cd0;
pub const PXAR_ACL_DEFAULT_USER: u64 = 0xbdf03df9bd010a91;
pub const PXAR_ACL_DEFAULT_GROUP: u64 = 0xa0cb1168782d1f51;
pub const PXAR_FCAPS: u64 = 0xf7267db0afed0629;
pub const PXAR_QUOTA_PROJID: u64 = 0x161baf2d8772a72b;
/// Marks item as hardlink
/// compute_goodbye_hash(b"__PROXMOX_FORMAT_HARDLINK__");
pub const PXAR_FORMAT_HARDLINK: u64 = 0x2c5e06f634f65b86;
/// Marks the beginning of the payload (actual content) of regular files
pub const PXAR_PAYLOAD: u64 = 0x8b9e1d93d6dcffc9;
/// Marks item as entry of goodbye table
pub const PXAR_GOODBYE: u64 = 0xdfd35c5e8327c403;
/// The end marker used in the GOODBYE object
pub const PXAR_GOODBYE_TAIL_MARKER: u64 = 0x57446fa533702943;
#[derive(Debug, Endian)]
#[repr(C)]
pub struct PxarHeader {
/// The item type (see `PXAR_` constants).
pub htype: u64,
/// The size of the item, including the size of `PxarHeader`.
pub size: u64,
}
#[derive(Endian)]
#[repr(C)]
pub struct PxarEntry {
pub mode: u64,
pub flags: u64,
pub uid: u32,
pub gid: u32,
pub mtime: u64,
}
#[derive(Endian)]
#[repr(C)]
pub struct PxarDevice {
pub major: u64,
pub minor: u64,
}
#[derive(Endian)]
#[repr(C)]
pub struct PxarGoodbyeItem {
/// SipHash24 of the directory item name. The last GOODBYE item
/// uses the special hash value `PXAR_GOODBYE_TAIL_MARKER`.
pub hash: u64,
/// The offset from the start of the GOODBYE object to the start
/// of the matching directory item (point to a FILENAME). The last
/// GOODBYE item points to the start of the matching ENTRY
/// object.
pub offset: u64,
/// The overall size of the directory item. The last GOODBYE item
/// repeats the size of the GOODBYE item.
pub size: u64,
}
/// Helper function to extract file names from binary archive.
pub fn read_os_string(buffer: &[u8]) -> std::ffi::OsString {
let len = buffer.len();
use std::os::unix::ffi::OsStrExt;
let name = if len > 0 && buffer[len - 1] == 0 {
std::ffi::OsStr::from_bytes(&buffer[0..len - 1])
} else {
std::ffi::OsStr::from_bytes(&buffer)
};
name.into()
}
#[derive(Debug, Eq)]
#[repr(C)]
pub struct PxarXAttr {
pub name: Vec<u8>,
pub value: Vec<u8>,
}
impl Ord for PxarXAttr {
fn cmp(&self, other: &PxarXAttr) -> Ordering {
self.name.cmp(&other.name)
}
}
impl PartialOrd for PxarXAttr {
fn partial_cmp(&self, other: &PxarXAttr) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for PxarXAttr {
fn eq(&self, other: &PxarXAttr) -> bool {
self.name == other.name
}
}
#[derive(Debug)]
#[repr(C)]
pub struct PxarFCaps {
pub data: Vec<u8>,
}
#[derive(Debug, Endian, Eq)]
#[repr(C)]
pub struct PxarACLUser {
pub uid: u64,
pub permissions: u64,
//pub name: Vec<u64>, not impl for now
}
// TODO if also name is impl, sort by uid, then by name and last by permissions
impl Ord for PxarACLUser {
fn cmp(&self, other: &PxarACLUser) -> Ordering {
match self.uid.cmp(&other.uid) {
// uids are equal, entries ordered by permissions
Ordering::Equal => self.permissions.cmp(&other.permissions),
// uids are different, entries ordered by uid
uid_order => uid_order,
}
}
}
impl PartialOrd for PxarACLUser {
fn partial_cmp(&self, other: &PxarACLUser) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for PxarACLUser {
fn eq(&self, other: &PxarACLUser) -> bool {
self.uid == other.uid && self.permissions == other.permissions
}
}
#[derive(Debug, Endian, Eq)]
#[repr(C)]
pub struct PxarACLGroup {
pub gid: u64,
pub permissions: u64,
//pub name: Vec<u64>, not impl for now
}
// TODO if also name is impl, sort by gid, then by name and last by permissions
impl Ord for PxarACLGroup {
fn cmp(&self, other: &PxarACLGroup) -> Ordering {
match self.gid.cmp(&other.gid) {
// gids are equal, entries are ordered by permissions
Ordering::Equal => self.permissions.cmp(&other.permissions),
// gids are different, entries ordered by gid
gid_ordering => gid_ordering,
}
}
}
impl PartialOrd for PxarACLGroup {
fn partial_cmp(&self, other: &PxarACLGroup) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for PxarACLGroup {
fn eq(&self, other: &PxarACLGroup) -> bool {
self.gid == other.gid && self.permissions == other.permissions
}
}
#[derive(Debug, Endian)]
#[repr(C)]
pub struct PxarACLGroupObj {
pub permissions: u64,
}
#[derive(Debug, Endian)]
#[repr(C)]
pub struct PxarACLDefault {
pub user_obj_permissions: u64,
pub group_obj_permissions: u64,
pub other_permissions: u64,
pub mask_permissions: u64,
}
pub(crate) struct PxarACL {
pub users: Vec<PxarACLUser>,
pub groups: Vec<PxarACLGroup>,
pub group_obj: Option<PxarACLGroupObj>,
pub default: Option<PxarACLDefault>,
}
pub const PXAR_ACL_PERMISSION_READ: u64 = 4;
pub const PXAR_ACL_PERMISSION_WRITE: u64 = 2;
pub const PXAR_ACL_PERMISSION_EXECUTE: u64 = 1;
#[derive(Debug, Endian)]
#[repr(C)]
pub struct PxarQuotaProjID {
pub projid: u64,
}
#[derive(Debug, Default)]
pub struct PxarAttributes {
pub xattrs: Vec<PxarXAttr>,
pub fcaps: Option<PxarFCaps>,
pub quota_projid: Option<PxarQuotaProjID>,
pub acl_user: Vec<PxarACLUser>,
pub acl_group: Vec<PxarACLGroup>,
pub acl_group_obj: Option<PxarACLGroupObj>,
pub acl_default: Option<PxarACLDefault>,
pub acl_default_user: Vec<PxarACLUser>,
pub acl_default_group: Vec<PxarACLGroup>,
}
/// Create SipHash values for goodby tables.
//pub fn compute_goodbye_hash(name: &std::ffi::CStr) -> u64 {
pub fn compute_goodbye_hash(name: &[u8]) -> u64 {
use std::hash::Hasher;
let mut hasher = SipHasher24::new_with_keys(0x8574442b0f1d84b3, 0x2736ed30d1c22ec1);
hasher.write(name);
hasher.finish()
}
pub fn check_ca_header<T>(head: &PxarHeader, htype: u64) -> Result<(), Error> {
if head.htype != htype {
bail!(
"got wrong header type ({:016x} != {:016x})",
head.htype,
htype
);
}
if head.size != (std::mem::size_of::<T>() + std::mem::size_of::<PxarHeader>()) as u64 {
bail!("got wrong header size for type {:016x}", htype);
}
Ok(())
}
/// The format requires to build sorted directory lookup tables in
/// memory, so we restrict the number of allowed entries to limit
/// maximum memory usage.
pub const ENCODER_MAX_ENTRIES: usize = 1024 * 1024;

1421
src/pxar/fuse.rs
View File

File diff suppressed because it is too large Load Diff

36
src/pxar/helper.rs
View File

@ -1,36 +0,0 @@
use libc;
use nix::sys::stat::FileStat;
#[inline(always)]
pub fn is_directory(stat: &FileStat) -> bool {
(stat.st_mode & libc::S_IFMT) == libc::S_IFDIR
}
#[inline(always)]
pub fn is_symlink(stat: &FileStat) -> bool {
(stat.st_mode & libc::S_IFMT) == libc::S_IFLNK
}
#[inline(always)]
pub fn is_reg_file(stat: &FileStat) -> bool {
(stat.st_mode & libc::S_IFMT) == libc::S_IFREG
}
#[inline(always)]
pub fn is_block_dev(stat: &FileStat) -> bool {
(stat.st_mode & libc::S_IFMT) == libc::S_IFBLK
}
#[inline(always)]
pub fn is_char_dev(stat: &FileStat) -> bool {
(stat.st_mode & libc::S_IFMT) == libc::S_IFCHR
}
#[inline(always)]
pub fn is_fifo(stat: &FileStat) -> bool {
(stat.st_mode & libc::S_IFMT) == libc::S_IFIFO
}
#[inline(always)]
pub fn is_socket(stat: &FileStat) -> bool {
(stat.st_mode & libc::S_IFMT) == libc::S_IFSOCK
}

514
src/pxar/match_pattern.rs
View File

@ -1,514 +0,0 @@
//! `MatchPattern` defines a match pattern used to match filenames encountered
//! during encoding or decoding of a `pxar` archive.
//! `fnmatch` is used internally to match filenames against the patterns.
//! Shell wildcard pattern can be used to match multiple filenames, see manpage
//! `glob(7)`.
//! `**` is treated special, as it matches multiple directories in a path.
use std::ffi::{CStr, CString};
use std::fs::File;
use std::io::Read;
use std::os::unix::io::{FromRawFd, RawFd};
use anyhow::{bail, Error};
use libc::{c_char, c_int};
use nix::errno::Errno;
use nix::fcntl;
use nix::fcntl::{AtFlags, OFlag};
use nix::sys::stat;
use nix::sys::stat::{FileStat, Mode};
use nix::NixPath;
pub const FNM_NOMATCH: c_int = 1;
extern "C" {
fn fnmatch(pattern: *const c_char, string: *const c_char, flags: c_int) -> c_int;
}
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum MatchType {
None,
Positive,
Negative,
PartialPositive,
PartialNegative,
}
/// `MatchPattern` provides functionality for filename glob pattern matching
/// based on glibc's `fnmatch`.
/// Positive matches return `MatchType::PartialPositive` or `MatchType::Positive`.
/// Patterns starting with `!` are interpreted as negation, meaning they will
/// return `MatchType::PartialNegative` or `MatchType::Negative`.
/// No matches result in `MatchType::None`.
/// # Examples:
/// ```
/// # use std::ffi::CString;
/// # use self::proxmox_backup::pxar::{MatchPattern, MatchType};
/// # fn main() -> Result<(), anyhow::Error> {
/// let filename = CString::new("some.conf")?;
/// let is_dir = false;
///
/// /// Positive match of any file ending in `.conf` in any subdirectory
/// let positive = MatchPattern::from_line(b"**/*.conf")?.unwrap();
/// let m_positive = positive.as_slice().matches_filename(&filename, is_dir)?;
/// assert!(m_positive == MatchType::Positive);
///
/// /// Negative match of filenames starting with `s`
/// let negative = MatchPattern::from_line(b"![s]*")?.unwrap();
/// let m_negative = negative.as_slice().matches_filename(&filename, is_dir)?;
/// assert!(m_negative == MatchType::Negative);
/// # Ok(())
/// # }
/// ```
#[derive(Clone, Eq, PartialOrd)]
pub struct MatchPattern {
pattern: Vec<u8>,
match_positive: bool,
match_dir_only: bool,
}
impl std::cmp::PartialEq for MatchPattern {
fn eq(&self, other: &Self) -> bool {
self.pattern == other.pattern
&& self.match_positive == other.match_positive
&& self.match_dir_only == other.match_dir_only
}
}
impl std::cmp::Ord for MatchPattern {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
(&self.pattern, &self.match_positive, &self.match_dir_only)
.cmp(&(&other.pattern, &other.match_positive, &other.match_dir_only))
}
}
impl MatchPattern {
/// Read a list of `MatchPattern` from file.
/// The file is read line by line (lines terminated by newline character),
/// each line may only contain one pattern.
/// Leading `/` are ignored and lines starting with `#` are interpreted as
/// comments and not included in the resulting list.
/// Patterns ending in `/` will match only directories.
///
/// On success, a list of match pattern is returned as well as the raw file
/// byte buffer together with the files stats.
/// This is done in order to avoid reading the file more than once during
/// encoding of the archive.
pub fn from_file<P: ?Sized + NixPath>(
parent_fd: RawFd,
filename: &P,
) -> Result<Option<(Vec<MatchPattern>, Vec<u8>, FileStat)>, nix::Error> {
let stat = match stat::fstatat(parent_fd, filename, AtFlags::AT_SYMLINK_NOFOLLOW) {
Ok(stat) => stat,
Err(nix::Error::Sys(Errno::ENOENT)) => return Ok(None),
Err(err) => return Err(err),
};
let filefd = fcntl::openat(parent_fd, filename, OFlag::O_NOFOLLOW, Mode::empty())?;
let mut file = unsafe { File::from_raw_fd(filefd) };
let mut content_buffer = Vec::new();
let _bytes = file.read_to_end(&mut content_buffer)
.map_err(|_| Errno::EIO)?;
let mut match_pattern = Vec::new();
for line in content_buffer.split(|&c| c == b'\n') {
if line.is_empty() {
continue;
}
if let Some(pattern) = Self::from_line(line)? {
match_pattern.push(pattern);
}
}
Ok(Some((match_pattern, content_buffer, stat)))
}
/// Interpret a byte buffer as a sinlge line containing a valid
/// `MatchPattern`.
/// Pattern starting with `#` are interpreted as comments, returning `Ok(None)`.
/// Pattern starting with '!' are interpreted as negative match pattern.
/// Pattern with trailing `/` match only against directories.
/// `.` as well as `..` and any pattern containing `\0` are invalid and will
/// result in an error with Errno::EINVAL.
pub fn from_line(line: &[u8]) -> Result<Option<MatchPattern>, nix::Error> {
let mut input = line;
if input.starts_with(b"#") {
return Ok(None);
}
let match_positive = if input.starts_with(b"!") {
// Reduce slice view to exclude "!"
input = &input[1..];
false
} else {
true
};
// Paths ending in / match only directory names (no filenames)
let match_dir_only = if input.ends_with(b"/") {
let len = input.len();
input = &input[..len - 1];
true
} else {
false
};
// Ignore initial slash
if input.starts_with(b"/") {
input = &input[1..];
}
if input.is_empty() || input == b"." || input == b".." || input.contains(&b'\0') {
return Err(nix::Error::Sys(Errno::EINVAL));
}
Ok(Some(MatchPattern {
pattern: input.to_vec(),
match_positive,
match_dir_only,
}))
}
/// Create a `MatchPatternSlice` of the `MatchPattern` to give a view of the
/// `MatchPattern` without copying its content.
pub fn as_slice<'a>(&'a self) -> MatchPatternSlice<'a> {
MatchPatternSlice {
pattern: self.pattern.as_slice(),
match_positive: self.match_positive,
match_dir_only: self.match_dir_only,
}
}
/// Dump the content of the `MatchPattern` to stdout.
/// Intended for debugging purposes only.
pub fn dump(&self) {
match (self.match_positive, self.match_dir_only) {
(true, true) => println!("{:#?}/", self.pattern),
(true, false) => println!("{:#?}", self.pattern),
(false, true) => println!("!{:#?}/", self.pattern),
(false, false) => println!("!{:#?}", self.pattern),
}
}
/// Convert a list of MatchPattern to bytes in order to write them to e.g.
/// a file.
pub fn to_bytes(patterns: &[MatchPattern]) -> Vec<u8> {
let mut slices = Vec::new();
for pattern in patterns {
slices.push(pattern.as_slice());
}
MatchPatternSlice::to_bytes(&slices)
}
/// Invert the match type for this MatchPattern.
pub fn invert(&mut self) {
self.match_positive = !self.match_positive;
}
}
#[derive(Clone)]
pub struct MatchPatternSlice<'a> {
pattern: &'a [u8],
match_positive: bool,
match_dir_only: bool,
}
impl<'a> MatchPatternSlice<'a> {
/// Returns the pattern before the first `/` encountered as `MatchPatternSlice`.
/// If no slash is encountered, the `MatchPatternSlice` will be a copy of the
/// original pattern.
/// ```
/// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
/// # fn main() -> Result<(), anyhow::Error> {
/// let pattern = MatchPattern::from_line(b"some/match/pattern/")?.unwrap();
/// let slice = pattern.as_slice();
/// let front = slice.get_front_pattern();
/// /// ... will be the same as ...
/// let front_pattern = MatchPattern::from_line(b"some")?.unwrap();
/// let front_slice = front_pattern.as_slice();
/// # Ok(())
/// # }
/// ```
pub fn get_front_pattern(&'a self) -> MatchPatternSlice<'a> {
let (front, _) = self.split_at_slash();
MatchPatternSlice {
pattern: front,
match_positive: self.match_positive,
match_dir_only: self.match_dir_only,
}
}
/// Returns the pattern after the first encountered `/` as `MatchPatternSlice`.
/// If no slash is encountered, the `MatchPatternSlice` will be empty.
/// ```
/// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
/// # fn main() -> Result<(), anyhow::Error> {
/// let pattern = MatchPattern::from_line(b"some/match/pattern/")?.unwrap();
/// let slice = pattern.as_slice();
/// let rest = slice.get_rest_pattern();
/// /// ... will be the same as ...
/// let rest_pattern = MatchPattern::from_line(b"match/pattern/")?.unwrap();
/// let rest_slice = rest_pattern.as_slice();
/// # Ok(())
/// # }
/// ```
pub fn get_rest_pattern(&'a self) -> MatchPatternSlice<'a> {
let (_, rest) = self.split_at_slash();
MatchPatternSlice {
pattern: rest,
match_positive: self.match_positive,
match_dir_only: self.match_dir_only,
}
}
/// Splits the `MatchPatternSlice` at the first slash encountered and returns the
/// content before (front pattern) and after the slash (rest pattern),
/// omitting the slash itself.
/// Slices starting with `**/` are an exception to this, as the corresponding
/// `MatchPattern` is intended to match multiple directories.
/// These pattern slices therefore return a `*` as front pattern and the original
/// pattern itself as rest pattern.
fn split_at_slash(&'a self) -> (&'a [u8], &'a [u8]) {
let pattern = if self.pattern.starts_with(b"./") {
&self.pattern[2..]
} else {
self.pattern
};
let (mut front, mut rest) = match pattern.iter().position(|&c| c == b'/') {
Some(ind) => {
let (front, rest) = pattern.split_at(ind);
(front, &rest[1..])
}
None => (pattern, &pattern[0..0]),
};
// '**' is treated such that it maches any directory
if front == b"**" {
front = b"*";
rest = pattern;
}
(front, rest)
}
/// Convert a list of `MatchPatternSlice`s to bytes in order to write them to e.g.
/// a file.
pub fn to_bytes(patterns: &[MatchPatternSlice]) -> Vec<u8> {
let mut buffer = Vec::new();
for pattern in patterns {
if !pattern.match_positive { buffer.push(b'!'); }
buffer.extend_from_slice(&pattern.pattern);
if pattern.match_dir_only { buffer.push(b'/'); }
buffer.push(b'\n');
}
buffer
}
/// Match the given filename against this `MatchPatternSlice`.
/// If the filename matches the pattern completely, `MatchType::Positive` or
/// `MatchType::Negative` is returned, depending if the match pattern is was
/// declared as positive (no `!` prefix) or negative (`!` prefix).
/// If the pattern matched only up to the first slash of the pattern,
/// `MatchType::PartialPositive` or `MatchType::PartialNegatie` is returned.
/// If the pattern was postfixed by a trailing `/` a match is only valid if
/// the parameter `is_dir` equals `true`.
/// No match results in `MatchType::None`.
pub fn matches_filename(&self, filename: &CStr, is_dir: bool) -> Result<MatchType, Error> {
let mut res = MatchType::None;
let (front, _) = self.split_at_slash();
let front = CString::new(front).unwrap();
let fnmatch_res = unsafe {
let front_ptr = front.as_ptr() as *const libc::c_char;
let filename_ptr = filename.as_ptr() as *const libc::c_char;
fnmatch(front_ptr, filename_ptr, 0)
};
if fnmatch_res < 0 {
bail!("error in fnmatch inside of MatchPattern");
}
if fnmatch_res == 0 {
res = if self.match_positive {
MatchType::PartialPositive
} else {
MatchType::PartialNegative
};
}
let full = if self.pattern.starts_with(b"**/") {
CString::new(&self.pattern[3..]).unwrap()
} else {
CString::new(&self.pattern[..]).unwrap()
};
let fnmatch_res = unsafe {
let full_ptr = full.as_ptr() as *const libc::c_char;
let filename_ptr = filename.as_ptr() as *const libc::c_char;
fnmatch(full_ptr, filename_ptr, 0)
};
if fnmatch_res < 0 {
bail!("error in fnmatch inside of MatchPattern");
}
if fnmatch_res == 0 {
res = if self.match_positive {
MatchType::Positive
} else {
MatchType::Negative
};
}
if !is_dir && self.match_dir_only {
res = MatchType::None;
}
if !is_dir && (res == MatchType::PartialPositive || res == MatchType::PartialNegative) {
res = MatchType::None;
}
Ok(res)
}
/// Match the given filename against the set of `MatchPatternSlice`s.
///
/// A positive match is intended to includes the full subtree (unless another
/// negative match excludes entries later).
/// The `MatchType` together with an updated `MatchPatternSlice` list for passing
/// to the matched child is returned.
/// ```
/// # use std::ffi::CString;
/// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
/// # fn main() -> Result<(), anyhow::Error> {
/// let patterns = vec![
/// MatchPattern::from_line(b"some/match/pattern/")?.unwrap(),
/// MatchPattern::from_line(b"to_match/")?.unwrap()
/// ];
/// let mut slices = Vec::new();
/// for pattern in &patterns {
/// slices.push(pattern.as_slice());
/// }
/// let filename = CString::new("some")?;
/// let is_dir = true;
/// let (match_type, child_pattern) = MatchPatternSlice::match_filename_include(
/// &filename,
/// is_dir,
/// &slices
/// )?;
/// assert_eq!(match_type, MatchType::PartialPositive);
/// /// child pattern will be the same as ...
/// let pattern = MatchPattern::from_line(b"match/pattern/")?.unwrap();
/// let slice = pattern.as_slice();
///
/// let filename = CString::new("to_match")?;
/// let is_dir = true;
/// let (match_type, child_pattern) = MatchPatternSlice::match_filename_include(
/// &filename,
/// is_dir,
/// &slices
/// )?;
/// assert_eq!(match_type, MatchType::Positive);
/// /// child pattern will be the same as ...
/// let pattern = MatchPattern::from_line(b"**/*")?.unwrap();
/// let slice = pattern.as_slice();
/// # Ok(())
/// # }
/// ```
pub fn match_filename_include(
filename: &CStr,
is_dir: bool,
match_pattern: &'a [MatchPatternSlice<'a>],
) -> Result<(MatchType, Vec<MatchPatternSlice<'a>>), Error> {
let mut child_pattern = Vec::new();
let mut match_state = MatchType::None;
for pattern in match_pattern {
match pattern.matches_filename(filename, is_dir)? {
MatchType::None => continue,
MatchType::Positive => match_state = MatchType::Positive,
MatchType::Negative => match_state = MatchType::Negative,
MatchType::PartialPositive => {
if match_state != MatchType::Negative && match_state != MatchType::Positive {
match_state = MatchType::PartialPositive;
}
child_pattern.push(pattern.get_rest_pattern());
}
MatchType::PartialNegative => {
if match_state == MatchType::PartialPositive {
match_state = MatchType::PartialNegative;
}
child_pattern.push(pattern.get_rest_pattern());
}
}
}
Ok((match_state, child_pattern))
}
/// Match the given filename against the set of `MatchPatternSlice`s.
///
/// A positive match is intended to exclude the full subtree, independent of
/// matches deeper down the tree.
/// The `MatchType` together with an updated `MatchPattern` list for passing
/// to the matched child is returned.
/// ```
/// # use std::ffi::CString;
/// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
/// # fn main() -> Result<(), anyhow::Error> {
/// let patterns = vec![
/// MatchPattern::from_line(b"some/match/pattern/")?.unwrap(),
/// MatchPattern::from_line(b"to_match/")?.unwrap()
/// ];
/// let mut slices = Vec::new();
/// for pattern in &patterns {
/// slices.push(pattern.as_slice());
/// }
/// let filename = CString::new("some")?;
/// let is_dir = true;
/// let (match_type, child_pattern) = MatchPatternSlice::match_filename_exclude(
/// &filename,
/// is_dir,
/// &slices,
/// )?;
/// assert_eq!(match_type, MatchType::PartialPositive);
/// /// child pattern will be the same as ...
/// let pattern = MatchPattern::from_line(b"match/pattern/")?.unwrap();
/// let slice = pattern.as_slice();
///
/// let filename = CString::new("to_match")?;
/// let is_dir = true;
/// let (match_type, child_pattern) = MatchPatternSlice::match_filename_exclude(
/// &filename,
/// is_dir,
/// &slices,
/// )?;
/// assert_eq!(match_type, MatchType::Positive);
/// /// child pattern will be empty
/// # Ok(())
/// # }
/// ```
pub fn match_filename_exclude(
filename: &CStr,
is_dir: bool,
match_pattern: &'a [MatchPatternSlice<'a>],
) -> Result<(MatchType, Vec<MatchPatternSlice<'a>>), Error> {
let mut child_pattern = Vec::new();
let mut match_state = MatchType::None;
for pattern in match_pattern {
match pattern.matches_filename(filename, is_dir)? {
MatchType::None => {}
MatchType::Positive => match_state = MatchType::Positive,
MatchType::Negative => match_state = MatchType::Negative,
match_type => {
if match_state != MatchType::Positive && match_state != MatchType::Negative {
match_state = match_type;
}
child_pattern.push(pattern.get_rest_pattern());
}
}
}
Ok((match_state, child_pattern))
}
}

342
src/pxar/metadata.rs Normal file
View File

@ -0,0 +1,342 @@
use std::ffi::{CStr, CString};
use std::os::unix::ffi::OsStrExt;
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::path::Path;
use anyhow::{bail, format_err, Error};
use nix::errno::Errno;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
use pxar::Metadata;
use proxmox::sys::error::SysError;
use proxmox::tools::fd::RawFdNum;
use proxmox::{c_result, c_try};
use crate::pxar::flags;
use crate::pxar::tools::perms_from_metadata;
use crate::tools::{acl, fs, xattr};
//
// utility functions
//
fn flags_contain(flags: u64, test_flag: u64) -> bool {
0 != (flags & test_flag)
}
fn allow_notsupp<E: SysError>(err: E) -> Result<(), E> {
if err.is_errno(Errno::EOPNOTSUPP) {
Ok(())
} else {
Err(err)
}
}
fn allow_notsupp_remember<E: SysError>(err: E, not_supp: &mut bool) -> Result<(), E> {
if err.is_errno(Errno::EOPNOTSUPP) {
*not_supp = true;
Ok(())
} else {
Err(err)
}
}
fn nsec_to_update_timespec(mtime_nsec: u64) -> [libc::timespec; 2] {
// restore mtime
const UTIME_OMIT: i64 = (1 << 30) - 2;
const NANOS_PER_SEC: i64 = 1_000_000_000;
let sec = (mtime_nsec as i64) / NANOS_PER_SEC;
let nsec = (mtime_nsec as i64) % NANOS_PER_SEC;
let times: [libc::timespec; 2] = [
libc::timespec {
tv_sec: 0,
tv_nsec: UTIME_OMIT,
},
libc::timespec {
tv_sec: sec,
tv_nsec: nsec,
},
];
times
}
//
// metadata application:
//
pub fn apply_at(
flags: u64,
metadata: &Metadata,
parent: RawFd,
file_name: &CStr,
) -> Result<(), Error> {
let fd = proxmox::tools::fd::Fd::openat(
&unsafe { RawFdNum::from_raw_fd(parent) },
file_name,
OFlag::O_PATH | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
Mode::empty(),
)?;
apply(flags, metadata, fd.as_raw_fd(), file_name)
}
pub fn apply_with_path<T: AsRef<Path>>(
flags: u64,
metadata: &Metadata,
fd: RawFd,
file_name: T,
) -> Result<(), Error> {
apply(
flags,
metadata,
fd,
&CString::new(file_name.as_ref().as_os_str().as_bytes())?,
)
}
pub fn apply(flags: u64, metadata: &Metadata, fd: RawFd, file_name: &CStr) -> Result<(), Error> {
let c_proc_path = CString::new(format!("/proc/self/fd/{}", fd)).unwrap();
let c_proc_path = c_proc_path.as_ptr();
if metadata.stat.flags != 0 {
todo!("apply flags!");
}
unsafe {
// UID and GID first, as this fails if we lose access anyway.
c_result!(libc::chown(
c_proc_path,
metadata.stat.uid,
metadata.stat.gid
))
.map(drop)
.or_else(allow_notsupp)?;
}
let mut skip_xattrs = false;
apply_xattrs(flags, c_proc_path, metadata, &mut skip_xattrs)?;
add_fcaps(flags, c_proc_path, metadata, &mut skip_xattrs)?;
apply_acls(flags, c_proc_path, metadata)?;
apply_quota_project_id(flags, fd, metadata)?;
// Finally mode and time. We may lose access with mode, but the changing the mode also
// affects times.
if !metadata.is_symlink() {
c_result!(unsafe { libc::chmod(c_proc_path, perms_from_metadata(metadata)?.bits()) })
.map(drop)
.or_else(allow_notsupp)?;
}
let res = c_result!(unsafe {
libc::utimensat(
libc::AT_FDCWD,
c_proc_path,
nsec_to_update_timespec(metadata.stat.mtime).as_ptr(),
0,
)
});
match res {
Ok(_) => (),
Err(ref err) if err.is_errno(Errno::EOPNOTSUPP) => (),
Err(ref err) if err.is_errno(Errno::EPERM) => {
println!(
"failed to restore mtime attribute on {:?}: {}",
file_name, err
);
}
Err(err) => return Err(err.into()),
}
Ok(())
}
fn add_fcaps(
flags: u64,
c_proc_path: *const libc::c_char,
metadata: &Metadata,
skip_xattrs: &mut bool,
) -> Result<(), Error> {
if *skip_xattrs || !flags_contain(flags, flags::WITH_FCAPS) {
return Ok(());
}
let fcaps = match metadata.fcaps.as_ref() {
Some(fcaps) => fcaps,
None => return Ok(()),
};
c_result!(unsafe {
libc::setxattr(
c_proc_path,
xattr::xattr_name_fcaps().as_ptr(),
fcaps.data.as_ptr() as *const libc::c_void,
fcaps.data.len(),
0,
)
})
.map(drop)
.or_else(|err| allow_notsupp_remember(err, skip_xattrs))?;
Ok(())
}
fn apply_xattrs(
flags: u64,
c_proc_path: *const libc::c_char,
metadata: &Metadata,
skip_xattrs: &mut bool,
) -> Result<(), Error> {
if *skip_xattrs || !flags_contain(flags, flags::WITH_XATTRS) {
return Ok(());
}
for xattr in &metadata.xattrs {
if *skip_xattrs {
return Ok(());
}
if !xattr::is_valid_xattr_name(xattr.name()) {
println!("skipping invalid xattr named {:?}", xattr.name());
continue;
}
c_result!(unsafe {
libc::setxattr(
c_proc_path,
xattr.name().as_ptr() as *const libc::c_char,
xattr.value().as_ptr() as *const libc::c_void,
xattr.value().len(),
0,
)
})
.map(drop)
.or_else(|err| allow_notsupp_remember(err, &mut *skip_xattrs))?;
}
Ok(())
}
fn apply_acls(
flags: u64,
c_proc_path: *const libc::c_char,
metadata: &Metadata,
) -> Result<(), Error> {
if !flags_contain(flags, flags::WITH_ACL) || metadata.acl.is_empty() {
return Ok(());
}
let mut acl = acl::ACL::init(5)?;
// acl type access:
acl.add_entry_full(
acl::ACL_USER_OBJ,
None,
acl::mode_user_to_acl_permissions(metadata.stat.mode),
)?;
acl.add_entry_full(
acl::ACL_OTHER,
None,
acl::mode_other_to_acl_permissions(metadata.stat.mode),
)?;
match metadata.acl.group_obj.as_ref() {
Some(group_obj) => {
acl.add_entry_full(
acl::ACL_MASK,
None,
acl::mode_group_to_acl_permissions(metadata.stat.mode),
)?;
acl.add_entry_full(acl::ACL_GROUP_OBJ, None, group_obj.permissions.0)?;
}
None => {
acl.add_entry_full(
acl::ACL_GROUP_OBJ,
None,
acl::mode_group_to_acl_permissions(metadata.stat.mode),
)?;
}
}
for user in &metadata.acl.users {
acl.add_entry_full(acl::ACL_USER, Some(user.uid), user.permissions.0)?;
}
for group in &metadata.acl.groups {
acl.add_entry_full(acl::ACL_GROUP, Some(group.gid), group.permissions.0)?;
}
if !acl.is_valid() {
bail!("Error while restoring ACL - ACL invalid");
}
c_try!(unsafe { acl::acl_set_file(c_proc_path, acl::ACL_TYPE_ACCESS, acl.ptr,) });
drop(acl);
// acl type default:
if let Some(default) = metadata.acl.default.as_ref() {
let mut acl = acl::ACL::init(5)?;
acl.add_entry_full(acl::ACL_USER_OBJ, None, default.user_obj_permissions.0)?;
acl.add_entry_full(acl::ACL_GROUP_OBJ, None, default.group_obj_permissions.0)?;
acl.add_entry_full(acl::ACL_OTHER, None, default.other_permissions.0)?;
if default.mask_permissions != pxar::format::acl::Permissions::NO_MASK {
acl.add_entry_full(acl::ACL_MASK, None, default.mask_permissions.0)?;
}
for user in &metadata.acl.default_users {
acl.add_entry_full(acl::ACL_USER, Some(user.uid), user.permissions.0)?;
}
for group in &metadata.acl.default_groups {
acl.add_entry_full(acl::ACL_GROUP, Some(group.gid), group.permissions.0)?;
}
if !acl.is_valid() {
bail!("Error while restoring ACL - ACL invalid");
}
c_try!(unsafe { acl::acl_set_file(c_proc_path, acl::ACL_TYPE_DEFAULT, acl.ptr,) });
}
Ok(())
}
fn apply_quota_project_id(flags: u64, fd: RawFd, metadata: &Metadata) -> Result<(), Error> {
if !flags_contain(flags, flags::WITH_QUOTA_PROJID) {
return Ok(());
}
let projid = match metadata.quota_project_id {
Some(projid) => projid,
None => return Ok(()),
};
let mut fsxattr = fs::FSXAttr::default();
unsafe {
fs::fs_ioc_fsgetxattr(fd, &mut fsxattr).map_err(|err| {
format_err!(
"error while getting fsxattr to restore quota project id - {}",
err
)
})?;
fsxattr.fsx_projid = projid.projid as u32;
fs::fs_ioc_fssetxattr(fd, &fsxattr).map_err(|err| {
format_err!(
"error while setting fsxattr to restore quota project id - {}",
err
)
})?;
}
Ok(())
}

1180
src/pxar/sequential_decoder.rs
View File

File diff suppressed because it is too large Load Diff

192
src/pxar/tools.rs Normal file
View File

@ -0,0 +1,192 @@
//! Some common methods used within the pxar code.
use std::convert::TryFrom;
use std::ffi::OsStr;
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use anyhow::{bail, format_err, Error};
use nix::sys::stat::Mode;
use pxar::{mode, Entry, EntryKind, Metadata};
/// Get the file permissions as `nix::Mode`
pub fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
let mode = meta.stat.get_permission_bits();
u32::try_from(mode)
.map_err(drop)
.and_then(|mode| Mode::from_bits(mode).ok_or(()))
.map_err(|_| format_err!("mode contains illegal bits: 0x{:x} (0o{:o})", mode, mode))
}
/// Make sure path is relative and not '.' or '..'.
pub fn assert_relative_path<S: AsRef<OsStr> + ?Sized>(path: &S) -> Result<(), Error> {
assert_relative_path_do(Path::new(path))
}
fn assert_relative_path_do(path: &Path) -> Result<(), Error> {
if !path.is_relative() {
bail!("bad absolute file name in archive: {:?}", path);
}
let mut components = path.components();
match components.next() {
Some(std::path::Component::Normal(_)) => (),
_ => bail!("invalid path component in archive: {:?}", path),
}
if components.next().is_some() {
bail!(
"invalid path with multiple components in archive: {:?}",
path
);
}
Ok(())
}
#[rustfmt::skip]
fn symbolic_mode(c: u64, special: bool, special_x: u8, special_no_x: u8) -> [u8; 3] {
[
if 0 != c & 4 { b'r' } else { b'-' },
if 0 != c & 2 { b'w' } else { b'-' },
match (c & 1, special) {
(0, false) => b'-',
(0, true) => special_no_x,
(_, false) => b'x',
(_, true) => special_x,
}
]
}
fn mode_string(entry: &Entry) -> String {
// https://www.gnu.org/software/coreutils/manual/html_node/What-information-is-listed.html#What-information-is-listed
// additionally we use:
// file type capital 'L' hard links
// a second '+' after the mode to show non-acl xattr presence
//
// Trwxrwxrwx++ uid/gid size mtime filename [-> destination]
let meta = entry.metadata();
let mode = meta.stat.mode;
let type_char = if entry.is_hardlink() {
'L'
} else {
match mode & mode::IFMT {
mode::IFREG => '-',
mode::IFBLK => 'b',
mode::IFCHR => 'c',
mode::IFDIR => 'd',
mode::IFLNK => 'l',
mode::IFIFO => 'p',
mode::IFSOCK => 's',
_ => '?',
}
};
let fmt_u = symbolic_mode((mode >> 6) & 7, 0 != mode & mode::ISUID, b's', b'S');
let fmt_g = symbolic_mode((mode >> 3) & 7, 0 != mode & mode::ISGID, b's', b'S');
let fmt_o = symbolic_mode((mode >> 3) & 7, 0 != mode & mode::ISVTX, b't', b'T');
let has_acls = if meta.acl.is_empty() { ' ' } else { '+' };
let has_xattrs = if meta.xattrs.is_empty() { ' ' } else { '+' };
format!(
"{}{}{}{}{}{}",
type_char,
unsafe { std::str::from_utf8_unchecked(&fmt_u) },
unsafe { std::str::from_utf8_unchecked(&fmt_g) },
unsafe { std::str::from_utf8_unchecked(&fmt_o) },
has_acls,
has_xattrs,
)
}
pub fn format_single_line_entry(entry: &Entry) -> String {
use chrono::offset::TimeZone;
let mode_string = mode_string(entry);
let meta = entry.metadata();
let mtime = meta.mtime_as_duration();
let mtime = chrono::Local.timestamp(mtime.as_secs() as i64, mtime.subsec_nanos());
let (size, link) = match entry.kind() {
EntryKind::File { size, .. } => (format!("{}", *size), String::new()),
EntryKind::Symlink(link) => ("0".to_string(), format!(" -> {:?}", link.as_os_str())),
EntryKind::Hardlink(link) => ("0".to_string(), format!(" -> {:?}", link.as_os_str())),
EntryKind::Device(dev) => (format!("{},{}", dev.major, dev.minor), String::new()),
_ => ("0".to_string(), String::new()),
};
format!(
"{} {:<13} {} {:>8} {:?}{}",
mode_string,
format!("{}/{}", meta.stat.uid, meta.stat.gid),
mtime.format("%Y-%m-%d %H:%M:%S"),
size,
entry.path(),
link,
)
}
pub fn format_multi_line_entry(entry: &Entry) -> String {
use chrono::offset::TimeZone;
let mode_string = mode_string(entry);
let meta = entry.metadata();
let mtime = meta.mtime_as_duration();
let mtime = chrono::Local.timestamp(mtime.as_secs() as i64, mtime.subsec_nanos());
let (size, link, type_name) = match entry.kind() {
EntryKind::File { size, .. } => (format!("{}", *size), String::new(), "file"),
EntryKind::Symlink(link) => (
"0".to_string(),
format!(" -> {:?}", link.as_os_str()),
"symlink",
),
EntryKind::Hardlink(link) => (
"0".to_string(),
format!(" -> {:?}", link.as_os_str()),
"symlink",
),
EntryKind::Device(dev) => (
format!("{},{}", dev.major, dev.minor),
String::new(),
if meta.stat.is_chardev() {
"characters pecial file"
} else if meta.stat.is_blockdev() {
"block special file"
} else {
"device"
},
),
EntryKind::Socket => ("0".to_string(), String::new(), "socket"),
EntryKind::Fifo => ("0".to_string(), String::new(), "fifo"),
EntryKind::Directory => ("0".to_string(), String::new(), "directory"),
EntryKind::GoodbyeTable => ("0".to_string(), String::new(), "bad entry"),
};
let file_name = match std::str::from_utf8(entry.path().as_os_str().as_bytes()) {
Ok(name) => std::borrow::Cow::Borrowed(name),
Err(_) => std::borrow::Cow::Owned(format!("{:?}", entry.path())),
};
format!(
" File: {}{}\n \
Size: {:<13} Type: {}\n\
Access: ({:o}/{}) Uid: {:<5} Gid: {:<5}\n\
Modify: {}\n",
file_name,
link,
size,
type_name,
meta.file_mode(),
mode_string,
meta.stat.uid,
meta.stat.gid,
mtime.format("%Y-%m-%d %H:%M:%S"),
)
}

6
src/tools/acl.rs
View File

@ -49,7 +49,8 @@ pub const ACL_EA_VERSION: u32 = 0x0002;
#[link(name = "acl")]
extern "C" {
fn acl_get_file(path: *const c_char, acl_type: ACLType) -> *mut c_void;
fn acl_set_file(path: *const c_char, acl_type: ACLType, acl: *mut c_void) -> c_int;
// FIXME: remove 'pub' after the cleanup
pub(crate) fn acl_set_file(path: *const c_char, acl_type: ACLType, acl: *mut c_void) -> c_int;
fn acl_get_fd(fd: RawFd) -> *mut c_void;
fn acl_get_entry(acl: *const c_void, entry_id: c_int, entry: *mut *mut c_void) -> c_int;
fn acl_create_entry(acl: *mut *mut c_void, entry: *mut *mut c_void) -> c_int;
@ -68,7 +69,8 @@ extern "C" {
#[derive(Debug)]
pub struct ACL {
ptr: *mut c_void,
// FIXME: remove 'pub' after the cleanup
pub(crate) ptr: *mut c_void,
}
impl Drop for ACL {

21
src/tools/xattr.rs
View File

@ -16,6 +16,22 @@ pub fn xattr_name_fcaps() -> &'static CStr {
c_str!("security.capability")
}
/// `"system.posix_acl_access"` as a CStr to avoid typos.
///
/// This cannot be `const` until `const_cstr_unchecked` is stable.
#[inline]
pub fn xattr_acl_access() -> &'static CStr {
c_str!("system.posix_acl_access")
}
/// `"system.posix_acl_default"` as a CStr to avoid typos.
///
/// This cannot be `const` until `const_cstr_unchecked` is stable.
#[inline]
pub fn xattr_acl_default() -> &'static CStr {
c_str!("system.posix_acl_default")
}
/// Result of `flistxattr`, allows iterating over the attributes as a list of `&CStr`s.
///
/// Listing xattrs produces a list separated by zeroes, inherently making them available as `&CStr`
@ -139,6 +155,11 @@ pub fn is_security_capability(name: &CStr) -> bool {
name.to_bytes() == xattr_name_fcaps().to_bytes()
}
pub fn is_acl(name: &CStr) -> bool {
name.to_bytes() == xattr_acl_access().to_bytes()
|| name.to_bytes() == xattr_acl_default().to_bytes()
}
/// Check if the passed name buffer starts with a valid xattr namespace prefix
/// and is within the length limit of 255 bytes
pub fn is_valid_xattr_name(c_name: &CStr) -> bool {

23
tests/catar.rs
View File

@ -14,30 +14,27 @@ fn run_test(dir_name: &str) -> Result<(), Error> {
.status()
.expect("failed to execute casync");
let mut writer = std::fs::OpenOptions::new()
let writer = std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open("test-proxmox.catar")?;
let writer = pxar::encoder::sync::StandardWriter::new(writer);
let mut dir = nix::dir::Dir::open(
let dir = nix::dir::Dir::open(
dir_name, nix::fcntl::OFlag::O_NOFOLLOW,
nix::sys::stat::Mode::empty())?;
let path = std::path::PathBuf::from(dir_name);
let catalog = None::<&mut catalog::DummyCatalogWriter>;
Encoder::encode(
path,
&mut dir,
&mut writer,
catalog,
create_archive(
dir,
writer,
Vec::new(),
flags::DEFAULT,
None,
false,
false,
flags::DEFAULT,
Vec::new(),
|_| Ok(()),
ENCODER_MAX_ENTRIES,
None,
)?;
Command::new("cmp")