switch to external pxar and fuse crates

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>

src/api2/backup.rs

@@ -628,9 +628,9 @@ fn dynamic_chunk_index(
 
         let count = index.index_count();
         for pos in 0..count {
-            let (start, end, digest) = index.chunk_info(pos)?;
+            let info = index.chunk_info(pos)?;
-            let size = (end - start) as u32;
+            let size = info.size() as u32;
-            env.register_chunk(digest, size)?;
+            env.register_chunk(info.digest, size)?;
         }
 
         let reader = DigestListEncoder::new(Box::new(index));

src/backup/catalog.rs

@@ -1,18 +1,18 @@
-use anyhow::{bail, format_err, Error};
-use std::fmt;
-use std::ffi::{CStr, CString, OsStr};
-use std::os::unix::ffi::OsStrExt;
-use std::io::{Read, Write, Seek, SeekFrom};
 use std::convert::TryFrom;
+use std::ffi::{CStr, CString, OsStr};
+use std::fmt;
+use std::io::{Read, Write, Seek, SeekFrom};
+use std::os::unix::ffi::OsStrExt;
 
+use anyhow::{bail, format_err, Error};
 use chrono::offset::{TimeZone, Local};
 
-use proxmox::tools::io::ReadExt;
+use pathpatterns::{MatchList, MatchType};
 use proxmox::sys::error::io_err_other;
+use proxmox::tools::io::ReadExt;
 
-use crate::pxar::catalog::BackupCatalogWriter;
-use crate::pxar::{MatchPattern, MatchPatternSlice, MatchType};
 use crate::backup::file_formats::PROXMOX_CATALOG_FILE_MAGIC_1_0;
+use crate::pxar::catalog::BackupCatalogWriter;
 use crate::tools::runtime::block_on;
 
 #[repr(u8)]
@@ -63,7 +63,7 @@ pub struct DirEntry {
 }
 
 /// Used to specific additional attributes inside DirEntry
-#[derive(Clone, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum DirEntryAttribute {
     Directory { start: u64 },
     File { size: u64, mtime: u64 },
@@ -106,6 +106,23 @@ impl DirEntry {
         }
     }
 
+    /// Get file mode bits for this entry to be used with the `MatchList` api.
+    pub fn get_file_mode(&self) -> Option<u32> {
+        Some(
+            match self.attr {
+                DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR,
+                DirEntryAttribute::File { .. } => pxar::mode::IFREG,
+                DirEntryAttribute::Symlink => pxar::mode::IFLNK,
+                DirEntryAttribute::Hardlink => return None,
+                DirEntryAttribute::BlockDevice => pxar::mode::IFBLK,
+                DirEntryAttribute::CharDevice => pxar::mode::IFCHR,
+                DirEntryAttribute::Fifo => pxar::mode::IFIFO,
+                DirEntryAttribute::Socket => pxar::mode::IFSOCK,
+            }
+            as u32
+        )
+    }
+
     /// Check if DirEntry is a directory
     pub fn is_directory(&self) -> bool {
         match self.attr {
@@ -476,7 +493,7 @@ impl <R: Read + Seek> CatalogReader<R> {
         &mut self,
         parent: &DirEntry,
         filename: &[u8],
-    ) -> Result<DirEntry, Error>  {
+    ) -> Result<Option<DirEntry>, Error>  {
 
         let start = match parent.attr {
             DirEntryAttribute::Directory { start } => start,
@@ -496,10 +513,7 @@ impl <R: Read + Seek> CatalogReader<R> {
             Ok(false) // stop parsing
         })?;
 
-        match item {
+        Ok(item)
-            None => bail!("no such file"),
-            Some(entry) => Ok(entry),
-        }
     }
 
     /// Read the raw directory info block from current reader position.
@@ -555,38 +569,30 @@ impl <R: Read + Seek> CatalogReader<R> {
     /// provided callback on them.
     pub fn find(
         &mut self,
-        mut entry: &mut Vec<DirEntry>,
+        parent: &DirEntry,
-        pattern: &[MatchPatternSlice],
+        file_path: &mut Vec<u8>,
-        callback: &Box<fn(&[DirEntry])>,
+        match_list: &impl MatchList, //&[MatchEntry],
+        callback: &mut dyn FnMut(&[u8]) -> Result<(), Error>,
     ) -> Result<(), Error> {
-        let parent = entry.last().unwrap();
+        let file_len = file_path.len();
-        if !parent.is_directory() {
-            return Ok(())
-        }
-
         for e in self.read_dir(parent)?  {
-            match MatchPatternSlice::match_filename_include(
+            let is_dir = e.is_directory();
-                &CString::new(e.name.clone())?,
+            file_path.truncate(file_len);
-                e.is_directory(),
+            if !e.name.starts_with(b"/") {
-                pattern,
+                file_path.reserve(e.name.len() + 1);
-            )? {
+                file_path.push(b'/');
-                (MatchType::Positive, _) => {
-                    entry.push(e);
-                    callback(&entry);
-                    let pattern = MatchPattern::from_line(b"**/*").unwrap().unwrap();
-                    let child_pattern = vec![pattern.as_slice()];
-                    self.find(&mut entry, &child_pattern, callback)?;
-                    entry.pop();
             }
-                (MatchType::PartialPositive, child_pattern)
+            file_path.extend(&e.name);
-                | (MatchType::PartialNegative, child_pattern) => {
+            match match_list.matches(&file_path, e.get_file_mode()) {
-                    entry.push(e);
+                Some(MatchType::Exclude) => continue,
-                    self.find(&mut entry, &child_pattern, callback)?;
+                Some(MatchType::Include) => callback(&file_path)?,
-                    entry.pop();
+                None => (),
             }
-                _ => {}
+            if is_dir {
+                self.find(&e, file_path, match_list, callback)?;
             }
         }
+        file_path.truncate(file_len);
 
         Ok(())
     }

src/backup/dynamic_index.rs

@@ -1,6 +1,7 @@
 use std::convert::TryInto;
 use std::fs::File;
 use std::io::{BufWriter, Seek, SeekFrom, Write};
+use std::ops::Range;
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -13,6 +14,7 @@ use proxmox::tools::vec;
 
 use super::chunk_stat::ChunkStat;
 use super::chunk_store::ChunkStore;
+use super::index::ChunkReadInfo;
 use super::read_chunk::ReadChunk;
 use super::Chunker;
 use super::IndexFile;
@@ -136,7 +138,7 @@ impl DynamicIndexReader {
     }
 
     #[allow(clippy::cast_ptr_alignment)]
-    pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> {
+    pub fn chunk_info(&self, pos: usize) -> Result<ChunkReadInfo, Error> {
         if pos >= self.index_entries {
             bail!("chunk index out of range");
         }
@@ -157,7 +159,10 @@ impl DynamicIndexReader {
             );
         }
 
-        Ok((start, end, unsafe { digest.assume_init() }))
+        Ok(ChunkReadInfo {
+            range: start..end,
+            digest: unsafe { digest.assume_init() },
+        })
     }
 
     #[inline]
@@ -258,6 +263,25 @@ impl IndexFile for DynamicIndexReader {
     }
 }
 
+struct CachedChunk {
+    range: Range<u64>,
+    data: Vec<u8>,
+}
+
+impl CachedChunk {
+    /// Perform sanity checks on the range and data size:
+    pub fn new(range: Range<u64>, data: Vec<u8>) -> Result<Self, Error> {
+        if data.len() as u64 != range.end - range.start {
+            bail!(
+                "read chunk with wrong size ({} != {})",
+                data.len(),
+                range.end - range.start,
+            );
+        }
+        Ok(Self { range, data })
+    }
+}
+
 pub struct BufferedDynamicReader<S> {
     store: S,
     index: DynamicIndexReader,
@@ -266,7 +290,7 @@ pub struct BufferedDynamicReader<S> {
     buffered_chunk_idx: usize,
     buffered_chunk_start: u64,
     read_offset: u64,
-    lru_cache: crate::tools::lru_cache::LruCache<usize, (u64, u64, Vec<u8>)>,
+    lru_cache: crate::tools::lru_cache::LruCache<usize, CachedChunk>,
 }
 
 struct ChunkCacher<'a, S> {
@@ -274,10 +298,12 @@ struct ChunkCacher<'a, S> {
     index: &'a DynamicIndexReader,
 }
 
-impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, (u64, u64, Vec<u8>)> for ChunkCacher<'a, S> {
+impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher<usize, CachedChunk> for ChunkCacher<'a, S> {
-    fn fetch(&mut self, index: usize) -> Result<Option<(u64, u64, Vec<u8>)>, anyhow::Error> {
+    fn fetch(&mut self, index: usize) -> Result<Option<CachedChunk>, Error> {
-        let (start, end, digest) = self.index.chunk_info(index)?;
+        let info = self.index.chunk_info(index)?;
-        self.store.read_chunk(&digest).and_then(|data| Ok(Some((start, end, data))))
+        let range = info.range;
+        let data = self.store.read_chunk(&info.digest)?;
+        CachedChunk::new(range, data).map(Some)
     }
 }
 
@@ -301,7 +327,8 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
     }
 
     fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> {
-        let (start, end, data) = self.lru_cache.access(
+        //let (start, end, data) = self.lru_cache.access(
+        let cached_chunk = self.lru_cache.access(
             idx,
             &mut ChunkCacher {
                 store: &mut self.store,
@@ -309,21 +336,13 @@ impl<S: ReadChunk> BufferedDynamicReader<S> {
             },
         )?.ok_or_else(|| format_err!("chunk not found by cacher"))?;
 
-        if (*end - *start) != data.len() as u64 {
-            bail!(
-                "read chunk with wrong size ({} != {}",
-                (*end - *start),
-                data.len()
-            );
-        }
-
         // fixme: avoid copy
         self.read_buffer.clear();
-        self.read_buffer.extend_from_slice(&data);
+        self.read_buffer.extend_from_slice(&cached_chunk.data);
 
         self.buffered_chunk_idx = idx;
 
-        self.buffered_chunk_start = *start;
+        self.buffered_chunk_start = cached_chunk.range.start;
         //println!("BUFFER {} {}",  self.buffered_chunk_start, end);
         Ok(())
     }

src/backup/index.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::ops::Range;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
@@ -6,6 +7,18 @@ use bytes::{Bytes, BytesMut};
 use anyhow::{format_err, Error};
 use futures::*;
 
+pub struct ChunkReadInfo {
+    pub range: Range<u64>,
+    pub digest: [u8; 32],
+}
+
+impl ChunkReadInfo {
+    #[inline]
+    pub fn size(&self) -> u64 {
+        self.range.end - self.range.start
+    }
+}
+
 /// Trait to get digest list from index files
 ///
 /// To allow easy iteration over all used chunks.

src/bin/proxmox-backup-client.rs

@@ -1,13 +1,25 @@
-use anyhow::{bail, format_err, Error};
-use nix::unistd::{fork, ForkResult, pipe};
-use std::os::unix::io::RawFd;
-use chrono::{Local, DateTime, Utc, TimeZone};
-use std::path::{Path, PathBuf};
 use std::collections::{HashSet, HashMap};
 use std::ffi::OsStr;
-use std::io::{Write, Seek, SeekFrom};
+use std::io::{self, Write, Seek, SeekFrom};
 use std::os::unix::fs::OpenOptionsExt;
+use std::os::unix::io::RawFd;
+use std::path::{Path, PathBuf};
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
 
+use anyhow::{bail, format_err, Error};
+use chrono::{Local, DateTime, Utc, TimeZone};
+use futures::future::FutureExt;
+use futures::select;
+use futures::stream::{StreamExt, TryStreamExt};
+use nix::unistd::{fork, ForkResult, pipe};
+use serde_json::{json, Value};
+use tokio::signal::unix::{signal, SignalKind};
+use tokio::sync::mpsc;
+use xdg::BaseDirectories;
+
+use pathpatterns::{MatchEntry, MatchType, PatternFlag};
 use proxmox::{sortable, identity};
 use proxmox::tools::fs::{file_get_contents, file_get_json, replace_file, CreateOptions, image_size};
 use proxmox::sys::linux::tty;
@@ -20,16 +32,7 @@ use proxmox_backup::tools;
 use proxmox_backup::api2::types::*;
 use proxmox_backup::client::*;
 use proxmox_backup::backup::*;
-use proxmox_backup::pxar::{ self, catalog::* };
+use proxmox_backup::pxar::catalog::*;
-
-use serde_json::{json, Value};
-//use hyper::Body;
-use std::sync::{Arc, Mutex};
-//use regex::Regex;
-use xdg::BaseDirectories;
-
-use futures::*;
-use tokio::sync::mpsc;
 
 const ENV_VAR_PBS_FINGERPRINT: &str = "PBS_FINGERPRINT";
 const ENV_VAR_PBS_PASSWORD: &str = "PBS_PASSWORD";
@@ -243,7 +246,7 @@ async fn backup_directory<P: AsRef<Path>>(
     skip_lost_and_found: bool,
     crypt_config: Option<Arc<CryptConfig>>,
     catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
-    exclude_pattern: Vec<pxar::MatchPattern>,
+    exclude_pattern: Vec<MatchEntry>,
     entries_max: usize,
 ) -> Result<BackupStats, Error> {
 
@@ -769,7 +772,7 @@ fn spawn_catalog_upload(
                type: Integer,
                description: "Max number of entries to hold in memory.",
                optional: true,
-               default: pxar::ENCODER_MAX_ENTRIES as isize,
+               default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize,
            },
            "verbose": {
                type: Boolean,
@@ -812,17 +815,19 @@ async fn create_backup(
 
     let include_dev = param["include-dev"].as_array();
 
-    let entries_max = param["entries-max"].as_u64().unwrap_or(pxar::ENCODER_MAX_ENTRIES as u64);
+    let entries_max = param["entries-max"].as_u64()
+        .unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64);
 
     let empty = Vec::new();
-    let arg_pattern = param["exclude"].as_array().unwrap_or(&empty);
+    let exclude_args = param["exclude"].as_array().unwrap_or(&empty);
 
-    let mut pattern_list = Vec::with_capacity(arg_pattern.len());
+    let mut exclude_list = Vec::with_capacity(exclude_args.len());
-    for s in arg_pattern {
+    for entry in exclude_args {
-        let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
+        let entry = entry.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
-        let p = pxar::MatchPattern::from_line(l.as_bytes())?
+        exclude_list.push(
-            .ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
+            MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
-        pattern_list.push(p);
+                .map_err(|err| format_err!("invalid exclude pattern entry: {}", err))?
+        );
     }
 
     let mut devices = if all_file_systems { None } else { Some(HashSet::new()) };
@@ -966,7 +971,7 @@ async fn create_backup(
                     skip_lost_and_found,
                     crypt_config.clone(),
                     catalog.clone(),
-                    pattern_list.clone(),
+                    exclude_list.clone(),
                     entries_max as usize,
                 ).await?;
                 manifest.add_file(target, stats.size, stats.csum)?;
@@ -1246,18 +1251,19 @@ async fn restore(param: Value) -> Result<Value, Error> {
         let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 
         if let Some(target) = target {
-
+            proxmox_backup::pxar::extract_archive(
-            let feature_flags = pxar::flags::DEFAULT;
+                pxar::decoder::Decoder::from_std(reader)?,
-            let mut decoder = pxar::SequentialDecoder::new(&mut reader, feature_flags);
+                Path::new(target),
-            decoder.set_callback(move |path| {
+                &[],
+                proxmox_backup::pxar::flags::DEFAULT,
+                allow_existing_dirs,
+                |path| {
                     if verbose {
-                    eprintln!("{:?}", path);
+                        println!("{:?}", path);
                     }
-                Ok(())
+                },
-            });
+            )
-            decoder.set_allow_existing_dirs(allow_existing_dirs);
+            .map_err(|err| format_err!("error extracting archive - {}", err))?;
-
-            decoder.restore(Path::new(target), &Vec::new())?;
         } else {
             let mut writer = std::fs::OpenOptions::new()
                 .write(true)
@@ -1966,6 +1972,41 @@ fn mount(
     }
 }
 
+use proxmox_backup::client::RemoteChunkReader;
+/// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better
+/// async use!
+///
+/// Ideally BufferedDynamicReader gets replaced so the LruCache maps to `BroadcastFuture<Chunk>`,
+/// so that we can properly access it from multiple threads simultaneously while not issuing
+/// duplicate simultaneous reads over http.
+struct BufferedDynamicReadAt {
+    inner: Mutex<BufferedDynamicReader<RemoteChunkReader>>,
+}
+
+impl BufferedDynamicReadAt {
+    fn new(inner: BufferedDynamicReader<RemoteChunkReader>) -> Self {
+        Self {
+            inner: Mutex::new(inner),
+        }
+    }
+}
+
+impl pxar::accessor::ReadAt for BufferedDynamicReadAt {
+    fn poll_read_at(
+        self: Pin<&Self>,
+        _cx: &mut Context,
+        buf: &mut [u8],
+        offset: u64,
+    ) -> Poll<io::Result<usize>> {
+        use std::io::Read;
+        tokio::task::block_in_place(move || {
+            let mut reader = self.inner.lock().unwrap();
+            reader.seek(SeekFrom::Start(offset))?;
+            Poll::Ready(Ok(reader.read(buf)?))
+        })
+    }
+}
+
 async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
     let repo = extract_repository_from_value(&param)?;
     let archive_name = tools::required_string_param(&param, "archive-name")?;
@@ -2015,15 +2056,19 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
         let most_used = index.find_most_used_chunks(8);
         let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, most_used);
         let reader = BufferedDynamicReader::new(index, chunk_reader);
-        let decoder = pxar::Decoder::new(reader)?;
+        let archive_size = reader.archive_size();
+        let reader: proxmox_backup::pxar::fuse::Reader =
+            Arc::new(BufferedDynamicReadAt::new(reader));
+        let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
         let options = OsStr::new("ro,default_permissions");
-        let mut session = pxar::fuse::Session::new(decoder, &options, pipe.is_none())
-            .map_err(|err| format_err!("pxar mount failed: {}", err))?;
 
-        // Mount the session but not call fuse deamonize as this will cause
+        let session = proxmox_backup::pxar::fuse::Session::mount(
-        // issues with the runtime after the fork
+            decoder,
-        let deamonize = false;
+            &options,
-        session.mount(&Path::new(target), deamonize)?;
+            false,
+            Path::new(target),
+        )
+        .map_err(|err| format_err!("pxar mount failed: {}", err))?;
 
         if let Some(pipe) = pipe {
             nix::unistd::chdir(Path::new("/")).unwrap();
@@ -2045,8 +2090,13 @@ async fn mount_do(param: Value, pipe: Option<RawFd>) -> Result<Value, Error> {
             nix::unistd::close(pipe).unwrap();
         }
 
-        let multithreaded = true;
+        let mut interrupt = signal(SignalKind::interrupt())?;
-        session.run_loop(multithreaded)?;
+        select! {
+            res = session.fuse() => res?,
+            _ = interrupt.recv().fuse() => {
+                // exit on interrupted
+            }
+        }
     } else {
         bail!("unknown archive file extension (expected .pxar)");
     }
@@ -2129,11 +2179,10 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
     let most_used = index.find_most_used_chunks(8);
     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), most_used);
     let reader = BufferedDynamicReader::new(index, chunk_reader);
-    let mut decoder = pxar::Decoder::new(reader)?;
+    let archive_size = reader.archive_size();
-    decoder.set_callback(|path| {
+    let reader: proxmox_backup::pxar::fuse::Reader =
-        println!("{:?}", path);
+        Arc::new(BufferedDynamicReadAt::new(reader));
-        Ok(())
+    let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?;
-    });
 
     let tmpfile = client.download(CATALOG_NAME, tmpfile).await?;
     let index = DynamicIndexReader::new(tmpfile)
@@ -2161,10 +2210,10 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
         catalog_reader,
         &server_archive_name,
         decoder,
-    )?;
+    ).await?;
 
     println!("Starting interactive shell");
-    state.shell()?;
+    state.shell().await?;
 
     record_repository(&repo);
 

src/bin/pxar.rs

@@ -1,66 +1,21 @@
-extern crate proxmox_backup;
+use std::collections::HashSet;
+use std::ffi::OsStr;
+use std::fs::OpenOptions;
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::{Path, PathBuf};
 
 use anyhow::{format_err, Error};
+use futures::future::FutureExt;
+use futures::select;
+use tokio::signal::unix::{signal, SignalKind};
+
+use pathpatterns::{MatchEntry, MatchType, PatternFlag};
 
-use proxmox::{sortable, identity};
-use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment};
-use proxmox::api::schema::*;
 use proxmox::api::cli::*;
+use proxmox::api::api;
 
 use proxmox_backup::tools;
-
+use proxmox_backup::pxar::{flags, fuse, format_single_line_entry, ENCODER_MAX_ENTRIES};
-use serde_json::{Value};
-
-use std::io::Write;
-use std::path::{Path, PathBuf};
-use std::fs::OpenOptions;
-use std::ffi::OsStr;
-use std::os::unix::fs::OpenOptionsExt;
-use std::os::unix::io::AsRawFd;
-use std::collections::HashSet;
-
-use proxmox_backup::pxar;
-
-fn dump_archive_from_reader<R: std::io::Read>(
-    reader: &mut R,
-    feature_flags: u64,
-    verbose: bool,
-) -> Result<(), Error> {
-    let mut decoder = pxar::SequentialDecoder::new(reader, feature_flags);
-
-    let stdout = std::io::stdout();
-    let mut out = stdout.lock();
-
-    let mut path = PathBuf::new();
-    decoder.dump_entry(&mut path, verbose, &mut out)?;
-
-    Ok(())
-}
-
-fn dump_archive(
-    param: Value,
-    _info: &ApiMethod,
-    _rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
-
-    let archive = tools::required_string_param(&param, "archive")?;
-    let verbose = param["verbose"].as_bool().unwrap_or(false);
-
-    let feature_flags = pxar::flags::DEFAULT;
-
-    if archive == "-" {
-        let stdin = std::io::stdin();
-        let mut reader = stdin.lock();
-        dump_archive_from_reader(&mut reader, feature_flags, verbose)?;
-    } else {
-        if verbose { println!("PXAR dump: {}", archive); }
-        let file = std::fs::File::open(archive)?;
-        let mut reader = std::io::BufReader::new(file);
-        dump_archive_from_reader(&mut reader, feature_flags, verbose)?;
-    }
-
-    Ok(Value::Null)
-}
 
 fn extract_archive_from_reader<R: std::io::Read>(
     reader: &mut R,
@@ -68,124 +23,283 @@ fn extract_archive_from_reader<R: std::io::Read>(
     feature_flags: u64,
     allow_existing_dirs: bool,
     verbose: bool,
-    pattern: Option<Vec<pxar::MatchPattern>>
+    match_list: &[MatchEntry],
 ) -> Result<(), Error> {
-    let mut decoder = pxar::SequentialDecoder::new(reader, feature_flags);
+    proxmox_backup::pxar::extract_archive(
-    decoder.set_callback(move |path| {
+        pxar::decoder::Decoder::from_std(reader)?,
+        Path::new(target),
+        &match_list,
+        feature_flags,
+        allow_existing_dirs,
+        |path| {
             if verbose {
                 println!("{:?}", path);
             }
-        Ok(())
+        },
-    });
+    )
-    decoder.set_allow_existing_dirs(allow_existing_dirs);
-
-    let pattern = pattern.unwrap_or_else(Vec::new);
-    decoder.restore(Path::new(target), &pattern)?;
-
-    Ok(())
 }
 
+#[api(
+    input: {
+        properties: {
+            archive: {
+                description: "Archive name.",
+            },
+            pattern: {
+                description: "List of paths or pattern matching files to restore",
+                type: Array,
+                items: {
+                    type: String,
+                    description: "Path or pattern matching files to restore.",
+                },
+                optional: true,
+            },
+            target: {
+                description: "Target directory",
+                optional: true,
+            },
+            verbose: {
+                description: "Verbose output.",
+                optional: true,
+                default: false,
+            },
+            "no-xattrs": {
+                description: "Ignore extended file attributes.",
+                optional: true,
+                default: false,
+            },
+            "no-fcaps": {
+                description: "Ignore file capabilities.",
+                optional: true,
+                default: false,
+            },
+            "no-acls": {
+                description: "Ignore access control list entries.",
+                optional: true,
+                default: false,
+            },
+            "allow-existing-dirs": {
+                description: "Allows directories to already exist on restore.",
+                optional: true,
+                default: false,
+            },
+            "files-from": {
+                description: "File containing match pattern for files to restore.",
+                optional: true,
+            },
+            "no-device-nodes": {
+                description: "Ignore device nodes.",
+                optional: true,
+                default: false,
+            },
+            "no-fifos": {
+                description: "Ignore fifos.",
+                optional: true,
+                default: false,
+            },
+            "no-sockets": {
+                description: "Ignore sockets.",
+                optional: true,
+                default: false,
+            },
+        },
+    },
+)]
+/// Extract an archive.
 fn extract_archive(
-    param: Value,
+    archive: String,
-    _info: &ApiMethod,
+    pattern: Option<Vec<String>>,
-    _rpcenv: &mut dyn RpcEnvironment,
+    target: Option<String>,
-) -> Result<Value, Error> {
+    verbose: bool,
-
+    no_xattrs: bool,
-    let archive = tools::required_string_param(&param, "archive")?;
+    no_fcaps: bool,
-    let target = param["target"].as_str().unwrap_or(".");
+    no_acls: bool,
-    let verbose = param["verbose"].as_bool().unwrap_or(false);
+    allow_existing_dirs: bool,
-    let no_xattrs = param["no-xattrs"].as_bool().unwrap_or(false);
+    files_from: Option<String>,
-    let no_fcaps = param["no-fcaps"].as_bool().unwrap_or(false);
+    no_device_nodes: bool,
-    let no_acls = param["no-acls"].as_bool().unwrap_or(false);
+    no_fifos: bool,
-    let no_device_nodes = param["no-device-nodes"].as_bool().unwrap_or(false);
+    no_sockets: bool,
-    let no_fifos = param["no-fifos"].as_bool().unwrap_or(false);
+) -> Result<(), Error> {
-    let no_sockets = param["no-sockets"].as_bool().unwrap_or(false);
+    let mut feature_flags = flags::DEFAULT;
-    let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
-    let files_from = param["files-from"].as_str();
-    let empty = Vec::new();
-    let arg_pattern = param["pattern"].as_array().unwrap_or(&empty);
-
-    let mut feature_flags = pxar::flags::DEFAULT;
     if no_xattrs {
-        feature_flags ^= pxar::flags::WITH_XATTRS;
+        feature_flags ^= flags::WITH_XATTRS;
     }
     if no_fcaps {
-        feature_flags ^= pxar::flags::WITH_FCAPS;
+        feature_flags ^= flags::WITH_FCAPS;
     }
     if no_acls {
-        feature_flags ^= pxar::flags::WITH_ACL;
+        feature_flags ^= flags::WITH_ACL;
     }
     if no_device_nodes {
-        feature_flags ^= pxar::flags::WITH_DEVICE_NODES;
+        feature_flags ^= flags::WITH_DEVICE_NODES;
     }
     if no_fifos {
-        feature_flags ^= pxar::flags::WITH_FIFOS;
+        feature_flags ^= flags::WITH_FIFOS;
     }
     if no_sockets {
-        feature_flags ^= pxar::flags::WITH_SOCKETS;
+        feature_flags ^= flags::WITH_SOCKETS;
     }
 
-    let mut pattern_list = Vec::new();
+    let pattern = pattern.unwrap_or_else(Vec::new);
-    if let Some(filename) = files_from {
+    let target = target.as_ref().map_or_else(|| ".", String::as_str);
-        let dir = nix::dir::Dir::open("./", nix::fcntl::OFlag::O_RDONLY, nix::sys::stat::Mode::empty())?;
+
-        if let Some((mut pattern, _, _)) = pxar::MatchPattern::from_file(dir.as_raw_fd(), filename)? {
+    let mut match_list = Vec::new();
-            pattern_list.append(&mut pattern);
+    if let Some(filename) = &files_from {
+        for line in proxmox_backup::tools::file_get_non_comment_lines(filename)? {
+            let line = line
+                .map_err(|err| format_err!("error reading {}: {}", filename, err))?;
+            match_list.push(
+                MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, MatchType::Include)
+                    .map_err(|err| format_err!("bad pattern in file '{}': {}", filename, err))?,
+            );
         }
     }
 
-    for s in arg_pattern {
+    for entry in pattern {
-        let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
+        match_list.push(
-        let p = pxar::MatchPattern::from_line(l.as_bytes())?
+            MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Include)
-            .ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
+                .map_err(|err| format_err!("error in pattern: {}", err))?,
-        pattern_list.push(p);
+        );
     }
 
-    let pattern = if pattern_list.is_empty() {
-        None
-    } else {
-        Some(pattern_list)
-    };
-
     if archive == "-" {
         let stdin = std::io::stdin();
         let mut reader = stdin.lock();
-        extract_archive_from_reader(&mut reader, target, feature_flags, allow_existing_dirs, verbose, pattern)?;
+        extract_archive_from_reader(
+            &mut reader,
+            &target,
+            feature_flags,
+            allow_existing_dirs,
+            verbose,
+            &match_list,
+        )?;
     } else {
-        if verbose { println!("PXAR extract: {}", archive); }
+        if verbose {
+            println!("PXAR extract: {}", archive);
+        }
         let file = std::fs::File::open(archive)?;
         let mut reader = std::io::BufReader::new(file);
-        extract_archive_from_reader(&mut reader, target, feature_flags, allow_existing_dirs, verbose, pattern)?;
+        extract_archive_from_reader(
+            &mut reader,
+            &target,
+            feature_flags,
+            allow_existing_dirs,
+            verbose,
+            &match_list,
+        )?;
     }
 
-    Ok(Value::Null)
+    Ok(())
 }
 
+#[api(
+    input: {
+        properties: {
+            archive: {
+                description: "Archive name.",
+            },
+            source: {
+                description: "Source directory.",
+            },
+            verbose: {
+                description: "Verbose output.",
+                optional: true,
+                default: false,
+            },
+            "no-xattrs": {
+                description: "Ignore extended file attributes.",
+                optional: true,
+                default: false,
+            },
+            "no-fcaps": {
+                description: "Ignore file capabilities.",
+                optional: true,
+                default: false,
+            },
+            "no-acls": {
+                description: "Ignore access control list entries.",
+                optional: true,
+                default: false,
+            },
+            "all-file-systems": {
+                description: "Include mounted sudirs.",
+                optional: true,
+                default: false,
+            },
+            "no-device-nodes": {
+                description: "Ignore device nodes.",
+                optional: true,
+                default: false,
+            },
+            "no-fifos": {
+                description: "Ignore fifos.",
+                optional: true,
+                default: false,
+            },
+            "no-sockets": {
+                description: "Ignore sockets.",
+                optional: true,
+                default: false,
+            },
+            exclude: {
+                description: "List of paths or pattern matching files to exclude.",
+                optional: true,
+                type: Array,
+                items: {
+                    description: "Path or pattern matching files to restore",
+                    type: String,
+                },
+            },
+            "entries-max": {
+                description: "Max number of entries loaded at once into memory",
+                optional: true,
+                default: ENCODER_MAX_ENTRIES as isize,
+                minimum: 0,
+                maximum: std::isize::MAX,
+            },
+        },
+    },
+)]
+/// Create a new .pxar archive.
 fn create_archive(
-    param: Value,
+    archive: String,
-    _info: &ApiMethod,
+    source: String,
-    _rpcenv: &mut dyn RpcEnvironment,
+    verbose: bool,
-) -> Result<Value, Error> {
+    no_xattrs: bool,
+    no_fcaps: bool,
+    no_acls: bool,
+    all_file_systems: bool,
+    no_device_nodes: bool,
+    no_fifos: bool,
+    no_sockets: bool,
+    exclude: Option<Vec<String>>,
+    entries_max: isize,
+) -> Result<(), Error> {
+    let exclude_list = {
+        let input = exclude.unwrap_or_else(Vec::new);
+        let mut exclude = Vec::with_capacity(input.len());
+        for entry in input {
+            exclude.push(
+                MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
+                    .map_err(|err| format_err!("error in exclude pattern: {}", err))?,
+            );
+        }
+        exclude
+    };
 
-    let archive = tools::required_string_param(&param, "archive")?;
+    let device_set = if all_file_systems {
-    let source = tools::required_string_param(&param, "source")?;
+        None
-    let verbose = param["verbose"].as_bool().unwrap_or(false);
+    } else {
-    let all_file_systems = param["all-file-systems"].as_bool().unwrap_or(false);
+        Some(HashSet::new())
-    let no_xattrs = param["no-xattrs"].as_bool().unwrap_or(false);
+    };
-    let no_fcaps = param["no-fcaps"].as_bool().unwrap_or(false);
-    let no_acls = param["no-acls"].as_bool().unwrap_or(false);
-    let no_device_nodes = param["no-device-nodes"].as_bool().unwrap_or(false);
-    let no_fifos = param["no-fifos"].as_bool().unwrap_or(false);
-    let no_sockets = param["no-sockets"].as_bool().unwrap_or(false);
-    let empty = Vec::new();
-    let exclude_pattern = param["exclude"].as_array().unwrap_or(&empty);
-    let entries_max = param["entries-max"].as_u64().unwrap_or(pxar::ENCODER_MAX_ENTRIES as u64);
-
-    let devices = if all_file_systems { None } else { Some(HashSet::new()) };
 
     let source = PathBuf::from(source);
 
-    let mut dir = nix::dir::Dir::open(
+    let dir = nix::dir::Dir::open(
-        &source, nix::fcntl::OFlag::O_NOFOLLOW, nix::sys::stat::Mode::empty())?;
+        &source,
+        nix::fcntl::OFlag::O_NOFOLLOW,
+        nix::sys::stat::Mode::empty(),
+    )?;
 
     let file = OpenOptions::new()
         .create_new(true)
@@ -193,332 +307,150 @@ fn create_archive(
         .mode(0o640)
         .open(archive)?;
 
-    let mut writer = std::io::BufWriter::with_capacity(1024*1024, file);
+    let writer = std::io::BufWriter::with_capacity(1024 * 1024, file);
-    let mut feature_flags = pxar::flags::DEFAULT;
+    let mut feature_flags = flags::DEFAULT;
     if no_xattrs {
-        feature_flags ^= pxar::flags::WITH_XATTRS;
+        feature_flags ^= flags::WITH_XATTRS;
     }
     if no_fcaps {
-        feature_flags ^= pxar::flags::WITH_FCAPS;
+        feature_flags ^= flags::WITH_FCAPS;
     }
     if no_acls {
-        feature_flags ^= pxar::flags::WITH_ACL;
+        feature_flags ^= flags::WITH_ACL;
     }
     if no_device_nodes {
-        feature_flags ^= pxar::flags::WITH_DEVICE_NODES;
+        feature_flags ^= flags::WITH_DEVICE_NODES;
     }
     if no_fifos {
-        feature_flags ^= pxar::flags::WITH_FIFOS;
+        feature_flags ^= flags::WITH_FIFOS;
     }
     if no_sockets {
-        feature_flags ^= pxar::flags::WITH_SOCKETS;
+        feature_flags ^= flags::WITH_SOCKETS;
     }
 
-    let mut pattern_list = Vec::new();
+    let writer = pxar::encoder::sync::StandardWriter::new(writer);
-    for s in exclude_pattern {
+    proxmox_backup::pxar::create_archive(
-        let l = s.as_str().ok_or_else(|| format_err!("Invalid pattern string slice"))?;
+        dir,
-        let p = pxar::MatchPattern::from_line(l.as_bytes())?
+        writer,
-            .ok_or_else(|| format_err!("Invalid match pattern in arguments"))?;
+        exclude_list,
-        pattern_list.push(p);
-    }
-
-    let catalog = None::<&mut pxar::catalog::DummyCatalogWriter>;
-    pxar::Encoder::encode(
-        source,
-        &mut dir,
-        &mut writer,
-        catalog,
-        devices,
-        verbose,
-        false,
         feature_flags,
-        pattern_list,
+        device_set,
+        true,
+        |path| {
+            if verbose {
+                println!("{:?}", path);
+            }
+            Ok(())
+        },
         entries_max as usize,
+        None,
     )?;
 
-    writer.flush()?;
+    Ok(())
-
-    Ok(Value::Null)
 }
 
+#[api(
+    input: {
+        properties: {
+            archive: { description: "Archive name." },
+            mountpoint: { description: "Mountpoint for the file system." },
+            verbose: {
+                description: "Verbose output, running in the foreground (for debugging).",
+                optional: true,
+                default: false,
+            },
+        },
+    },
+)]
 /// Mount the archive to the provided mountpoint via FUSE.
-fn mount_archive(
+async fn mount_archive(
-    param: Value,
+    archive: String,
-    _info: &ApiMethod,
+    mountpoint: String,
-    _rpcenv: &mut dyn RpcEnvironment,
+    verbose: bool,
-) -> Result<Value, Error> {
+) -> Result<(), Error> {
-    let archive = tools::required_string_param(&param, "archive")?;
+    let archive = Path::new(&archive);
-    let mountpoint = tools::required_string_param(&param, "mountpoint")?;
+    let mountpoint = Path::new(&mountpoint);
-    let verbose = param["verbose"].as_bool().unwrap_or(false);
-    let no_mt = param["no-mt"].as_bool().unwrap_or(false);
-
-    let archive = Path::new(archive);
-    let mountpoint = Path::new(mountpoint);
     let options = OsStr::new("ro,default_permissions");
-    let mut session = pxar::fuse::Session::from_path(&archive, &options, verbose)
-        .map_err(|err| format_err!("pxar mount failed: {}", err))?;
-    // Mount the session and deamonize if verbose is not set
-    session.mount(&mountpoint, !verbose)?;
-    session.run_loop(!no_mt)?;
 
-    Ok(Value::Null)
+    let session = fuse::Session::mount_path(&archive, &options, verbose, mountpoint)
+        .await
+        .map_err(|err| format_err!("pxar mount failed: {}", err))?;
+
+    let mut interrupt = signal(SignalKind::interrupt())?;
+
+    select! {
+        res = session.fuse() => res?,
+        _ = interrupt.recv().fuse() => {
+            if verbose {
+                eprintln!("interrupted");
+            }
+        }
+    }
+
+    Ok(())
 }
 
-#[sortable]
+#[api(
-const API_METHOD_CREATE_ARCHIVE: ApiMethod = ApiMethod::new(
+    input: {
-    &ApiHandler::Sync(&create_archive),
+        properties: {
-    &ObjectSchema::new(
+            archive: {
-        "Create new .pxar archive.",
+                description: "Archive name.",
-        &sorted!([
+            },
-            (
+            verbose: {
-                "archive",
+                description: "Verbose output.",
-                false,
+                optional: true,
-                &StringSchema::new("Archive name").schema()
+                default: false,
-            ),
+            },
-            (
+        },
-                "source",
+    },
-                false,
+)]
-                &StringSchema::new("Source directory.").schema()
+/// List the contents of an archive.
-            ),
+fn dump_archive(archive: String, verbose: bool) -> Result<(), Error> {
-            (
+    for entry in pxar::decoder::Decoder::open(archive)? {
-                "verbose",
+        let entry = entry?;
-                true,
-                &BooleanSchema::new("Verbose output.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-xattrs",
-                true,
-                &BooleanSchema::new("Ignore extended file attributes.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-fcaps",
-                true,
-                &BooleanSchema::new("Ignore file capabilities.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-acls",
-                true,
-                &BooleanSchema::new("Ignore access control list entries.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "all-file-systems",
-                true,
-                &BooleanSchema::new("Include mounted sudirs.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-device-nodes",
-                true,
-                &BooleanSchema::new("Ignore device nodes.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-fifos",
-                true,
-                &BooleanSchema::new("Ignore fifos.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-sockets",
-                true,
-                &BooleanSchema::new("Ignore sockets.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "exclude",
-                true,
-                &ArraySchema::new(
-                    "List of paths or pattern matching files to exclude.",
-                    &StringSchema::new("Path or pattern matching files to restore.").schema()
-                ).schema()
-            ),
-            (
-                "entries-max",
-                true,
-                &IntegerSchema::new("Max number of entries loaded at once into memory")
-                    .default(pxar::ENCODER_MAX_ENTRIES as isize)
-                    .minimum(0)
-                    .maximum(std::isize::MAX)
-                    .schema()
-            ),
-        ]),
-    )
-);
 
-#[sortable]
+        if verbose {
-const API_METHOD_EXTRACT_ARCHIVE: ApiMethod = ApiMethod::new(
+            println!("{}", format_single_line_entry(&entry));
-    &ApiHandler::Sync(&extract_archive),
+        } else {
-    &ObjectSchema::new(
+            println!("{:?}", entry.path());
-        "Extract an archive.",
+        }
-        &sorted!([
+    }
-            (
+    Ok(())
-                "archive",
+}
-                false,
-                &StringSchema::new("Archive name.").schema()
-            ),
-            (
-                "pattern",
-                true,
-                &ArraySchema::new(
-                    "List of paths or pattern matching files to restore",
-                    &StringSchema::new("Path or pattern matching files to restore.").schema()
-                ).schema()
-            ),
-            (
-                "target",
-                true,
-                &StringSchema::new("Target directory.").schema()
-            ),
-            (
-                "verbose",
-                true,
-                &BooleanSchema::new("Verbose output.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-xattrs",
-                true,
-                &BooleanSchema::new("Ignore extended file attributes.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-fcaps",
-                true,
-                &BooleanSchema::new("Ignore file capabilities.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-acls",
-                true,
-                &BooleanSchema::new("Ignore access control list entries.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "allow-existing-dirs",
-                true,
-                &BooleanSchema::new("Allows directories to already exist on restore.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "files-from",
-                true,
-                &StringSchema::new("Match pattern for files to restore.").schema()
-            ),
-            (
-                "no-device-nodes",
-                true,
-                &BooleanSchema::new("Ignore device nodes.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-fifos",
-                true,
-                &BooleanSchema::new("Ignore fifos.")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-sockets",
-                true,
-                &BooleanSchema::new("Ignore sockets.")
-                    .default(false)
-                    .schema()
-            ),
-        ]),
-    )
-);
-
-#[sortable]
-const API_METHOD_MOUNT_ARCHIVE: ApiMethod = ApiMethod::new(
-    &ApiHandler::Sync(&mount_archive),
-    &ObjectSchema::new(
-        "Mount the archive as filesystem via FUSE.",
-        &sorted!([
-            (
-                "archive",
-                false,
-                &StringSchema::new("Archive name.").schema()
-            ),
-            (
-                "mountpoint",
-                false,
-                &StringSchema::new("Mountpoint for the filesystem root.").schema()
-            ),
-            (
-                "verbose",
-                true,
-                &BooleanSchema::new("Verbose output, keeps process running in foreground (for debugging).")
-                    .default(false)
-                    .schema()
-            ),
-            (
-                "no-mt",
-                true,
-                &BooleanSchema::new("Run in single threaded mode (for debugging).")
-                    .default(false)
-                    .schema()
-            ),
-        ]),
-    )
-);
-
-#[sortable]
-const API_METHOD_DUMP_ARCHIVE: ApiMethod = ApiMethod::new(
-    &ApiHandler::Sync(&dump_archive),
-    &ObjectSchema::new(
-        "List the contents of an archive.",
-        &sorted!([
-            ( "archive", false, &StringSchema::new("Archive name.").schema()),
-            ( "verbose", true, &BooleanSchema::new("Verbose output.")
-               .default(false)
-               .schema()
-            ),
-        ])
-    )
-);
 
 fn main() {
-
     let cmd_def = CliCommandMap::new()
-        .insert("create", CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
+        .insert(
+            "create",
+            CliCommand::new(&API_METHOD_CREATE_ARCHIVE)
                 .arg_param(&["archive", "source"])
                 .completion_cb("archive", tools::complete_file_name)
-            .completion_cb("source", tools::complete_file_name)
+                .completion_cb("source", tools::complete_file_name),
         )
-        .insert("extract", CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
+        .insert(
+            "extract",
+            CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE)
                 .arg_param(&["archive", "target"])
                 .completion_cb("archive", tools::complete_file_name)
                 .completion_cb("target", tools::complete_file_name)
-            .completion_cb("files-from", tools::complete_file_name)
+                .completion_cb("files-from", tools::complete_file_name),
         )
-        .insert("mount", CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
+        .insert(
+            "mount",
+            CliCommand::new(&API_METHOD_MOUNT_ARCHIVE)
                 .arg_param(&["archive", "mountpoint"])
                 .completion_cb("archive", tools::complete_file_name)
-            .completion_cb("mountpoint", tools::complete_file_name)
+                .completion_cb("mountpoint", tools::complete_file_name),
         )
-        .insert("list", CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
+        .insert(
+            "list",
+            CliCommand::new(&API_METHOD_DUMP_ARCHIVE)
                 .arg_param(&["archive"])
-            .completion_cb("archive", tools::complete_file_name)
+                .completion_cb("archive", tools::complete_file_name),
         );
 
     let rpcenv = CliEnvironment::new();
-    run_cli_command(cmd_def, rpcenv, None);
+    run_cli_command(cmd_def, rpcenv, Some(|future| {
+        proxmox_backup::tools::runtime::main(future)
+    }));
 }

src/client.rs

@@ -3,8 +3,8 @@
 //! This library implements the client side to access the backups
 //! server using https.
 
-pub mod pipe_to_stream;
 mod merge_known_chunks;
+pub mod pipe_to_stream;
 
 mod http_client;
 pub use http_client::*;
@@ -24,9 +24,6 @@ pub use remote_chunk_reader::*;
 mod pxar_backup_stream;
 pub use pxar_backup_stream::*;
 
-mod pxar_decode_writer;
-pub use pxar_decode_writer::*;
-
 mod backup_repo;
 pub use backup_repo::*;
 

src/client/pxar_backup_stream.rs

@@ -9,12 +9,12 @@ use std::thread;
 
 use anyhow::{format_err, Error};
 use futures::stream::Stream;
-
+use nix::dir::Dir;
 use nix::fcntl::OFlag;
 use nix::sys::stat::Mode;
-use nix::dir::Dir;
 
-use crate::pxar;
+use pathpatterns::MatchEntry;
+
 use crate::backup::CatalogWriter;
 
 /// Stream implementation to encode and upload .pxar archives.
@@ -29,7 +29,6 @@ pub struct PxarBackupStream {
 }
 
 impl Drop for PxarBackupStream {
-
     fn drop(&mut self) {
         self.rx = None;
         self.child.take().unwrap().join().unwrap();
@@ -37,45 +36,48 @@ impl Drop for PxarBackupStream {
 }
 
 impl PxarBackupStream {
-
     pub fn new<W: Write + Send + 'static>(
-        mut dir: Dir,
+        dir: Dir,
-        path: PathBuf,
+        _path: PathBuf,
         device_set: Option<HashSet<u64>>,
-        verbose: bool,
+        _verbose: bool,
         skip_lost_and_found: bool,
         catalog: Arc<Mutex<CatalogWriter<W>>>,
-        exclude_pattern: Vec<pxar::MatchPattern>,
+        exclude_pattern: Vec<MatchEntry>,
         entries_max: usize,
     ) -> Result<Self, Error> {
-
         let (tx, rx) = std::sync::mpsc::sync_channel(10);
 
-        let buffer_size = 256*1024;
+        let buffer_size = 256 * 1024;
 
         let error = Arc::new(Mutex::new(None));
-        let error2 = error.clone();
+        let child = std::thread::Builder::new()
+            .name("PxarBackupStream".to_string())
+            .spawn({
+                let error = Arc::clone(&error);
+                move || {
+                    let mut catalog_guard = catalog.lock().unwrap();
+                    let writer = std::io::BufWriter::with_capacity(
+                        buffer_size,
+                        crate::tools::StdChannelWriter::new(tx),
+                    );
 
-        let catalog = catalog.clone();
+                    let writer = pxar::encoder::sync::StandardWriter::new(writer);
-        let child = std::thread::Builder::new().name("PxarBackupStream".to_string()).spawn(move || {
+                    if let Err(err) = crate::pxar::create_archive(
-            let mut guard = catalog.lock().unwrap();
+                        dir,
-            let mut writer = std::io::BufWriter::with_capacity(buffer_size, crate::tools::StdChannelWriter::new(tx));
+                        writer,
-
-            if let Err(err) = pxar::Encoder::encode(
-                path,
-                &mut dir,
-                &mut writer,
-                Some(&mut *guard),
-                device_set,
-                verbose,
-                skip_lost_and_found,
-                pxar::flags::DEFAULT,
                         exclude_pattern,
+                        crate::pxar::flags::DEFAULT,
+                        device_set,
+                        skip_lost_and_found,
+                        |_| Ok(()),
                         entries_max,
+                        Some(&mut *catalog_guard),
                     ) {
-                let mut error = error2.lock().unwrap();
+                        let mut error = error.lock().unwrap();
                         *error = Some(err.to_string());
                     }
+                }
             })?;
 
         Ok(Self {
@@ -91,23 +93,31 @@ impl PxarBackupStream {
         verbose: bool,
         skip_lost_and_found: bool,
         catalog: Arc<Mutex<CatalogWriter<W>>>,
-        exclude_pattern: Vec<pxar::MatchPattern>,
+        exclude_pattern: Vec<MatchEntry>,
         entries_max: usize,
     ) -> Result<Self, Error> {
-
         let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
         let path = std::path::PathBuf::from(dirname);
 
-        Self::new(dir, path, device_set, verbose, skip_lost_and_found, catalog, exclude_pattern, entries_max)
+        Self::new(
+            dir,
+            path,
+            device_set,
+            verbose,
+            skip_lost_and_found,
+            catalog,
+            exclude_pattern,
+            entries_max,
+        )
     }
 }
 
 impl Stream for PxarBackupStream {
-
     type Item = Result<Vec<u8>, Error>;
 
     fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Option<Self::Item>> {
-        { // limit lock scope
+        {
+            // limit lock scope
             let error = self.error.lock().unwrap();
             if let Some(ref msg) = *error {
                 return Poll::Ready(Some(Err(format_err!("{}", msg))));

src/client/pxar_decode_writer.rs

@@ -1,70 +0,0 @@
-use anyhow::{Error};
-
-use std::thread;
-use std::os::unix::io::FromRawFd;
-use std::path::{Path, PathBuf};
-use std::io::Write;
-
-use crate::pxar;
-
-/// Writer implementation to deccode a .pxar archive (download).
-
-pub struct PxarDecodeWriter {
-    pipe: Option<std::fs::File>,
-    child: Option<thread::JoinHandle<()>>,
-}
-
-impl Drop for PxarDecodeWriter {
-
-    fn drop(&mut self) {
-        drop(self.pipe.take());
-        self.child.take().unwrap().join().unwrap();
-    }
-}
-
-impl PxarDecodeWriter {
-
-    pub fn new(base: &Path, verbose: bool) -> Result<Self, Error> {
-        let (rx, tx) = nix::unistd::pipe()?;
-
-        let base = PathBuf::from(base);
-        
-        let child = thread::spawn(move|| {
-            let mut reader = unsafe { std::fs::File::from_raw_fd(rx) };
-            let mut decoder = pxar::SequentialDecoder::new(&mut reader, pxar::flags::DEFAULT);
-            decoder.set_callback(move |path| {
-                if verbose {
-                    println!("{:?}", path);
-                }
-                Ok(())
-            });
-
-            if let Err(err) = decoder.restore(&base, &Vec::new()) {
-                eprintln!("pxar decode failed - {}", err);
-            }
-        });
-
-        let pipe = unsafe { std::fs::File::from_raw_fd(tx) };
-
-        Ok(Self { pipe: Some(pipe), child: Some(child) })
-    }
-}
-
-impl Write for PxarDecodeWriter {
-
-    fn write(&mut self, buffer: &[u8]) -> Result<usize, std::io::Error> {
-        let pipe = match self.pipe {
-            Some(ref mut pipe) => pipe,
-            None => unreachable!(),
-        };
-        pipe.write(buffer)
-    }
-
-    fn flush(&mut self) -> Result<(), std::io::Error> {
-        let pipe = match self.pipe {
-            Some(ref mut pipe) => pipe,
-            None => unreachable!(),
-        };
-        pipe.flush()
-    }
-}

src/pxar.rs

@@ -47,33 +47,21 @@
 //! (user, group, acl, ...) because this is already defined by the
 //! linked `ENTRY`.
 
-mod binary_search_tree;
-pub use binary_search_tree::*;
-
-pub mod flags;
-pub use flags::*;
-
-mod format_definition;
-pub use format_definition::*;
-
-mod encoder;
-pub use encoder::*;
-
-mod sequential_decoder;
-pub use sequential_decoder::*;
-
-mod decoder;
-pub use decoder::*;
-
-mod match_pattern;
-pub use match_pattern::*;
-
-mod dir_stack;
-pub use dir_stack::*;
-
-pub mod fuse;
-pub use fuse::*;
-
 pub mod catalog;
+pub(crate) mod create;
+pub(crate) mod dir_stack;
+pub(crate) mod extract;
+pub(crate) mod metadata;
+pub mod flags;
+pub mod fuse;
+pub(crate) mod tools;
 
-mod helper;
+pub use create::create_archive;
+pub use extract::extract_archive;
+
+/// The format requires to build sorted directory lookup tables in
+/// memory, so we restrict the number of allowed entries to limit
+/// maximum memory usage.
+pub const ENCODER_MAX_ENTRIES: usize = 1024 * 1024;
+
+pub use tools::{format_multi_line_entry, format_single_line_entry};

src/pxar/binary_search_tree.rs

@@ -1,229 +0,0 @@
-//! Helpers to generate a binary search tree stored in an array from a
-//! sorted array.
-//!
-//! Specifically, for any given sorted array 'input' permute the
-//! array so that the following rule holds:
-//!
-//! For each array item with index i, the item at 2i+1 is smaller and
-//! the item 2i+2 is larger.
-//!
-//! This structure permits efficient (meaning: O(log(n)) binary
-//! searches: start with item i=0 (i.e. the root of the BST), compare
-//! the value with the searched item, if smaller proceed at item
-//! 2i+1, if larger proceed at item 2i+2, and repeat, until either
-//! the item is found, or the indexes grow beyond the array size,
-//! which means the entry does not exist.
-//!
-//! Effectively this implements bisection, but instead of jumping
-//! around wildly in the array during a single search we only search
-//! with strictly monotonically increasing indexes.
-//!
-//! Algorithm is from casync (camakebst.c), simplified and optimized
-//! for rust. Permutation function originally by L. Bressel, 2017. We
-//! pass permutation info to user provided callback, which actually
-//! implements the data copy.
-//!
-//! The Wikipedia Artikel for [Binary
-//! Heap](https://en.wikipedia.org/wiki/Binary_heap) gives a short
-//! intro howto store binary trees using an array.
-
-use std::cmp::Ordering;
-
-#[allow(clippy::many_single_char_names)]
-fn copy_binary_search_tree_inner<F:  FnMut(usize, usize)>(
-    copy_func: &mut F,
-    // we work on input array input[o..o+n]
-    n: usize,
-    o: usize,
-    e: usize,
-    i: usize,
-) {
-    let p = 1 << e;
-
-    let t = p + (p>>1) - 1;
-
-    let m = if n > t {
-        // |...........p.............t....n........(2p)|
-        p - 1
-    } else {
-        // |...........p.....n.......t.............(2p)|
-        p - 1 - (t-n)
-    };
-
-    (copy_func)(o+m, i);
-
-    if m > 0 {
-        copy_binary_search_tree_inner(copy_func, m, o, e-1, i*2+1);
-    }
-
-    if (m + 1) < n {
-        copy_binary_search_tree_inner(copy_func, n-m-1, o+m+1, e-1, i*2+2);
-    }
-}
-
-/// This function calls the provided `copy_func()` with the permutation
-/// info.
-///
-/// ```
-/// # use proxmox_backup::pxar::copy_binary_search_tree;
-/// copy_binary_search_tree(5, |src, dest| {
-///    println!("Copy {} to {}", src, dest);
-/// });
-/// ```
-///
-/// This will produce the following output:
-///
-/// ```no-compile
-/// Copy 3 to 0
-/// Copy 1 to 1
-/// Copy 0 to 3
-/// Copy 2 to 4
-/// Copy 4 to 2
-/// ```
-///
-/// So this generates the following permutation: `[3,1,4,0,2]`.
-
-pub fn copy_binary_search_tree<F:  FnMut(usize, usize)>(
-    n: usize,
-    mut copy_func: F,
-) {
-    if n == 0 { return };
-    let e = (64 - n.leading_zeros() - 1) as usize; // fast log2(n)
-
-    copy_binary_search_tree_inner(&mut copy_func, n, 0, e, 0);
-}
-
-
-/// This function searches for the index where the comparison by the provided
-/// `compare()` function returns `Ordering::Equal`.
-/// The order of the comparison matters (noncommutative) and should be search
-/// value compared to value at given index as shown in the examples.
-/// The parameter `skip_multiples` defines the number of matches to ignore while
-/// searching before returning the index in order to lookup duplicate entries in
-/// the tree.
-///
-/// ```
-/// # use proxmox_backup::pxar::{copy_binary_search_tree, search_binary_tree_by};
-/// let mut vals = vec![0,1,2,2,2,3,4,5,6,6,7,8,8,8];
-///
-/// let clone = vals.clone();
-/// copy_binary_search_tree(vals.len(), |s, d| {
-///     vals[d] = clone[s];
-/// });
-/// let should_be = vec![5,2,8,1,3,6,8,0,2,2,4,6,7,8];
-/// assert_eq!(vals, should_be);
-///
-/// let find = 8;
-/// let skip_multiples = 0;
-/// let idx = search_binary_tree_by(0, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
-/// assert_eq!(idx, Some(2));
-///
-/// let find = 8;
-/// let skip_multiples = 1;
-/// let idx = search_binary_tree_by(2, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
-/// assert_eq!(idx, Some(6));
-///
-/// let find = 8;
-/// let skip_multiples = 1;
-/// let idx = search_binary_tree_by(6, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
-/// assert_eq!(idx, Some(13));
-///
-/// let find = 5;
-/// let skip_multiples = 1;
-/// let idx = search_binary_tree_by(0, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
-/// assert!(idx.is_none());
-///
-/// let find = 5;
-/// let skip_multiples = 0;
-/// // if start index is equal to the array length, `None` is returned.
-/// let idx = search_binary_tree_by(vals.len(), vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
-/// assert!(idx.is_none());
-///
-/// let find = 5;
-/// let skip_multiples = 0;
-/// // if start index is larger than length, `None` is returned.
-/// let idx = search_binary_tree_by(vals.len() + 1, vals.len(), skip_multiples, |idx| find.cmp(&vals[idx]));
-/// assert!(idx.is_none());
-/// ```
-
-pub fn search_binary_tree_by<F: Copy + Fn(usize) -> Ordering>(
-    start: usize,
-    size: usize,
-    skip_multiples: usize,
-    compare: F
-) -> Option<usize> {
-    if start >= size {
-        return None;
-    }
-
-    let mut skip = skip_multiples;
-    let cmp = compare(start);
-    if cmp == Ordering::Equal {
-        if skip == 0 {
-            // Found matching hash and want this one
-            return Some(start);
-        }
-        // Found matching hash, but we should skip the first `skip_multiple`,
-        // so continue search with reduced skip count.
-        skip -= 1;
-    }
-
-    if cmp == Ordering::Less || cmp == Ordering::Equal {
-        let res = search_binary_tree_by(2 * start + 1, size, skip, compare);
-        if res.is_some() {
-            return res;
-        }
-    }
-
-    if cmp == Ordering::Greater || cmp == Ordering::Equal {
-        let res = search_binary_tree_by(2 * start + 2, size, skip, compare);
-        if res.is_some() {
-            return res;
-        }
-    }
-
-    None
-}
-
-#[test]
-fn test_binary_search_tree() {
-
-    fn run_test(len: usize) -> Vec<usize> {
-
-        const MARKER: usize = 0xfffffff;
-        let mut output = vec![];
-        for _i in 0..len { output.push(MARKER); }
-        copy_binary_search_tree(len, |s, d| {
-            assert!(output[d] == MARKER);
-            output[d] = s;
-        });
-        if len < 32 { println!("GOT:{}:{:?}", len, output); }
-        for i in 0..len {
-            assert!(output[i] != MARKER);
-        }
-        output
-    }
-
-    assert!(run_test(0).len() == 0);
-    assert!(run_test(1) == [0]);
-    assert!(run_test(2) == [1,0]);
-    assert!(run_test(3) == [1,0,2]);
-    assert!(run_test(4) == [2,1,3,0]);
-    assert!(run_test(5) == [3,1,4,0,2]);
-    assert!(run_test(6) == [3,1,5,0,2,4]);
-    assert!(run_test(7) == [3,1,5,0,2,4,6]);
-    assert!(run_test(8) == [4,2,6,1,3,5,7,0]);
-    assert!(run_test(9) == [5,3,7,1,4,6,8,0,2]);
-    assert!(run_test(10) == [6,3,8,1,5,7,9,0,2,4]);
-    assert!(run_test(11) == [7,3,9,1,5,8,10,0,2,4,6]);
-    assert!(run_test(12) == [7,3,10,1,5,9,11,0,2,4,6,8]);
-    assert!(run_test(13) == [7,3,11,1,5,9,12,0,2,4,6,8,10]);
-    assert!(run_test(14) == [7,3,11,1,5,9,13,0,2,4,6,8,10,12]);
-    assert!(run_test(15) == [7,3,11,1,5,9,13,0,2,4,6,8,10,12,14]);
-    assert!(run_test(16) == [8,4,12,2,6,10,14,1,3,5,7,9,11,13,15,0]);
-    assert!(run_test(17) == [9,5,13,3,7,11,15,1,4,6,8,10,12,14,16,0,2]);
-
-    for len in 18..1000 {
-        run_test(len);
-    }
-}

src/pxar/create.rs

@@ -0,0 +1,769 @@
+use std::collections::{HashSet, HashMap};
+use std::convert::TryFrom;
+use std::ffi::{CStr, CString, OsStr};
+use std::fmt;
+use std::os::unix::ffi::OsStrExt;
+use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
+use std::path::{Path, PathBuf};
+
+use anyhow::{bail, format_err, Error};
+use nix::dir::Dir;
+use nix::errno::Errno;
+use nix::fcntl::OFlag;
+use nix::sys::stat::{FileStat, Mode};
+
+use pathpatterns::{MatchEntry, MatchList, MatchType, PatternFlag};
+use pxar::Metadata;
+use pxar::encoder::LinkOffset;
+
+use proxmox::sys::error::SysError;
+use proxmox::tools::fd::RawFdNum;
+
+use crate::pxar::catalog::BackupCatalogWriter;
+use crate::pxar::flags;
+use crate::pxar::tools::assert_relative_path;
+use crate::tools::{acl, fs, xattr, Fd};
+
+fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
+    let mut fs_stat = std::mem::MaybeUninit::uninit();
+    let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
+    Errno::result(res)?;
+    let fs_stat = unsafe { fs_stat.assume_init() };
+
+    Ok(fs_stat.f_type)
+}
+
+pub fn is_virtual_file_system(magic: i64) -> bool {
+    use proxmox::sys::linux::magic::*;
+
+    match magic {
+        BINFMTFS_MAGIC |
+        CGROUP2_SUPER_MAGIC |
+        CGROUP_SUPER_MAGIC |
+        CONFIGFS_MAGIC |
+        DEBUGFS_MAGIC |
+        DEVPTS_SUPER_MAGIC |
+        EFIVARFS_MAGIC |
+        FUSE_CTL_SUPER_MAGIC |
+        HUGETLBFS_MAGIC |
+        MQUEUE_MAGIC |
+        NFSD_MAGIC |
+        PROC_SUPER_MAGIC |
+        PSTOREFS_MAGIC |
+        RPCAUTH_GSSMAGIC |
+        SECURITYFS_MAGIC |
+        SELINUX_MAGIC |
+        SMACK_MAGIC |
+        SYSFS_MAGIC => true,
+        _ => false
+    }
+}
+
+#[derive(Debug)]
+struct ArchiveError {
+    path: PathBuf,
+    error: Error,
+}
+
+impl ArchiveError {
+    fn new(path: PathBuf, error: Error) -> Self {
+        Self { path, error }
+    }
+}
+
+impl std::error::Error for ArchiveError {}
+
+impl fmt::Display for ArchiveError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "error at {:?}: {}", self.path, self.error)
+    }
+}
+
+#[derive(Eq, PartialEq, Hash)]
+struct HardLinkInfo {
+    st_dev: u64,
+    st_ino: u64,
+}
+
+struct Archiver<'a, 'b> {
+    /// FIXME: use bitflags!() for feature_flags
+    feature_flags: u64,
+    fs_feature_flags: u64,
+    fs_magic: i64,
+    excludes: &'a [MatchEntry],
+    callback: &'a mut dyn FnMut(&Path) -> Result<(), Error>,
+    catalog: Option<&'b mut dyn BackupCatalogWriter>,
+    path: PathBuf,
+    entry_counter: usize,
+    entry_limit: usize,
+    current_st_dev: libc::dev_t,
+    device_set: Option<HashSet<u64>>,
+    hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
+}
+
+type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::SeqWrite>;
+
+pub fn create_archive<T, F>(
+    source_dir: Dir,
+    mut writer: T,
+    mut excludes: Vec<MatchEntry>,
+    feature_flags: u64,
+    mut device_set: Option<HashSet<u64>>,
+    skip_lost_and_found: bool,
+    mut callback: F,
+    entry_limit: usize,
+    catalog: Option<&mut dyn BackupCatalogWriter>,
+) -> Result<(), Error>
+where
+    T: pxar::encoder::SeqWrite,
+    F: FnMut(&Path) -> Result<(), Error>,
+{
+    let fs_magic = detect_fs_type(source_dir.as_raw_fd())?;
+    if is_virtual_file_system(fs_magic) {
+        bail!("refusing to backup a virtual file system");
+    }
+
+    let fs_feature_flags = flags::feature_flags_from_magic(fs_magic);
+
+    let stat = nix::sys::stat::fstat(source_dir.as_raw_fd())?;
+    let metadata = get_metadata(
+        source_dir.as_raw_fd(),
+        &stat,
+        feature_flags & fs_feature_flags,
+        fs_magic,
+    )
+    .map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?;
+
+    if let Some(ref mut set) = device_set {
+        set.insert(stat.st_dev);
+    }
+
+    let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite;
+    let mut encoder = Encoder::new(writer, &metadata)?;
+
+    if skip_lost_and_found {
+        excludes.push(MatchEntry::parse_pattern(
+            "**/lost+found",
+            PatternFlag::PATH_NAME,
+            MatchType::Exclude,
+        )?);
+    }
+
+    let mut archiver = Archiver {
+        feature_flags,
+        fs_feature_flags,
+        fs_magic,
+        callback: &mut callback,
+        excludes: &excludes,
+        catalog,
+        path: PathBuf::new(),
+        entry_counter: 0,
+        entry_limit,
+        current_st_dev: stat.st_dev,
+        device_set,
+        hardlinks: HashMap::new(),
+    };
+
+    archiver.archive_dir_contents(&mut encoder, source_dir)?;
+    encoder.finish()?;
+    Ok(())
+}
+
+struct FileListEntry {
+    name: CString,
+    path: PathBuf,
+    stat: FileStat,
+}
+
+impl<'a, 'b> Archiver<'a, 'b> {
+    fn flags(&self) -> u64 {
+        self.feature_flags & self.fs_feature_flags
+    }
+
+    fn wrap_err(&self, err: Error) -> Error {
+        if err.downcast_ref::<ArchiveError>().is_some() {
+            err
+        } else {
+            ArchiveError::new(self.path.clone(), err).into()
+        }
+    }
+
+    fn archive_dir_contents(&mut self, encoder: &mut Encoder, mut dir: Dir) -> Result<(), Error> {
+        let entry_counter = self.entry_counter;
+
+        let file_list = self.generate_directory_file_list(&mut dir)?;
+
+        let dir_fd = dir.as_raw_fd();
+
+        let old_path = std::mem::take(&mut self.path);
+        for file_entry in file_list {
+            (self.callback)(Path::new(OsStr::from_bytes(file_entry.name.to_bytes())))?;
+            self.path = file_entry.path;
+            self.add_entry(encoder, dir_fd, &file_entry.name, &file_entry.stat)
+                .map_err(|err| self.wrap_err(err))?;
+        }
+        self.path = old_path;
+        self.entry_counter = entry_counter;
+
+        Ok(())
+    }
+
+    fn generate_directory_file_list(&mut self, dir: &mut Dir) -> Result<Vec<FileListEntry>, Error> {
+        let dir_fd = dir.as_raw_fd();
+
+        let mut file_list = Vec::new();
+
+        for file in dir.iter() {
+            let file = file?;
+
+            let file_name = file.file_name().to_owned();
+            let file_name_bytes = file_name.to_bytes();
+            if file_name_bytes == b"." || file_name_bytes == b".." {
+                continue;
+            }
+
+            // FIXME: deal with `.pxarexclude-cli`
+
+            if file_name_bytes == b".pxarexclude" {
+                // FIXME: handle this file!
+                continue;
+            }
+
+            let os_file_name = OsStr::from_bytes(file_name_bytes);
+            assert_relative_path(os_file_name)?;
+            let full_path = self.path.join(os_file_name);
+
+            let stat = match nix::sys::stat::fstatat(
+                dir_fd,
+                file_name.as_c_str(),
+                nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
+            ) {
+                Ok(stat) => stat,
+                Err(ref err) if err.not_found() => continue,
+                Err(err) => bail!("stat failed on {:?}: {}", full_path, err),
+            };
+
+            if self
+                .excludes
+                .matches(full_path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
+                == Some(MatchType::Exclude)
+            {
+                continue;
+            }
+
+            self.entry_counter += 1;
+            if self.entry_counter > self.entry_limit {
+                bail!("exceeded allowed number of file entries (> {})",self.entry_limit);
+            }
+
+            file_list.push(FileListEntry {
+                name: file_name,
+                path: full_path,
+                stat
+            });
+        }
+
+        file_list.sort_unstable_by(|a, b| a.name.cmp(&b.name));
+
+        Ok(file_list)
+    }
+
+    fn add_entry(
+        &mut self,
+        encoder: &mut Encoder,
+        parent: RawFd,
+        c_file_name: &CStr,
+        stat: &FileStat,
+    ) -> Result<(), Error> {
+        use pxar::format::mode;
+
+        let file_mode = stat.st_mode & libc::S_IFMT;
+        let open_mode = if !(file_mode == libc::S_IFREG || file_mode == libc::S_IFDIR) {
+            OFlag::O_PATH
+        } else {
+            OFlag::empty()
+        };
+
+        let fd = Fd::openat(
+            &unsafe { RawFdNum::from_raw_fd(parent) },
+            c_file_name,
+            open_mode | OFlag::O_RDONLY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC | OFlag::O_NOCTTY,
+            Mode::empty(),
+        )?;
+
+        let metadata = get_metadata(fd.as_raw_fd(), &stat, self.flags(), self.fs_magic)?;
+
+        if self
+            .excludes
+            .matches(self.path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
+            == Some(MatchType::Exclude)
+        {
+            return Ok(());
+        }
+
+        let file_name: &Path = OsStr::from_bytes(c_file_name.to_bytes()).as_ref();
+        match metadata.file_type() {
+            mode::IFREG => {
+                let link_info = HardLinkInfo {
+                    st_dev: stat.st_dev,
+                    st_ino: stat.st_ino,
+                };
+
+                if stat.st_nlink > 1 {
+                    if let Some((path, offset)) = self.hardlinks.get(&link_info) {
+                        if let Some(ref mut catalog) = self.catalog {
+                            catalog.add_hardlink(c_file_name)?;
+                        }
+
+                        encoder.add_hardlink(file_name, path, *offset)?;
+
+                        return Ok(());
+                    }
+                }
+
+                let file_size = stat.st_size as u64;
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.add_file(c_file_name, file_size, metadata.stat.mtime)?;
+                }
+
+                let offset: LinkOffset =
+                    self.add_regular_file(encoder, fd, file_name, &metadata, file_size)?;
+
+                if stat.st_nlink > 1 {
+                    self.hardlinks.insert(link_info, (self.path.clone(), offset));
+                }
+
+                Ok(())
+            }
+            mode::IFDIR => {
+                let dir = Dir::from_fd(fd.into_raw_fd())?;
+                self.add_directory(encoder, dir, c_file_name, &metadata, stat)
+            }
+            mode::IFSOCK => {
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.add_socket(c_file_name)?;
+                }
+
+                Ok(encoder.add_socket(&metadata, file_name)?)
+            }
+            mode::IFIFO => {
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.add_fifo(c_file_name)?;
+                }
+
+                Ok(encoder.add_fifo(&metadata, file_name)?)
+            }
+            mode::IFLNK => {
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.add_symlink(c_file_name)?;
+                }
+
+                self.add_symlink(encoder, fd, file_name, &metadata)
+            }
+            mode::IFBLK => {
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.add_block_device(c_file_name)?;
+                }
+
+                self.add_device(encoder, file_name, &metadata, &stat)
+            }
+            mode::IFCHR => {
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.add_char_device(c_file_name)?;
+                }
+
+                self.add_device(encoder, file_name, &metadata, &stat)
+            }
+            other => bail!(
+                "encountered unknown file type: 0x{:x} (0o{:o})",
+                other,
+                other
+            ),
+        }
+    }
+
+    fn add_directory(
+        &mut self,
+        encoder: &mut Encoder,
+        dir: Dir,
+        dir_name: &CStr,
+        metadata: &Metadata,
+        stat: &FileStat,
+    ) -> Result<(), Error> {
+        let dir_name = OsStr::from_bytes(dir_name.to_bytes());
+
+        let mut encoder = encoder.create_directory(dir_name, &metadata)?;
+
+        let old_fs_magic = self.fs_magic;
+        let old_fs_feature_flags = self.fs_feature_flags;
+        let old_st_dev = self.current_st_dev;
+
+        let mut skip_contents = false;
+        if old_st_dev != stat.st_dev {
+            self.fs_magic = detect_fs_type(dir.as_raw_fd())?;
+            self.fs_feature_flags = flags::feature_flags_from_magic(self.fs_magic);
+            self.current_st_dev = stat.st_dev;
+
+            if is_virtual_file_system(self.fs_magic) {
+                skip_contents = true;
+            } else if let Some(set) = &self.device_set {
+                skip_contents = !set.contains(&stat.st_dev);
+            }
+        }
+
+        let result = if skip_contents {
+            Ok(())
+        } else {
+            self.archive_dir_contents(&mut encoder, dir)
+        };
+
+        self.fs_magic = old_fs_magic;
+        self.fs_feature_flags = old_fs_feature_flags;
+        self.current_st_dev = old_st_dev;
+
+        encoder.finish()?;
+        result
+    }
+
+    fn add_regular_file(
+        &mut self,
+        encoder: &mut Encoder,
+        fd: Fd,
+        file_name: &Path,
+        metadata: &Metadata,
+        file_size: u64,
+    ) -> Result<LinkOffset, Error> {
+        let mut file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
+        let offset = encoder.add_file(metadata, file_name, file_size, &mut file)?;
+        Ok(offset)
+    }
+
+    fn add_symlink(
+        &mut self,
+        encoder: &mut Encoder,
+        fd: Fd,
+        file_name: &Path,
+        metadata: &Metadata,
+    ) -> Result<(), Error> {
+        let dest = nix::fcntl::readlinkat(fd.as_raw_fd(), &b""[..])?;
+        encoder.add_symlink(metadata, file_name, dest)?;
+        Ok(())
+    }
+
+    fn add_device(
+        &mut self,
+        encoder: &mut Encoder,
+        file_name: &Path,
+        metadata: &Metadata,
+        stat: &FileStat,
+    ) -> Result<(), Error> {
+        Ok(encoder.add_device(
+            metadata,
+            file_name,
+            pxar::format::Device::from_dev_t(stat.st_rdev),
+        )?)
+    }
+}
+
+fn get_metadata(fd: RawFd, stat: &FileStat, flags: u64, fs_magic: i64) -> Result<Metadata, Error> {
+    // required for some of these
+    let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
+
+    let mtime = u64::try_from(stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec)
+        .map_err(|_| format_err!("file with negative mtime"))?;
+
+    let mut meta = Metadata {
+        stat: pxar::Stat {
+            mode: u64::from(stat.st_mode),
+            flags: 0,
+            uid: stat.st_uid,
+            gid: stat.st_gid,
+            mtime,
+        },
+        ..Default::default()
+    };
+
+    get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags)?;
+    get_chattr(&mut meta, fd)?;
+    get_fat_attr(&mut meta, fd, fs_magic)?;
+    get_quota_project_id(&mut meta, fd, flags, fs_magic)?;
+    Ok(meta)
+}
+
+fn errno_is_unsupported(errno: Errno) -> bool {
+    match errno {
+        Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
+        _ => false,
+    }
+}
+
+fn get_fcaps(meta: &mut Metadata, fd: RawFd, flags: u64) -> Result<(), Error> {
+    if 0 == (flags & flags::WITH_FCAPS) {
+        return Ok(());
+    }
+
+    match xattr::fgetxattr(fd, xattr::xattr_name_fcaps()) {
+        Ok(data) => {
+            meta.fcaps = Some(pxar::format::FCaps { data });
+            Ok(())
+        }
+        Err(Errno::ENODATA) => Ok(()),
+        Err(Errno::EOPNOTSUPP) => Ok(()),
+        Err(Errno::EBADF) => Ok(()), // symlinks
+        Err(err) => bail!("failed to read file capabilities: {}", err),
+    }
+}
+
+fn get_xattr_fcaps_acl(
+    meta: &mut Metadata,
+    fd: RawFd,
+    proc_path: &Path,
+    flags: u64,
+) -> Result<(), Error> {
+    if 0 == (flags & flags::WITH_XATTRS) {
+        return Ok(());
+    }
+
+    let xattrs = match xattr::flistxattr(fd) {
+        Ok(names) => names,
+        Err(Errno::EOPNOTSUPP) => return Ok(()),
+        Err(Errno::EBADF) => return Ok(()), // symlinks
+        Err(err) => bail!("failed to read xattrs: {}", err),
+    };
+
+    for attr in &xattrs {
+        if xattr::is_security_capability(&attr) {
+            get_fcaps(meta, fd, flags)?;
+            continue;
+        }
+
+        if xattr::is_acl(&attr) {
+            get_acl(meta, proc_path, flags)?;
+            continue;
+        }
+
+        if !xattr::is_valid_xattr_name(&attr) {
+            continue;
+        }
+
+        match xattr::fgetxattr(fd, attr) {
+            Ok(data) => meta
+                .xattrs
+                .push(pxar::format::XAttr::new(attr.to_bytes(), data)),
+            Err(Errno::ENODATA) => (), // it got removed while we were iterating...
+            Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this
+            Err(Errno::EBADF) => (),   // symlinks, shouldn't be able to reach this either
+            Err(err) => bail!("error reading extended attribute {:?}: {}", attr, err),
+        }
+    }
+
+    Ok(())
+}
+
+fn get_chattr(metadata: &mut Metadata, fd: RawFd) -> Result<(), Error> {
+    let mut attr: usize = 0;
+
+    match unsafe { fs::read_attr_fd(fd, &mut attr) } {
+        Ok(_) => (),
+        Err(nix::Error::Sys(errno)) if errno_is_unsupported(errno) => {
+            return Ok(());
+        }
+        Err(err) => bail!("failed to read file attributes: {}", err),
+    }
+
+    metadata.stat.flags |= flags::feature_flags_from_chattr(attr as u32);
+
+    Ok(())
+}
+
+fn get_fat_attr(metadata: &mut Metadata, fd: RawFd, fs_magic: i64) -> Result<(), Error> {
+    use proxmox::sys::linux::magic::*;
+
+    if fs_magic != MSDOS_SUPER_MAGIC && fs_magic != FUSE_SUPER_MAGIC {
+        return Ok(());
+    }
+
+    let mut attr: u32 = 0;
+
+    match unsafe { fs::read_fat_attr_fd(fd, &mut attr) } {
+        Ok(_) => (),
+        Err(nix::Error::Sys(errno)) if errno_is_unsupported(errno) => {
+            return Ok(());
+        }
+        Err(err) => bail!("failed to read fat attributes: {}", err),
+    }
+
+    metadata.stat.flags |= flags::feature_flags_from_fat_attr(attr);
+
+    Ok(())
+}
+
+/// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
+fn get_quota_project_id(
+    metadata: &mut Metadata,
+    fd: RawFd,
+    flags: u64,
+    magic: i64,
+) -> Result<(), Error> {
+    if !(metadata.is_dir() || metadata.is_regular_file()) {
+        return Ok(());
+    }
+
+    if 0 == (flags & flags::WITH_QUOTA_PROJID) {
+        return Ok(());
+    }
+
+    use proxmox::sys::linux::magic::*;
+
+    match magic {
+        EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => (),
+        _ => return Ok(()),
+    }
+
+    let mut fsxattr = fs::FSXAttr::default();
+    let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
+
+    // On some FUSE filesystems it can happen that ioctl is not supported.
+    // For these cases projid is set to 0 while the error is ignored.
+    if let Err(err) = res {
+        let errno = err
+            .as_errno()
+            .ok_or_else(|| format_err!("error while reading quota project id"))?;
+        if errno_is_unsupported(errno) {
+            return Ok(());
+        } else {
+            bail!("error while reading quota project id ({})", errno);
+        }
+    }
+
+    let projid = fsxattr.fsx_projid as u64;
+    if projid != 0 {
+        metadata.quota_project_id = Some(pxar::format::QuotaProjectId { projid });
+    }
+    Ok(())
+}
+
+fn get_acl(metadata: &mut Metadata, proc_path: &Path, flags: u64) -> Result<(), Error> {
+    if 0 == (flags & flags::WITH_ACL) {
+        return Ok(());
+    }
+
+    if metadata.is_symlink() {
+        return Ok(());
+    }
+
+    get_acl_do(metadata, proc_path, acl::ACL_TYPE_ACCESS)?;
+
+    if metadata.is_dir() {
+        get_acl_do(metadata, proc_path, acl::ACL_TYPE_DEFAULT)?;
+    }
+
+    Ok(())
+}
+
+fn get_acl_do(
+    metadata: &mut Metadata,
+    proc_path: &Path,
+    acl_type: acl::ACLType,
+) -> Result<(), Error> {
+    // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
+    // to create a path for acl_get_file(). acl_get_fd() only allows to get
+    // ACL_TYPE_ACCESS attributes.
+    let acl = match acl::ACL::get_file(&proc_path, acl_type) {
+        Ok(acl) => acl,
+        // Don't bail if underlying endpoint does not support acls
+        Err(Errno::EOPNOTSUPP) => return Ok(()),
+        // Don't bail if the endpoint cannot carry acls
+        Err(Errno::EBADF) => return Ok(()),
+        // Don't bail if there is no data
+        Err(Errno::ENODATA) => return Ok(()),
+        Err(err) => bail!("error while reading ACL - {}", err),
+    };
+
+    process_acl(metadata, acl, acl_type)
+}
+
+fn process_acl(
+    metadata: &mut Metadata,
+    acl: acl::ACL,
+    acl_type: acl::ACLType,
+) -> Result<(), Error> {
+    use pxar::format::acl as pxar_acl;
+    use pxar::format::acl::{Group, GroupObject, Permissions, User};
+
+    let mut acl_user = Vec::new();
+    let mut acl_group = Vec::new();
+    let mut acl_group_obj = None;
+    let mut acl_default = None;
+    let mut user_obj_permissions = None;
+    let mut group_obj_permissions = None;
+    let mut other_permissions = None;
+    let mut mask_permissions = None;
+
+    for entry in &mut acl.entries() {
+        let tag = entry.get_tag_type()?;
+        let permissions = entry.get_permissions()?;
+        match tag {
+            acl::ACL_USER_OBJ => user_obj_permissions = Some(Permissions(permissions)),
+            acl::ACL_GROUP_OBJ => group_obj_permissions = Some(Permissions(permissions)),
+            acl::ACL_OTHER => other_permissions = Some(Permissions(permissions)),
+            acl::ACL_MASK => mask_permissions = Some(Permissions(permissions)),
+            acl::ACL_USER => {
+                acl_user.push(User {
+                    uid: entry.get_qualifier()?,
+                    permissions: Permissions(permissions),
+                });
+            }
+            acl::ACL_GROUP => {
+                acl_group.push(Group {
+                    gid: entry.get_qualifier()?,
+                    permissions: Permissions(permissions),
+                });
+            }
+            _ => bail!("Unexpected ACL tag encountered!"),
+        }
+    }
+
+    acl_user.sort();
+    acl_group.sort();
+
+    match acl_type {
+        acl::ACL_TYPE_ACCESS => {
+            // The mask permissions are mapped to the stat group permissions
+            // in case that the ACL group permissions were set.
+            // Only in that case we need to store the group permissions,
+            // in the other cases they are identical to the stat group permissions.
+            if let (Some(gop), true) = (group_obj_permissions, mask_permissions.is_some()) {
+                acl_group_obj = Some(GroupObject { permissions: gop });
+            }
+
+            metadata.acl.users = acl_user;
+            metadata.acl.groups = acl_group;
+        }
+        acl::ACL_TYPE_DEFAULT => {
+            if user_obj_permissions != None
+                || group_obj_permissions != None
+                || other_permissions != None
+                || mask_permissions != None
+            {
+                acl_default = Some(pxar_acl::Default {
+                    // The value is set to UINT64_MAX as placeholder if one
+                    // of the permissions is not set
+                    user_obj_permissions: user_obj_permissions.unwrap_or(Permissions::NO_MASK),
+                    group_obj_permissions: group_obj_permissions.unwrap_or(Permissions::NO_MASK),
+                    other_permissions: other_permissions.unwrap_or(Permissions::NO_MASK),
+                    mask_permissions: mask_permissions.unwrap_or(Permissions::NO_MASK),
+                });
+            }
+
+            metadata.acl.default_users = acl_user;
+            metadata.acl.default_groups = acl_group;
+        }
+        _ => bail!("Unexpected ACL type encountered"),
+    }
+
+    metadata.acl.group_obj = acl_group_obj;
+    metadata.acl.default = acl_default;
+
+    Ok(())
+}

src/pxar/decoder.rs

@@ -1,365 +0,0 @@
-//! *pxar* format decoder for seekable files
-//!
-//! This module contain the code to decode *pxar* archive files.
-
-use std::convert::TryFrom;
-use std::ffi::{OsString, OsStr};
-use std::io::{Read, Seek, SeekFrom};
-use std::path::{Path, PathBuf};
-use std::os::unix::ffi::OsStrExt;
-
-use anyhow::{bail, format_err, Error};
-use libc;
-
-use super::binary_search_tree::search_binary_tree_by;
-use super::format_definition::*;
-use super::sequential_decoder::SequentialDecoder;
-use super::match_pattern::MatchPattern;
-
-use proxmox::tools::io::ReadExt;
-
-pub struct DirectoryEntry {
-    /// Points to the `PxarEntry` of the directory
-    start: u64,
-    /// Points past the goodbye table tail
-    end: u64,
-    /// Filename of entry
-    pub filename: OsString,
-    /// Entry (mode, permissions)
-    pub entry: PxarEntry,
-    /// Extended attributes
-    pub xattr: PxarAttributes,
-    /// Payload size
-    pub size: u64,
-    /// Target path for symbolic links
-    pub target: Option<PathBuf>,
-    /// Start offset of the payload if present.
-    pub payload_offset: Option<u64>,
-}
-
-/// Trait to create ReadSeek Decoder trait objects.
-trait ReadSeek: Read + Seek {}
-impl <R: Read + Seek> ReadSeek for R {}
-
-// This one needs Read+Seek
-pub struct Decoder {
-    inner: SequentialDecoder<Box<dyn ReadSeek + Send>>,
-    root_start: u64,
-    root_end: u64,
-}
-
-const HEADER_SIZE: u64 = std::mem::size_of::<PxarHeader>() as u64;
-const GOODBYE_ITEM_SIZE: u64 = std::mem::size_of::<PxarGoodbyeItem>() as u64;
-
-impl Decoder {
-    pub fn new<R: Read + Seek + Send + 'static>(mut reader: R) -> Result<Self, Error> {
-        let root_end = reader.seek(SeekFrom::End(0))?;
-        let boxed_reader: Box<dyn ReadSeek + 'static + Send> = Box::new(reader);
-        let inner = SequentialDecoder::new(boxed_reader, super::flags::DEFAULT);
-  
-        Ok(Self { inner, root_start: 0, root_end })
-    }
-
-    pub fn set_callback<F: Fn(&Path) -> Result<(), Error> + Send + 'static>(&mut self, callback: F ) {
-        self.inner.set_callback(callback);
-    }
-
-    pub fn root(&mut self) -> Result<DirectoryEntry, Error> {
-        self.seek(SeekFrom::Start(0))?;
-        let header: PxarHeader = self.inner.read_item()?;
-        check_ca_header::<PxarEntry>(&header, PXAR_ENTRY)?;
-        let entry: PxarEntry = self.inner.read_item()?;
-        let (header, xattr) = self.inner.read_attributes()?;
-        let (size, payload_offset) = match header.htype {
-            PXAR_PAYLOAD => (header.size - HEADER_SIZE, Some(self.seek(SeekFrom::Current(0))?)),
-            _ => (0, None),
-        };
-
-        Ok(DirectoryEntry {
-            start: self.root_start,
-            end: self.root_end,
-            filename: OsString::new(), // Empty
-            entry,
-            xattr,
-            size,
-            target: None,
-            payload_offset,
-        })
-    }
-
-    fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
-        let pos = self.inner.get_reader_mut().seek(pos)?;
-        Ok(pos)
-    }
-
-    pub(crate) fn root_end_offset(&self) -> u64 {
-        self.root_end
-    }
-
-    /// Restore the subarchive starting at `dir` to the provided target `path`.
-    ///
-    /// Only restore the content matched by the MatchPattern `pattern`.
-    /// An empty Vec `pattern` means restore all.
-    pub fn restore(&mut self, dir: &DirectoryEntry, path: &Path, pattern: &Vec<MatchPattern>) -> Result<(), Error> {
-        let start = dir.start;
-        self.seek(SeekFrom::Start(start))?;
-        self.inner.restore(path, pattern)?;
-
-        Ok(())
-    }
-
-    pub(crate) fn read_directory_entry(
-        &mut self,
-        start: u64,
-        end: u64,
-    ) -> Result<DirectoryEntry, Error> {
-        self.seek(SeekFrom::Start(start))?;
-
-        let head: PxarHeader = self.inner.read_item()?;
-
-        if head.htype != PXAR_FILENAME {
-            bail!("wrong filename header type for object [{}..{}]", start, end);
-        }
-
-        let entry_start = start + head.size;
-
-        let filename = self.inner.read_filename(head.size)?;
-
-        let head: PxarHeader = self.inner.read_item()?;
-        if head.htype == PXAR_FORMAT_HARDLINK {
-            let (_, offset) = self.inner.read_hardlink(head.size)?;
-            // TODO: Howto find correct end offset for hardlink target?
-            // This is a bit tricky since we cannot find correct end in an efficient
-            // way, on the other hand it doesn't really matter (for now) since target
-            // is never a directory and end is not used in such cases.
-            return self.read_directory_entry(start - offset, end);
-        }
-        check_ca_header::<PxarEntry>(&head, PXAR_ENTRY)?;
-        let entry: PxarEntry = self.inner.read_item()?;
-        let (header, xattr) = self.inner.read_attributes()?;
-        let (size, payload_offset, target) = match header.htype {
-            PXAR_PAYLOAD =>
-                (header.size - HEADER_SIZE, Some(self.seek(SeekFrom::Current(0))?), None),
-            PXAR_SYMLINK =>
-                (header.size - HEADER_SIZE, None, Some(self.inner.read_link(header.size)?)),
-            _ => (0, None, None),
-        };
-
-        Ok(DirectoryEntry {
-            start: entry_start,
-            end,
-            filename,
-            entry,
-            xattr,
-            size,
-            target,
-            payload_offset,
-        })
-    }
-
-    /// Return the goodbye table based on the provided end offset.
-    ///
-    /// Get the goodbye table entries and the start and end offsets of the
-    /// items they reference.
-    /// If the start offset is provided, we use that to check the consistency of
-    /// the data, else the start offset calculated based on the goodbye tail is
-    /// used.
-    pub(crate) fn goodbye_table(
-        &mut self,
-        start: Option<u64>,
-        end: u64,
-    ) -> Result<Vec<(PxarGoodbyeItem, u64, u64)>, Error> {
-        self.seek(SeekFrom::Start(end - GOODBYE_ITEM_SIZE))?;
-
-        let tail: PxarGoodbyeItem = self.inner.read_item()?;
-        if tail.hash != PXAR_GOODBYE_TAIL_MARKER {
-            bail!("missing goodbye tail marker for object at offset {}", end);
-        }
-
-        // If the start offset was provided, we use and check based on that.
-        // If not, we rely on the offset calculated from the goodbye table entry.
-        let start = start.unwrap_or(end - tail.offset - tail.size);
-        let goodbye_table_size = tail.size;
-        if goodbye_table_size < (HEADER_SIZE + GOODBYE_ITEM_SIZE) {
-            bail!("short goodbye table size for object [{}..{}]", start, end);
-        }
-
-        let goodbye_inner_size = goodbye_table_size - HEADER_SIZE - GOODBYE_ITEM_SIZE;
-        if (goodbye_inner_size % GOODBYE_ITEM_SIZE) != 0 {
-            bail!(
-                "wrong goodbye inner table size for entry [{}..{}]",
-                start,
-                end
-            );
-        }
-
-        let goodbye_start = end - goodbye_table_size;
-        if tail.offset != (goodbye_start - start) {
-            bail!(
-                "wrong offset in goodbye tail marker for entry [{}..{}]",
-                start,
-                end
-            );
-        }
-
-        self.seek(SeekFrom::Start(goodbye_start))?;
-        let head: PxarHeader = self.inner.read_item()?;
-        if head.htype != PXAR_GOODBYE {
-            bail!(
-                "wrong goodbye table header type for entry [{}..{}]",
-                start,
-                end
-            );
-        }
-
-        if head.size != goodbye_table_size {
-            bail!("wrong goodbye table size for entry [{}..{}]", start, end);
-        }
-
-        let mut gb_entries = Vec::new();
-        for i in 0..goodbye_inner_size / GOODBYE_ITEM_SIZE {
-            let item: PxarGoodbyeItem = self.inner.read_item()?;
-            if item.offset > (goodbye_start - start) {
-                bail!(
-                    "goodbye entry {} offset out of range [{}..{}] {} {} {}",
-                    i,
-                    start,
-                    end,
-                    item.offset,
-                    goodbye_start,
-                    start
-                );
-            }
-            let item_start = goodbye_start - item.offset;
-            let item_end = item_start + item.size;
-            if item_end > goodbye_start {
-                bail!("goodbye entry {} end out of range [{}..{}]", i, start, end);
-            }
-            gb_entries.push((item, item_start, item_end));
-        }
-
-        Ok(gb_entries)
-    }
-
-    pub fn list_dir(&mut self, dir: &DirectoryEntry) -> Result<Vec<DirectoryEntry>, Error> {
-        let start = dir.start;
-        let end = dir.end;
-
-        //println!("list_dir1: {} {}", start, end);
-
-        if (end - start) < (HEADER_SIZE + GOODBYE_ITEM_SIZE) {
-            bail!("detected short object [{}..{}]", start, end);
-        }
-
-        let mut result = vec![];
-        let goodbye_table = self.goodbye_table(Some(start), end)?;
-        for (_, item_start, item_end) in goodbye_table {
-            let entry = self.read_directory_entry(item_start, item_end)?;
-            //println!("ENTRY: {} {} {:?}", item_start, item_end, entry.filename);
-            result.push(entry);
-        }
-
-        Ok(result)
-    }
-
-    pub fn print_filenames<W: std::io::Write>(
-        &mut self,
-        output: &mut W,
-        prefix: &mut PathBuf,
-        dir: &DirectoryEntry,
-    ) -> Result<(), Error> {
-        let mut list = self.list_dir(dir)?;
-
-        list.sort_unstable_by(|a, b| a.filename.cmp(&b.filename));
-
-        for item in &list {
-            prefix.push(item.filename.clone());
-
-            let mode = item.entry.mode as u32;
-
-            let ifmt = mode & libc::S_IFMT;
-
-            writeln!(output, "{:?}", prefix)?;
-
-            match ifmt {
-                libc::S_IFDIR => self.print_filenames(output, prefix, item)?,
-                libc::S_IFREG | libc::S_IFLNK | libc::S_IFBLK | libc::S_IFCHR => {}
-                _ => bail!("unknown item mode/type for {:?}", prefix),
-            }
-
-            prefix.pop();
-        }
-
-        Ok(())
-    }
-
-    /// Lookup the item identified by `filename` in the provided `DirectoryEntry`.
-    ///
-    /// Calculates the hash of the filename and searches for matching entries in
-    /// the goodbye table of the provided `DirectoryEntry`.
-    /// If found, also the filename is compared to avoid hash collision.
-    /// If the filename does not match, the search resumes with the next entry in
-    /// the goodbye table.
-    /// If there is no entry with matching `filename`, `Ok(None)` is returned.
-    pub fn lookup(
-        &mut self,
-        dir: &DirectoryEntry,
-        filename: &OsStr,
-    ) -> Result<Option<DirectoryEntry>, Error> {
-        let gbt = self.goodbye_table(Some(dir.start), dir.end)?;
-        let hash = compute_goodbye_hash(filename.as_bytes());
-
-        let mut start_idx = 0;
-        let mut skip_multiple = 0;
-        loop {
-            // Search for the next goodbye entry with matching hash.
-            let idx = search_binary_tree_by(
-                start_idx,
-                gbt.len(),
-                skip_multiple,
-                |idx| hash.cmp(&gbt[idx].0.hash),
-            );
-            let (_item, start, end) = match idx {
-                Some(idx) => &gbt[idx],
-                None => return Ok(None),
-            };
-
-            let entry = self.read_directory_entry(*start, *end)?;
-
-            // Possible hash collision, need to check if the found entry is indeed
-            // the filename to lookup.
-            if entry.filename == filename {
-                return Ok(Some(entry));
-            }
-            // Hash collision, check the next entry in the goodbye table by starting
-            // from given index but skipping one more match (so hash at index itself).
-            start_idx = idx.unwrap();
-            skip_multiple = 1;
-        }
-    }
-
-    /// Read the payload of the file given by `entry`.
-    ///
-    /// This will read a files payload as raw bytes starting from `offset` after
-    /// the payload marker, reading `size` bytes.
-    /// If the payload from `offset` to EOF is smaller than `size` bytes, the
-    /// buffer with reduced size is returned.
-    /// If `offset` is larger than the payload size of the `DirectoryEntry`, an
-    /// empty buffer is returned.
-    pub fn read(&mut self, entry: &DirectoryEntry, size: usize, offset: u64) -> Result<Vec<u8>, Error> {
-        let start_offset = entry.payload_offset
-            .ok_or_else(|| format_err!("entry has no payload offset"))?;
-        if offset >= entry.size {
-            return Ok(Vec::new());
-        }
-        let len = if u64::try_from(size)? > entry.size {
-            usize::try_from(entry.size)?
-        } else {
-            size
-        };
-        self.seek(SeekFrom::Start(start_offset + offset))?;
-        let data = self.inner.get_reader_mut().read_exact_allocated(len)?;
-
-        Ok(data)
-    }
-}

src/pxar/dir_stack.rs

@@ -1,118 +1,141 @@
-use std::ffi::{OsStr, OsString};
+use std::ffi::OsString;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::path::PathBuf;
 
-use anyhow::{format_err, Error};
+use anyhow::{bail, format_err, Error};
-use nix::errno::Errno;
+use nix::dir::Dir;
 use nix::fcntl::OFlag;
-use nix::sys::stat::Mode;
+use nix::sys::stat::{mkdirat, Mode};
-use nix::NixPath;
 
-use super::format_definition::{PxarAttributes, PxarEntry};
+use proxmox::sys::error::SysError;
+use pxar::Metadata;
+
+use crate::pxar::tools::{assert_relative_path, perms_from_metadata};
 
 pub struct PxarDir {
-    pub filename: OsString,
+    file_name: OsString,
-    pub entry: PxarEntry,
+    metadata: Metadata,
-    pub attr: PxarAttributes,
+    dir: Option<Dir>,
-    pub dir: Option<nix::dir::Dir>,
-}
-
-pub struct PxarDirStack {
-    root: RawFd,
-    data: Vec<PxarDir>,
 }
 
 impl PxarDir {
-    pub fn new(filename: &OsStr, entry: PxarEntry, attr: PxarAttributes) -> Self {
+    pub fn new(file_name: OsString, metadata: Metadata) -> Self {
         Self {
-            filename: filename.to_os_string(),
+            file_name,
-            entry,
+            metadata,
-            attr,
             dir: None,
         }
     }
 
-    fn create_dir(&self, parent: RawFd, create_new: bool) -> Result<nix::dir::Dir, nix::Error> {
+    pub fn with_dir(dir: Dir, metadata: Metadata) -> Self {
-        let res = self
+        Self {
-            .filename
+            file_name: OsString::from("."),
-            .with_nix_path(|cstr| unsafe { libc::mkdirat(parent, cstr.as_ptr(), libc::S_IRWXU) })?;
+            metadata,
-
+            dir: Some(dir),
-        match Errno::result(res) {
-            Ok(_) => {}
-            Err(err) => {
-                if err == nix::Error::Sys(nix::errno::Errno::EEXIST) {
-                    if create_new {
-                        return Err(err);
-                    }
-                } else {
-                    return Err(err);
-                }
         }
     }
 
-        let dir = nix::dir::Dir::openat(
+    fn create_dir(&mut self, parent: RawFd, allow_existing_dirs: bool) -> Result<RawFd, Error> {
+        match mkdirat(
             parent,
-            self.filename.as_os_str(),
+            self.file_name.as_os_str(),
+            perms_from_metadata(&self.metadata)?,
+        ) {
+            Ok(()) => (),
+            Err(err) => {
+                if !(allow_existing_dirs && err.already_exists()) {
+                    return Err(err.into());
+                }
+            }
+        }
+
+        self.open_dir(parent)
+    }
+
+    fn open_dir(&mut self, parent: RawFd) -> Result<RawFd, Error> {
+        let dir = Dir::openat(
+            parent,
+            self.file_name.as_os_str(),
             OFlag::O_DIRECTORY,
             Mode::empty(),
         )?;
 
-        Ok(dir)
+        let fd = dir.as_raw_fd();
+        self.dir = Some(dir);
+
+        Ok(fd)
     }
+
+    pub fn try_as_raw_fd(&self) -> Option<RawFd> {
+        self.dir.as_ref().map(AsRawFd::as_raw_fd)
+    }
+
+    pub fn metadata(&self) -> &Metadata {
+        &self.metadata
+    }
+}
+
+pub struct PxarDirStack {
+    dirs: Vec<PxarDir>,
+    path: PathBuf,
+    created: usize,
 }
 
 impl PxarDirStack {
-    pub fn new(parent: RawFd) -> Self {
+    pub fn new(root: Dir, metadata: Metadata) -> Self {
         Self {
-            root: parent,
+            dirs: vec![PxarDir::with_dir(root, metadata)],
-            data: Vec::new(),
+            path: PathBuf::from("/"),
+            created: 1, // the root directory exists
         }
     }
 
-    pub fn push(&mut self, dir: PxarDir) {
+    pub fn is_empty(&self) -> bool {
-        self.data.push(dir);
+        self.dirs.is_empty()
     }
 
-    pub fn pop(&mut self) -> Option<PxarDir> {
+    pub fn push(&mut self, file_name: OsString, metadata: Metadata) -> Result<(), Error> {
-        self.data.pop()
+        assert_relative_path(&file_name)?;
+        self.path.push(&file_name);
+        self.dirs.push(PxarDir::new(file_name, metadata));
+        Ok(())
     }
 
-    pub fn as_path_buf(&self) -> PathBuf {
+    pub fn pop(&mut self) -> Result<Option<PxarDir>, Error> {
-        let path: PathBuf = self.data.iter().map(|d| d.filename.clone()).collect();
+        let out = self.dirs.pop();
-        path
+        if !self.path.pop() {
+            if self.path.as_os_str() == "/" {
+                // we just finished the root directory, make sure this can only happen once:
+                self.path = PathBuf::new();
+            } else {
+                bail!("lost track of path");
+            }
+        }
+        self.created = self.created.min(self.dirs.len());
+        Ok(out)
     }
 
-    pub fn last(&self) -> Option<&PxarDir> {
+    pub fn last_dir_fd(&mut self, allow_existing_dirs: bool) -> Result<RawFd, Error> {
-        self.data.last()
+        // should not be possible given the way we use it:
+        assert!(!self.dirs.is_empty(), "PxarDirStack underrun");
+
+        let mut fd = self.dirs[self.created - 1]
+            .try_as_raw_fd()
+            .ok_or_else(|| format_err!("lost track of directory file descriptors"))?;
+        while self.created < self.dirs.len() {
+            fd = self.dirs[self.created].create_dir(fd, allow_existing_dirs)?;
+            self.created += 1;
         }
 
-    pub fn last_mut(&mut self) -> Option<&mut PxarDir> {
+        Ok(fd)
-        self.data.last_mut()
     }
 
-    pub fn last_dir_fd(&self) -> Option<RawFd> {
+    pub fn root_dir_fd(&self) -> Result<RawFd, Error> {
-        let last_dir = self.data.last()?;
+        // should not be possible given the way we use it:
-        match &last_dir.dir {
+        assert!(!self.dirs.is_empty(), "PxarDirStack underrun");
-            Some(d) => Some(d.as_raw_fd()),
-            None => None,
-        }
-    }
 
-    pub fn create_all_dirs(&mut self, create_new: bool) -> Result<RawFd, Error> {
+        self.dirs[0]
-        let mut current_fd = self.root;
+            .try_as_raw_fd()
-        for d in &mut self.data {
+            .ok_or_else(|| format_err!("lost track of directory file descriptors"))
-            match &d.dir {
-                Some(dir) => current_fd = dir.as_raw_fd(),
-                None => {
-                    let dir = d
-                        .create_dir(current_fd, create_new)
-                        .map_err(|err| format_err!("create dir failed - {}", err))?;
-                    current_fd = dir.as_raw_fd();
-                    d.dir = Some(dir);
-                }
-            }
-        }
-
-        Ok(current_fd)
     }
 }

src/pxar/extract.rs

@@ -0,0 +1,306 @@
+//! Code for extraction of pxar contents onto the file system.
+
+use std::convert::TryFrom;
+use std::ffi::{CStr, CString, OsStr};
+use std::io;
+use std::os::unix::ffi::OsStrExt;
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use std::path::Path;
+
+use anyhow::{bail, format_err, Error};
+use nix::dir::Dir;
+use nix::fcntl::OFlag;
+use nix::sys::stat::Mode;
+
+use pathpatterns::{MatchEntry, MatchList, MatchType};
+use pxar::format::Device;
+use pxar::Metadata;
+
+use proxmox::c_result;
+use proxmox::tools::fs::{create_path, CreateOptions};
+
+use crate::pxar::dir_stack::PxarDirStack;
+use crate::pxar::flags;
+use crate::pxar::metadata;
+
+struct Extractor<'a> {
+    /// FIXME: use bitflags!() for feature_flags
+    feature_flags: u64,
+    allow_existing_dirs: bool,
+    callback: &'a mut dyn FnMut(&Path),
+    dir_stack: PxarDirStack,
+}
+
+impl<'a> Extractor<'a> {
+    fn with_flag(&self, flag: u64) -> bool {
+        flag == (self.feature_flags & flag)
+    }
+}
+
+pub fn extract_archive<T, F>(
+    mut decoder: pxar::decoder::Decoder<T>,
+    destination: &Path,
+    match_list: &[MatchEntry],
+    feature_flags: u64,
+    allow_existing_dirs: bool,
+    mut callback: F,
+) -> Result<(), Error>
+where
+    T: pxar::decoder::SeqRead,
+    F: FnMut(&Path),
+{
+    // we use this to keep track of our directory-traversal
+    decoder.enable_goodbye_entries(true);
+
+    let root = decoder
+        .next()
+        .ok_or_else(|| format_err!("found empty pxar archive"))?
+        .map_err(|err| format_err!("error reading pxar archive: {}", err))?;
+
+    if !root.is_dir() {
+        bail!("pxar archive does not start with a directory entry!");
+    }
+
+    create_path(
+        &destination,
+        None,
+        Some(CreateOptions::new().perm(Mode::from_bits_truncate(0o700))),
+    )
+    .map_err(|err| format_err!("error creating directory {:?}: {}", destination, err))?;
+
+    let dir = Dir::open(
+        destination,
+        OFlag::O_DIRECTORY | OFlag::O_CLOEXEC,
+        Mode::empty(),
+    )
+    .map_err(|err| format_err!("unable to open target directory {:?}: {}", destination, err,))?;
+
+    let mut extractor = Extractor {
+        feature_flags,
+        allow_existing_dirs,
+        callback: &mut callback,
+        dir_stack: PxarDirStack::new(dir, root.metadata().clone()),
+    };
+
+    let mut match_stack = Vec::new();
+    let mut current_match = true;
+    while let Some(entry) = decoder.next() {
+        use pxar::EntryKind;
+
+        let entry = entry.map_err(|err| format_err!("error reading pxar archive: {}", err))?;
+
+        let file_name_os = entry.file_name();
+
+        // safety check: a file entry in an archive must never contain slashes:
+        if file_name_os.as_bytes().contains(&b'/') {
+            bail!("archive file entry contains slashes, which is invalid and a security concern");
+        }
+
+        let file_name = CString::new(file_name_os.as_bytes())
+            .map_err(|_| format_err!("encountered file name with null-bytes"))?;
+
+        let metadata = entry.metadata();
+
+        let match_result = match_list.matches(
+            entry.path().as_os_str().as_bytes(),
+            Some(metadata.file_type() as u32),
+        );
+
+        let did_match = match match_result {
+            Some(MatchType::Include) => true,
+            Some(MatchType::Exclude) => false,
+            None => current_match,
+        };
+        match (did_match, entry.kind()) {
+            (_, EntryKind::Directory) => {
+                extractor.callback(entry.path());
+
+                extractor
+                    .dir_stack
+                    .push(file_name_os.to_owned(), metadata.clone())?;
+
+                if current_match && match_result != Some(MatchType::Exclude) {
+                    // We're currently in a positive match and this directory does not match an
+                    // exclude entry, so make sure it is created:
+                    let _ = extractor
+                        .dir_stack
+                        .last_dir_fd(extractor.allow_existing_dirs)
+                        .map_err(|err| {
+                            format_err!("error creating entry {:?}: {}", file_name_os, err)
+                        })?;
+                }
+
+                // We're starting a new directory, push our old matching state and replace it with
+                // our new one:
+                match_stack.push(current_match);
+                current_match = did_match;
+
+                Ok(())
+            }
+            (_, EntryKind::GoodbyeTable) => {
+                // go up a directory
+                let dir = extractor
+                    .dir_stack
+                    .pop()
+                    .map_err(|err| format_err!("unexpected end of directory entry: {}", err))?
+                    .ok_or_else(|| format_err!("broken pxar archive (directory stack underrun)"))?;
+                // We left a directory, also get back our previous matching state. This is in sync
+                // with `dir_stack` so this should never be empty except for the final goodbye
+                // table, in which case we get back to the default of `true`.
+                current_match = match_stack.pop().unwrap_or(true);
+
+                if let Some(fd) = dir.try_as_raw_fd() {
+                    metadata::apply(extractor.feature_flags, dir.metadata(), fd, &file_name)
+                } else {
+                    Ok(())
+                }
+            }
+            (true, EntryKind::Symlink(link)) => {
+                extractor.callback(entry.path());
+                extractor.extract_symlink(&file_name, metadata, link.as_ref())
+            }
+            (true, EntryKind::Hardlink(link)) => {
+                extractor.callback(entry.path());
+                extractor.extract_hardlink(&file_name, metadata, link.as_os_str())
+            }
+            (true, EntryKind::Device(dev)) => {
+                if extractor.with_flag(flags::WITH_DEVICE_NODES) {
+                    extractor.callback(entry.path());
+                    extractor.extract_device(&file_name, metadata, dev)
+                } else {
+                    Ok(())
+                }
+            }
+            (true, EntryKind::Fifo) => {
+                if extractor.with_flag(flags::WITH_FIFOS) {
+                    extractor.callback(entry.path());
+                    extractor.extract_special(&file_name, metadata, 0)
+                } else {
+                    Ok(())
+                }
+            }
+            (true, EntryKind::Socket) => {
+                if extractor.with_flag(flags::WITH_SOCKETS) {
+                    extractor.callback(entry.path());
+                    extractor.extract_special(&file_name, metadata, 0)
+                } else {
+                    Ok(())
+                }
+            }
+            (true, EntryKind::File { size, .. }) => extractor.extract_file(
+                &file_name,
+                metadata,
+                *size,
+                &mut decoder.contents().ok_or_else(|| {
+                    format_err!("found regular file entry without contents in archive")
+                })?,
+            ),
+            (false, _) => Ok(()), // skip this
+        }
+        .map_err(|err| format_err!("error at entry {:?}: {}", file_name_os, err))?;
+    }
+
+    if !extractor.dir_stack.is_empty() {
+        bail!("unexpected eof while decoding pxar archive");
+    }
+
+    Ok(())
+}
+
+impl<'a> Extractor<'a> {
+    fn parent_fd(&mut self) -> Result<RawFd, Error> {
+        self.dir_stack.last_dir_fd(self.allow_existing_dirs)
+    }
+
+    fn callback(&mut self, path: &Path) {
+        (self.callback)(path)
+    }
+
+    fn extract_symlink(
+        &mut self,
+        file_name: &CStr,
+        metadata: &Metadata,
+        link: &OsStr,
+    ) -> Result<(), Error> {
+        let parent = self.parent_fd()?;
+        nix::unistd::symlinkat(link, Some(parent), file_name)?;
+        metadata::apply_at(self.feature_flags, metadata, parent, file_name)
+    }
+
+    fn extract_hardlink(
+        &mut self,
+        file_name: &CStr,
+        _metadata: &Metadata, // for now we don't use this because hardlinks don't need it...
+        link: &OsStr,
+    ) -> Result<(), Error> {
+        crate::pxar::tools::assert_relative_path(link)?;
+
+        let parent = self.parent_fd()?;
+        let root = self.dir_stack.root_dir_fd()?;
+        let target = CString::new(link.as_bytes())?;
+        nix::unistd::linkat(
+            Some(root),
+            target.as_c_str(),
+            Some(parent),
+            file_name,
+            nix::unistd::LinkatFlags::NoSymlinkFollow,
+        )?;
+
+        Ok(())
+    }
+
+    fn extract_device(
+        &mut self,
+        file_name: &CStr,
+        metadata: &Metadata,
+        device: &Device,
+    ) -> Result<(), Error> {
+        self.extract_special(file_name, metadata, device.to_dev_t())
+    }
+
+    fn extract_special(
+        &mut self,
+        file_name: &CStr,
+        metadata: &Metadata,
+        device: libc::dev_t,
+    ) -> Result<(), Error> {
+        let mode = metadata.stat.mode;
+        let mode = u32::try_from(mode).map_err(|_| {
+            format_err!(
+                "device node's mode contains illegal bits: 0x{:x} (0o{:o})",
+                mode,
+                mode,
+            )
+        })?;
+        let parent = self.parent_fd()?;
+        unsafe { c_result!(libc::mknodat(parent, file_name.as_ptr(), mode, device)) }
+            .map_err(|err| format_err!("failed to create device node: {}", err))?;
+
+        metadata::apply_at(self.feature_flags, metadata, parent, file_name)
+    }
+
+    fn extract_file(
+        &mut self,
+        file_name: &CStr,
+        metadata: &Metadata,
+        size: u64,
+        contents: &mut dyn io::Read,
+    ) -> Result<(), Error> {
+        let parent = self.parent_fd()?;
+        let mut file = unsafe {
+            std::fs::File::from_raw_fd(nix::fcntl::openat(
+                parent,
+                file_name,
+                OFlag::O_CREAT | OFlag::O_WRONLY | OFlag::O_CLOEXEC,
+                Mode::from_bits(0o600).unwrap(),
+            )?)
+        };
+
+        let extracted = io::copy(&mut *contents, &mut file)?;
+        if size != extracted {
+            bail!("extracted {} bytes of a file of {} bytes", extracted, size);
+        }
+
+        metadata::apply(self.feature_flags, metadata, file.as_raw_fd(), file_name)
+    }
+}

src/pxar/flags.rs

@@ -3,6 +3,8 @@
 //! Flags for known supported features for a given filesystem can be derived
 //! from the superblocks magic number.
 
+// FIXME: use bitflags!() here!
+
 /// FAT-style 2s time granularity
 pub const WITH_2SEC_TIME: u64                   = 0x40;
 /// Preserve read only flag of files

src/pxar/format_definition.rs

@@ -1,263 +0,0 @@
-//! *pxar* binary format definition
-//!
-//! Please note the all values are stored in little endian ordering.
-//!
-//! The Archive contains a list of items. Each item starts with a
-//! `PxarHeader`, followed by the item data.
-use std::cmp::Ordering;
-
-use endian_trait::Endian;
-use anyhow::{bail, Error};
-use siphasher::sip::SipHasher24;
-
-
-/// Header types identifying items stored in the archive
-pub const PXAR_ENTRY: u64 = 0x1396fabcea5bbb51;
-pub const PXAR_FILENAME: u64 = 0x6dbb6ebcb3161f0b;
-pub const PXAR_SYMLINK: u64 = 0x664a6fb6830e0d6c;
-pub const PXAR_DEVICE: u64 = 0xac3dace369dfe643;
-pub const PXAR_XATTR: u64 = 0xb8157091f80bc486;
-pub const PXAR_ACL_USER: u64 = 0x297dc88b2ef12faf;
-pub const PXAR_ACL_GROUP: u64 = 0x36f2acb56cb3dd0b;
-pub const PXAR_ACL_GROUP_OBJ: u64 = 0x23047110441f38f3;
-pub const PXAR_ACL_DEFAULT: u64 = 0xfe3eeda6823c8cd0;
-pub const PXAR_ACL_DEFAULT_USER: u64 = 0xbdf03df9bd010a91;
-pub const PXAR_ACL_DEFAULT_GROUP: u64 = 0xa0cb1168782d1f51;
-pub const PXAR_FCAPS: u64 = 0xf7267db0afed0629;
-pub const PXAR_QUOTA_PROJID: u64 = 0x161baf2d8772a72b;
-
-/// Marks item as hardlink
-/// compute_goodbye_hash(b"__PROXMOX_FORMAT_HARDLINK__");
-pub const PXAR_FORMAT_HARDLINK: u64 = 0x2c5e06f634f65b86;
-/// Marks the beginning of the payload (actual content) of regular files
-pub const PXAR_PAYLOAD: u64 = 0x8b9e1d93d6dcffc9;
-/// Marks item as entry of goodbye table
-pub const PXAR_GOODBYE: u64 = 0xdfd35c5e8327c403;
-/// The end marker used in the GOODBYE object
-pub const PXAR_GOODBYE_TAIL_MARKER: u64 = 0x57446fa533702943;
-
-#[derive(Debug, Endian)]
-#[repr(C)]
-pub struct PxarHeader {
-    /// The item type (see `PXAR_` constants).
-    pub htype: u64,
-    /// The size of the item, including the size of `PxarHeader`.
-    pub size: u64,
-}
-
-#[derive(Endian)]
-#[repr(C)]
-pub struct PxarEntry {
-    pub mode: u64,
-    pub flags: u64,
-    pub uid: u32,
-    pub gid: u32,
-    pub mtime: u64,
-}
-
-#[derive(Endian)]
-#[repr(C)]
-pub struct PxarDevice {
-    pub major: u64,
-    pub minor: u64,
-}
-
-#[derive(Endian)]
-#[repr(C)]
-pub struct PxarGoodbyeItem {
-    /// SipHash24 of the directory item name. The last GOODBYE item
-    /// uses the special hash value `PXAR_GOODBYE_TAIL_MARKER`.
-    pub hash: u64,
-    /// The offset from the start of the GOODBYE object to the start
-    /// of the matching directory item (point to a FILENAME). The last
-    /// GOODBYE item points to the start of the matching ENTRY
-    /// object.
-    pub offset: u64,
-    /// The overall size of the directory item. The last GOODBYE item
-    /// repeats the size of the GOODBYE item.
-    pub size: u64,
-}
-
-/// Helper function to extract file names from binary archive.
-pub fn read_os_string(buffer: &[u8]) -> std::ffi::OsString {
-    let len = buffer.len();
-
-    use std::os::unix::ffi::OsStrExt;
-
-    let name = if len > 0 && buffer[len - 1] == 0 {
-        std::ffi::OsStr::from_bytes(&buffer[0..len - 1])
-    } else {
-        std::ffi::OsStr::from_bytes(&buffer)
-    };
-
-    name.into()
-}
-
-#[derive(Debug, Eq)]
-#[repr(C)]
-pub struct PxarXAttr {
-    pub name: Vec<u8>,
-    pub value: Vec<u8>,
-}
-
-impl Ord for PxarXAttr {
-    fn cmp(&self, other: &PxarXAttr) -> Ordering {
-        self.name.cmp(&other.name)
-    }
-}
-
-impl PartialOrd for PxarXAttr {
-    fn partial_cmp(&self, other: &PxarXAttr) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl PartialEq for PxarXAttr {
-    fn eq(&self, other: &PxarXAttr) -> bool {
-        self.name == other.name
-    }
-}
-
-#[derive(Debug)]
-#[repr(C)]
-pub struct PxarFCaps {
-    pub data: Vec<u8>,
-}
-
-#[derive(Debug, Endian, Eq)]
-#[repr(C)]
-pub struct PxarACLUser {
-    pub uid: u64,
-    pub permissions: u64,
-    //pub name: Vec<u64>, not impl for now
-}
-
-// TODO if also name is impl, sort by uid, then by name and last by permissions
-impl Ord for PxarACLUser {
-    fn cmp(&self, other: &PxarACLUser) -> Ordering {
-        match self.uid.cmp(&other.uid) {
-            // uids are equal, entries ordered by permissions
-            Ordering::Equal => self.permissions.cmp(&other.permissions),
-            // uids are different, entries ordered by uid
-            uid_order => uid_order,
-        }
-    }
-}
-
-impl PartialOrd for PxarACLUser {
-    fn partial_cmp(&self, other: &PxarACLUser) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl PartialEq for PxarACLUser {
-    fn eq(&self, other: &PxarACLUser) -> bool {
-        self.uid == other.uid && self.permissions == other.permissions
-    }
-}
-
-#[derive(Debug, Endian, Eq)]
-#[repr(C)]
-pub struct PxarACLGroup {
-    pub gid: u64,
-    pub permissions: u64,
-    //pub name: Vec<u64>, not impl for now
-}
-
-// TODO if also name is impl, sort by gid, then by name and last by permissions
-impl Ord for PxarACLGroup {
-    fn cmp(&self, other: &PxarACLGroup) -> Ordering {
-        match self.gid.cmp(&other.gid) {
-            // gids are equal, entries are ordered by permissions
-            Ordering::Equal => self.permissions.cmp(&other.permissions),
-            // gids are different, entries ordered by gid
-            gid_ordering => gid_ordering,
-        }
-    }
-}
-
-impl PartialOrd for PxarACLGroup {
-    fn partial_cmp(&self, other: &PxarACLGroup) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl PartialEq for PxarACLGroup {
-    fn eq(&self, other: &PxarACLGroup) -> bool {
-        self.gid == other.gid && self.permissions == other.permissions
-    }
-}
-
-#[derive(Debug, Endian)]
-#[repr(C)]
-pub struct PxarACLGroupObj {
-    pub permissions: u64,
-}
-
-#[derive(Debug, Endian)]
-#[repr(C)]
-pub struct PxarACLDefault {
-    pub user_obj_permissions: u64,
-    pub group_obj_permissions: u64,
-    pub other_permissions: u64,
-    pub mask_permissions: u64,
-}
-
-pub(crate) struct PxarACL {
-    pub users: Vec<PxarACLUser>,
-    pub groups: Vec<PxarACLGroup>,
-    pub group_obj: Option<PxarACLGroupObj>,
-    pub default: Option<PxarACLDefault>,
-}
-
-pub const PXAR_ACL_PERMISSION_READ: u64 = 4;
-pub const PXAR_ACL_PERMISSION_WRITE: u64 = 2;
-pub const PXAR_ACL_PERMISSION_EXECUTE: u64 = 1;
-
-#[derive(Debug, Endian)]
-#[repr(C)]
-pub struct PxarQuotaProjID {
-    pub projid: u64,
-}
-
-#[derive(Debug, Default)]
-pub struct PxarAttributes {
-    pub xattrs: Vec<PxarXAttr>,
-    pub fcaps: Option<PxarFCaps>,
-    pub quota_projid: Option<PxarQuotaProjID>,
-    pub acl_user: Vec<PxarACLUser>,
-    pub acl_group: Vec<PxarACLGroup>,
-    pub acl_group_obj: Option<PxarACLGroupObj>,
-    pub acl_default: Option<PxarACLDefault>,
-    pub acl_default_user: Vec<PxarACLUser>,
-    pub acl_default_group: Vec<PxarACLGroup>,
-}
-
-/// Create SipHash values for goodby tables.
-//pub fn compute_goodbye_hash(name: &std::ffi::CStr) -> u64 {
-pub fn compute_goodbye_hash(name: &[u8]) -> u64 {
-    use std::hash::Hasher;
-    let mut hasher = SipHasher24::new_with_keys(0x8574442b0f1d84b3, 0x2736ed30d1c22ec1);
-    hasher.write(name);
-    hasher.finish()
-}
-
-pub fn check_ca_header<T>(head: &PxarHeader, htype: u64) -> Result<(), Error> {
-    if head.htype != htype {
-        bail!(
-            "got wrong header type ({:016x} != {:016x})",
-            head.htype,
-            htype
-        );
-    }
-    if head.size != (std::mem::size_of::<T>() + std::mem::size_of::<PxarHeader>()) as u64 {
-        bail!("got wrong header size for type {:016x}", htype);
-    }
-
-    Ok(())
-}
-
-/// The format requires to build sorted directory lookup tables in
-/// memory, so we restrict the number of allowed entries to limit
-/// maximum memory usage.
-pub const ENCODER_MAX_ENTRIES: usize = 1024 * 1024;

src/pxar/helper.rs

@@ -1,36 +0,0 @@
-use libc;
-use nix::sys::stat::FileStat;
-
-#[inline(always)]
-pub fn is_directory(stat: &FileStat) -> bool {
-    (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR
-}
-
-#[inline(always)]
-pub fn is_symlink(stat: &FileStat) -> bool {
-    (stat.st_mode & libc::S_IFMT) == libc::S_IFLNK
-}
-
-#[inline(always)]
-pub fn is_reg_file(stat: &FileStat) -> bool {
-    (stat.st_mode & libc::S_IFMT) == libc::S_IFREG
-}
-
-#[inline(always)]
-pub fn is_block_dev(stat: &FileStat) -> bool {
-    (stat.st_mode & libc::S_IFMT) == libc::S_IFBLK
-}
-
-#[inline(always)]
-pub fn is_char_dev(stat: &FileStat) -> bool {
-    (stat.st_mode & libc::S_IFMT) == libc::S_IFCHR
-}
-
-#[inline(always)]
-pub fn is_fifo(stat: &FileStat) -> bool {
-    (stat.st_mode & libc::S_IFMT) == libc::S_IFIFO
-}
-#[inline(always)]
-pub fn is_socket(stat: &FileStat) -> bool {
-    (stat.st_mode & libc::S_IFMT) == libc::S_IFSOCK
-}

src/pxar/match_pattern.rs

@@ -1,514 +0,0 @@
-//! `MatchPattern` defines a match pattern used to match filenames encountered
-//! during encoding or decoding of a `pxar` archive.
-//! `fnmatch` is used internally to match filenames against the patterns.
-//! Shell wildcard pattern can be used to match multiple filenames, see manpage
-//! `glob(7)`.
-//! `**` is treated special, as it matches multiple directories in a path.
-
-use std::ffi::{CStr, CString};
-use std::fs::File;
-use std::io::Read;
-use std::os::unix::io::{FromRawFd, RawFd};
-
-use anyhow::{bail, Error};
-use libc::{c_char, c_int};
-use nix::errno::Errno;
-use nix::fcntl;
-use nix::fcntl::{AtFlags, OFlag};
-use nix::sys::stat;
-use nix::sys::stat::{FileStat, Mode};
-use nix::NixPath;
-
-pub const FNM_NOMATCH: c_int = 1;
-
-extern "C" {
-    fn fnmatch(pattern: *const c_char, string: *const c_char, flags: c_int) -> c_int;
-}
-
-#[derive(Debug, PartialEq, Clone, Copy)]
-pub enum MatchType {
-    None,
-    Positive,
-    Negative,
-    PartialPositive,
-    PartialNegative,
-}
-
-/// `MatchPattern` provides functionality for filename glob pattern matching
-/// based on glibc's `fnmatch`.
-/// Positive matches return `MatchType::PartialPositive` or `MatchType::Positive`.
-/// Patterns starting with `!` are interpreted as negation, meaning they will
-/// return `MatchType::PartialNegative` or `MatchType::Negative`.
-/// No matches result in `MatchType::None`.
-/// # Examples:
-/// ```
-/// # use std::ffi::CString;
-/// # use self::proxmox_backup::pxar::{MatchPattern, MatchType};
-/// # fn main() -> Result<(), anyhow::Error> {
-/// let filename = CString::new("some.conf")?;
-/// let is_dir = false;
-///
-/// /// Positive match of any file ending in `.conf` in any subdirectory
-/// let positive = MatchPattern::from_line(b"**/*.conf")?.unwrap();
-/// let m_positive = positive.as_slice().matches_filename(&filename, is_dir)?;
-/// assert!(m_positive == MatchType::Positive);
-///
-/// /// Negative match of filenames starting with `s`
-/// let negative = MatchPattern::from_line(b"![s]*")?.unwrap();
-/// let m_negative = negative.as_slice().matches_filename(&filename, is_dir)?;
-/// assert!(m_negative == MatchType::Negative);
-/// # Ok(())
-/// # }
-/// ```
-#[derive(Clone, Eq, PartialOrd)]
-pub struct MatchPattern {
-    pattern: Vec<u8>,
-    match_positive: bool,
-    match_dir_only: bool,
-}
-
-impl std::cmp::PartialEq for MatchPattern {
-    fn eq(&self, other: &Self) -> bool {
-        self.pattern == other.pattern
-        && self.match_positive == other.match_positive
-        && self.match_dir_only == other.match_dir_only
-    }
-}
-
-impl std::cmp::Ord for MatchPattern {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        (&self.pattern, &self.match_positive, &self.match_dir_only)
-            .cmp(&(&other.pattern, &other.match_positive, &other.match_dir_only))
-    }
-}
-
-impl MatchPattern {
-    /// Read a list of `MatchPattern` from file.
-    /// The file is read line by line (lines terminated by newline character),
-    /// each line may only contain one pattern.
-    /// Leading `/` are ignored and lines starting with `#` are interpreted as
-    /// comments and not included in the resulting list.
-    /// Patterns ending in `/` will match only directories.
-    ///
-    /// On success, a list of match pattern is returned as well as the raw file
-    /// byte buffer together with the files stats.
-    /// This is done in order to avoid reading the file more than once during
-    /// encoding of the archive.
-    pub fn from_file<P: ?Sized + NixPath>(
-        parent_fd: RawFd,
-        filename: &P,
-    ) -> Result<Option<(Vec<MatchPattern>, Vec<u8>, FileStat)>, nix::Error> {
-        let stat = match stat::fstatat(parent_fd, filename, AtFlags::AT_SYMLINK_NOFOLLOW) {
-            Ok(stat) => stat,
-            Err(nix::Error::Sys(Errno::ENOENT)) => return Ok(None),
-            Err(err) => return Err(err),
-        };
-
-        let filefd = fcntl::openat(parent_fd, filename, OFlag::O_NOFOLLOW, Mode::empty())?;
-        let mut file = unsafe { File::from_raw_fd(filefd) };
-
-        let mut content_buffer = Vec::new();
-        let _bytes = file.read_to_end(&mut content_buffer)
-            .map_err(|_| Errno::EIO)?;
-
-        let mut match_pattern = Vec::new();
-        for line in content_buffer.split(|&c| c == b'\n') {
-            if line.is_empty() {
-                continue;
-            }
-            if let Some(pattern) = Self::from_line(line)? {
-                match_pattern.push(pattern);
-            }
-        }
-
-        Ok(Some((match_pattern, content_buffer, stat)))
-    }
-
-    /// Interpret a byte buffer as a sinlge line containing a valid
-    /// `MatchPattern`.
-    /// Pattern starting with `#` are interpreted as comments, returning `Ok(None)`.
-    /// Pattern starting with '!' are interpreted as negative match pattern.
-    /// Pattern with trailing `/` match only against directories.
-    /// `.` as well as `..` and any pattern containing `\0` are invalid and will
-    /// result in an error with Errno::EINVAL.
-    pub fn from_line(line: &[u8]) -> Result<Option<MatchPattern>, nix::Error> {
-        let mut input = line;
-
-        if input.starts_with(b"#") {
-            return Ok(None);
-        }
-
-        let match_positive = if input.starts_with(b"!") {
-            // Reduce slice view to exclude "!"
-            input = &input[1..];
-            false
-        } else {
-            true
-        };
-
-        // Paths ending in / match only directory names (no filenames)
-        let match_dir_only = if input.ends_with(b"/") {
-            let len = input.len();
-            input = &input[..len - 1];
-            true
-        } else {
-            false
-        };
-
-        // Ignore initial slash
-        if input.starts_with(b"/") {
-            input = &input[1..];
-        }
-
-        if input.is_empty() || input == b"." || input == b".." || input.contains(&b'\0') {
-            return Err(nix::Error::Sys(Errno::EINVAL));
-        }
-
-        Ok(Some(MatchPattern {
-            pattern: input.to_vec(),
-            match_positive,
-            match_dir_only,
-        }))
-    }
-
-
-    /// Create a `MatchPatternSlice` of the `MatchPattern` to give a view of the
-    /// `MatchPattern` without copying its content.
-    pub fn as_slice<'a>(&'a self) -> MatchPatternSlice<'a> {
-        MatchPatternSlice {
-            pattern: self.pattern.as_slice(),
-            match_positive: self.match_positive,
-            match_dir_only: self.match_dir_only,
-        }
-    }
-
-    /// Dump the content of the `MatchPattern` to stdout.
-    /// Intended for debugging purposes only.
-    pub fn dump(&self) {
-        match (self.match_positive, self.match_dir_only) {
-            (true, true) => println!("{:#?}/", self.pattern),
-            (true, false) => println!("{:#?}", self.pattern),
-            (false, true) => println!("!{:#?}/", self.pattern),
-            (false, false) => println!("!{:#?}", self.pattern),
-        }
-    }
-
-    /// Convert a list of MatchPattern to bytes in order to write them to e.g.
-    /// a file.
-    pub fn to_bytes(patterns: &[MatchPattern]) -> Vec<u8> {
-        let mut slices = Vec::new();
-        for pattern in patterns {
-            slices.push(pattern.as_slice());
-        }
-
-        MatchPatternSlice::to_bytes(&slices)
-    }
-
-    /// Invert the match type for this MatchPattern.
-    pub fn invert(&mut self) {
-        self.match_positive = !self.match_positive;
-    }
-}
-
-#[derive(Clone)]
-pub struct MatchPatternSlice<'a> {
-    pattern: &'a [u8],
-    match_positive: bool,
-    match_dir_only: bool,
-}
-
-impl<'a> MatchPatternSlice<'a> {
-    /// Returns the pattern before the first `/` encountered as `MatchPatternSlice`.
-    /// If no slash is encountered, the `MatchPatternSlice` will be a copy of the
-    /// original pattern.
-    /// ```
-    /// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
-    /// # fn main() -> Result<(), anyhow::Error> {
-    /// let pattern = MatchPattern::from_line(b"some/match/pattern/")?.unwrap();
-    /// let slice = pattern.as_slice();
-    /// let front = slice.get_front_pattern();
-    /// /// ... will be the same as ...
-    /// let front_pattern = MatchPattern::from_line(b"some")?.unwrap();
-    /// let front_slice = front_pattern.as_slice();
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn get_front_pattern(&'a self) -> MatchPatternSlice<'a> {
-        let (front, _) = self.split_at_slash();
-        MatchPatternSlice {
-            pattern: front,
-            match_positive: self.match_positive,
-            match_dir_only: self.match_dir_only,
-        }
-    }
-
-    /// Returns the pattern after the first encountered `/` as `MatchPatternSlice`.
-    /// If no slash is encountered, the `MatchPatternSlice` will be empty.
-    /// ```
-    /// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
-    /// # fn main() -> Result<(), anyhow::Error> {
-    /// let pattern = MatchPattern::from_line(b"some/match/pattern/")?.unwrap();
-    /// let slice = pattern.as_slice();
-    /// let rest = slice.get_rest_pattern();
-    /// /// ... will be the same as ...
-    /// let rest_pattern = MatchPattern::from_line(b"match/pattern/")?.unwrap();
-    /// let rest_slice = rest_pattern.as_slice();
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn get_rest_pattern(&'a self) -> MatchPatternSlice<'a> {
-        let (_, rest) = self.split_at_slash();
-        MatchPatternSlice {
-            pattern: rest,
-            match_positive: self.match_positive,
-            match_dir_only: self.match_dir_only,
-        }
-    }
-
-    /// Splits the `MatchPatternSlice` at the first slash encountered and returns the
-    /// content before (front pattern) and after the slash (rest pattern),
-    /// omitting the slash itself.
-    /// Slices starting with `**/` are an exception to this, as the corresponding
-    /// `MatchPattern` is intended to match multiple directories.
-    /// These pattern slices therefore return a `*` as front pattern and the original
-    /// pattern itself as rest pattern.
-    fn split_at_slash(&'a self) -> (&'a [u8], &'a [u8]) {
-        let pattern = if self.pattern.starts_with(b"./") {
-            &self.pattern[2..]
-        } else {
-            self.pattern
-        };
-
-        let (mut front, mut rest) = match pattern.iter().position(|&c| c == b'/') {
-            Some(ind) => {
-                let (front, rest) = pattern.split_at(ind);
-                (front, &rest[1..])
-            }
-            None => (pattern, &pattern[0..0]),
-        };
-        // '**' is treated such that it maches any directory
-        if front == b"**" {
-            front = b"*";
-            rest = pattern;
-        }
-
-        (front, rest)
-    }
-
-    /// Convert a list of `MatchPatternSlice`s to bytes in order to write them to e.g.
-    /// a file.
-    pub fn to_bytes(patterns: &[MatchPatternSlice]) -> Vec<u8> {
-        let mut buffer = Vec::new();
-        for pattern in patterns {
-            if !pattern.match_positive { buffer.push(b'!'); }
-            buffer.extend_from_slice(&pattern.pattern);
-            if pattern.match_dir_only { buffer.push(b'/'); }
-            buffer.push(b'\n');
-        }
-        buffer
-    }
-
-    /// Match the given filename against this `MatchPatternSlice`.
-    /// If the filename matches the pattern completely, `MatchType::Positive` or
-    /// `MatchType::Negative` is returned, depending if the match pattern is was
-    /// declared as positive (no `!` prefix) or negative (`!` prefix).
-    /// If the pattern matched only up to the first slash of the pattern,
-    /// `MatchType::PartialPositive` or `MatchType::PartialNegatie` is returned.
-    /// If the pattern was postfixed by a trailing `/` a match is only valid if
-    /// the parameter `is_dir` equals `true`.
-    /// No match results in `MatchType::None`.
-    pub fn matches_filename(&self, filename: &CStr, is_dir: bool) -> Result<MatchType, Error> {
-        let mut res = MatchType::None;
-        let (front, _) = self.split_at_slash();
-
-        let front = CString::new(front).unwrap();
-        let fnmatch_res = unsafe {
-            let front_ptr = front.as_ptr() as *const libc::c_char;
-            let filename_ptr = filename.as_ptr() as *const libc::c_char;
-            fnmatch(front_ptr, filename_ptr, 0)
-        };
-        if fnmatch_res < 0 {
-            bail!("error in fnmatch inside of MatchPattern");
-        }
-        if fnmatch_res == 0 {
-            res = if self.match_positive {
-                MatchType::PartialPositive
-            } else {
-                MatchType::PartialNegative
-            };
-        }
-
-        let full = if self.pattern.starts_with(b"**/") {
-            CString::new(&self.pattern[3..]).unwrap()
-        } else {
-            CString::new(&self.pattern[..]).unwrap()
-        };
-        let fnmatch_res = unsafe {
-            let full_ptr = full.as_ptr() as *const libc::c_char;
-            let filename_ptr = filename.as_ptr() as *const libc::c_char;
-            fnmatch(full_ptr, filename_ptr, 0)
-        };
-        if fnmatch_res < 0 {
-            bail!("error in fnmatch inside of MatchPattern");
-        }
-        if fnmatch_res == 0 {
-            res = if self.match_positive {
-                MatchType::Positive
-            } else {
-                MatchType::Negative
-            };
-        }
-
-        if !is_dir && self.match_dir_only {
-            res = MatchType::None;
-        }
-
-        if !is_dir && (res == MatchType::PartialPositive || res == MatchType::PartialNegative) {
-            res = MatchType::None;
-        }
-
-        Ok(res)
-    }
-
-    /// Match the given filename against the set of `MatchPatternSlice`s.
-    ///
-    /// A positive match is intended to includes the full subtree (unless another
-    /// negative match excludes entries later).
-    /// The `MatchType` together with an updated `MatchPatternSlice` list for passing
-    /// to the matched child is returned.
-    /// ```
-    /// # use std::ffi::CString;
-    /// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
-    /// # fn main() -> Result<(), anyhow::Error> {
-    /// let patterns = vec![
-    ///     MatchPattern::from_line(b"some/match/pattern/")?.unwrap(),
-    ///     MatchPattern::from_line(b"to_match/")?.unwrap()
-    /// ];
-    /// let mut slices = Vec::new();
-    /// for pattern in &patterns {
-    ///     slices.push(pattern.as_slice());
-    /// }
-    /// let filename = CString::new("some")?;
-    /// let is_dir = true;
-    /// let (match_type, child_pattern) = MatchPatternSlice::match_filename_include(
-    ///     &filename,
-    ///     is_dir,
-    ///     &slices
-    /// )?;
-    /// assert_eq!(match_type, MatchType::PartialPositive);
-    /// /// child pattern will be the same as ...
-    /// let pattern = MatchPattern::from_line(b"match/pattern/")?.unwrap();
-    /// let slice = pattern.as_slice();
-    ///
-    /// let filename = CString::new("to_match")?;
-    /// let is_dir = true;
-    /// let (match_type, child_pattern) = MatchPatternSlice::match_filename_include(
-    ///     &filename,
-    ///     is_dir,
-    ///     &slices
-    /// )?;
-    /// assert_eq!(match_type, MatchType::Positive);
-    /// /// child pattern will be the same as ...
-    /// let pattern = MatchPattern::from_line(b"**/*")?.unwrap();
-    /// let slice = pattern.as_slice();
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn match_filename_include(
-        filename: &CStr,
-        is_dir: bool,
-        match_pattern: &'a [MatchPatternSlice<'a>],
-    ) -> Result<(MatchType, Vec<MatchPatternSlice<'a>>), Error> {
-        let mut child_pattern = Vec::new();
-        let mut match_state = MatchType::None;
-
-        for pattern in match_pattern {
-            match pattern.matches_filename(filename, is_dir)? {
-                MatchType::None => continue,
-                MatchType::Positive => match_state = MatchType::Positive,
-                MatchType::Negative => match_state = MatchType::Negative,
-                MatchType::PartialPositive => {
-                    if match_state != MatchType::Negative && match_state != MatchType::Positive {
-                        match_state = MatchType::PartialPositive;
-                    }
-                    child_pattern.push(pattern.get_rest_pattern());
-                }
-                MatchType::PartialNegative => {
-                    if match_state == MatchType::PartialPositive {
-                        match_state = MatchType::PartialNegative;
-                    }
-                    child_pattern.push(pattern.get_rest_pattern());
-                }
-            }
-        }
-
-        Ok((match_state, child_pattern))
-    }
-
-    /// Match the given filename against the set of `MatchPatternSlice`s.
-    ///
-    /// A positive match is intended to exclude the full subtree, independent of
-    /// matches deeper down the tree.
-    /// The `MatchType` together with an updated `MatchPattern` list for passing
-    /// to the matched child is returned.
-    /// ```
-    /// # use std::ffi::CString;
-    /// # use self::proxmox_backup::pxar::{MatchPattern, MatchPatternSlice, MatchType};
-    /// # fn main() -> Result<(), anyhow::Error> {
-    /// let patterns = vec![
-    ///     MatchPattern::from_line(b"some/match/pattern/")?.unwrap(),
-    ///     MatchPattern::from_line(b"to_match/")?.unwrap()
-    /// ];
-    /// let mut slices = Vec::new();
-    /// for pattern in &patterns {
-    ///     slices.push(pattern.as_slice());
-    /// }
-    /// let filename = CString::new("some")?;
-    /// let is_dir = true;
-    /// let (match_type, child_pattern) = MatchPatternSlice::match_filename_exclude(
-    ///     &filename,
-    ///     is_dir,
-    ///     &slices,
-    /// )?;
-    /// assert_eq!(match_type, MatchType::PartialPositive);
-    /// /// child pattern will be the same as ...
-    /// let pattern = MatchPattern::from_line(b"match/pattern/")?.unwrap();
-    /// let slice = pattern.as_slice();
-    ///
-    /// let filename = CString::new("to_match")?;
-    /// let is_dir = true;
-    /// let (match_type, child_pattern) = MatchPatternSlice::match_filename_exclude(
-    ///     &filename,
-    ///     is_dir,
-    ///     &slices,
-    /// )?;
-    /// assert_eq!(match_type, MatchType::Positive);
-    /// /// child pattern will be empty
-    /// # Ok(())
-    /// # }
-    /// ```
-    pub fn match_filename_exclude(
-        filename: &CStr,
-        is_dir: bool,
-        match_pattern: &'a [MatchPatternSlice<'a>],
-    ) -> Result<(MatchType, Vec<MatchPatternSlice<'a>>), Error> {
-        let mut child_pattern = Vec::new();
-        let mut match_state = MatchType::None;
-
-        for pattern in match_pattern {
-            match pattern.matches_filename(filename, is_dir)? {
-                MatchType::None => {}
-                MatchType::Positive => match_state = MatchType::Positive,
-                MatchType::Negative => match_state = MatchType::Negative,
-                match_type => {
-                    if match_state != MatchType::Positive && match_state != MatchType::Negative {
-                        match_state = match_type;
-                    }
-                    child_pattern.push(pattern.get_rest_pattern());
-                }
-            }
-        }
-
-        Ok((match_state, child_pattern))
-    }
-}

src/pxar/metadata.rs

@@ -0,0 +1,342 @@
+use std::ffi::{CStr, CString};
+use std::os::unix::ffi::OsStrExt;
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use std::path::Path;
+
+use anyhow::{bail, format_err, Error};
+use nix::errno::Errno;
+use nix::fcntl::OFlag;
+use nix::sys::stat::Mode;
+
+use pxar::Metadata;
+
+use proxmox::sys::error::SysError;
+use proxmox::tools::fd::RawFdNum;
+use proxmox::{c_result, c_try};
+
+use crate::pxar::flags;
+use crate::pxar::tools::perms_from_metadata;
+use crate::tools::{acl, fs, xattr};
+
+//
+// utility functions
+//
+
+fn flags_contain(flags: u64, test_flag: u64) -> bool {
+    0 != (flags & test_flag)
+}
+
+fn allow_notsupp<E: SysError>(err: E) -> Result<(), E> {
+    if err.is_errno(Errno::EOPNOTSUPP) {
+        Ok(())
+    } else {
+        Err(err)
+    }
+}
+
+fn allow_notsupp_remember<E: SysError>(err: E, not_supp: &mut bool) -> Result<(), E> {
+    if err.is_errno(Errno::EOPNOTSUPP) {
+        *not_supp = true;
+        Ok(())
+    } else {
+        Err(err)
+    }
+}
+
+fn nsec_to_update_timespec(mtime_nsec: u64) -> [libc::timespec; 2] {
+    // restore mtime
+    const UTIME_OMIT: i64 = (1 << 30) - 2;
+    const NANOS_PER_SEC: i64 = 1_000_000_000;
+
+    let sec = (mtime_nsec as i64) / NANOS_PER_SEC;
+    let nsec = (mtime_nsec as i64) % NANOS_PER_SEC;
+
+    let times: [libc::timespec; 2] = [
+        libc::timespec {
+            tv_sec: 0,
+            tv_nsec: UTIME_OMIT,
+        },
+        libc::timespec {
+            tv_sec: sec,
+            tv_nsec: nsec,
+        },
+    ];
+
+    times
+}
+
+//
+// metadata application:
+//
+
+pub fn apply_at(
+    flags: u64,
+    metadata: &Metadata,
+    parent: RawFd,
+    file_name: &CStr,
+) -> Result<(), Error> {
+    let fd = proxmox::tools::fd::Fd::openat(
+        &unsafe { RawFdNum::from_raw_fd(parent) },
+        file_name,
+        OFlag::O_PATH | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW,
+        Mode::empty(),
+    )?;
+
+    apply(flags, metadata, fd.as_raw_fd(), file_name)
+}
+
+pub fn apply_with_path<T: AsRef<Path>>(
+    flags: u64,
+    metadata: &Metadata,
+    fd: RawFd,
+    file_name: T,
+) -> Result<(), Error> {
+    apply(
+        flags,
+        metadata,
+        fd,
+        &CString::new(file_name.as_ref().as_os_str().as_bytes())?,
+    )
+}
+
+pub fn apply(flags: u64, metadata: &Metadata, fd: RawFd, file_name: &CStr) -> Result<(), Error> {
+    let c_proc_path = CString::new(format!("/proc/self/fd/{}", fd)).unwrap();
+    let c_proc_path = c_proc_path.as_ptr();
+
+    if metadata.stat.flags != 0 {
+        todo!("apply flags!");
+    }
+
+    unsafe {
+        // UID and GID first, as this fails if we lose access anyway.
+        c_result!(libc::chown(
+            c_proc_path,
+            metadata.stat.uid,
+            metadata.stat.gid
+        ))
+        .map(drop)
+        .or_else(allow_notsupp)?;
+    }
+
+    let mut skip_xattrs = false;
+    apply_xattrs(flags, c_proc_path, metadata, &mut skip_xattrs)?;
+    add_fcaps(flags, c_proc_path, metadata, &mut skip_xattrs)?;
+    apply_acls(flags, c_proc_path, metadata)?;
+    apply_quota_project_id(flags, fd, metadata)?;
+
+    // Finally mode and time. We may lose access with mode, but the changing the mode also
+    // affects times.
+    if !metadata.is_symlink() {
+        c_result!(unsafe { libc::chmod(c_proc_path, perms_from_metadata(metadata)?.bits()) })
+            .map(drop)
+            .or_else(allow_notsupp)?;
+    }
+
+    let res = c_result!(unsafe {
+        libc::utimensat(
+            libc::AT_FDCWD,
+            c_proc_path,
+            nsec_to_update_timespec(metadata.stat.mtime).as_ptr(),
+            0,
+        )
+    });
+    match res {
+        Ok(_) => (),
+        Err(ref err) if err.is_errno(Errno::EOPNOTSUPP) => (),
+        Err(ref err) if err.is_errno(Errno::EPERM) => {
+            println!(
+                "failed to restore mtime attribute on {:?}: {}",
+                file_name, err
+            );
+        }
+        Err(err) => return Err(err.into()),
+    }
+
+    Ok(())
+}
+
+fn add_fcaps(
+    flags: u64,
+    c_proc_path: *const libc::c_char,
+    metadata: &Metadata,
+    skip_xattrs: &mut bool,
+) -> Result<(), Error> {
+    if *skip_xattrs || !flags_contain(flags, flags::WITH_FCAPS) {
+        return Ok(());
+    }
+    let fcaps = match metadata.fcaps.as_ref() {
+        Some(fcaps) => fcaps,
+        None => return Ok(()),
+    };
+
+    c_result!(unsafe {
+        libc::setxattr(
+            c_proc_path,
+            xattr::xattr_name_fcaps().as_ptr(),
+            fcaps.data.as_ptr() as *const libc::c_void,
+            fcaps.data.len(),
+            0,
+        )
+    })
+    .map(drop)
+    .or_else(|err| allow_notsupp_remember(err, skip_xattrs))?;
+
+    Ok(())
+}
+
+fn apply_xattrs(
+    flags: u64,
+    c_proc_path: *const libc::c_char,
+    metadata: &Metadata,
+    skip_xattrs: &mut bool,
+) -> Result<(), Error> {
+    if *skip_xattrs || !flags_contain(flags, flags::WITH_XATTRS) {
+        return Ok(());
+    }
+
+    for xattr in &metadata.xattrs {
+        if *skip_xattrs {
+            return Ok(());
+        }
+
+        if !xattr::is_valid_xattr_name(xattr.name()) {
+            println!("skipping invalid xattr named {:?}", xattr.name());
+            continue;
+        }
+
+        c_result!(unsafe {
+            libc::setxattr(
+                c_proc_path,
+                xattr.name().as_ptr() as *const libc::c_char,
+                xattr.value().as_ptr() as *const libc::c_void,
+                xattr.value().len(),
+                0,
+            )
+        })
+        .map(drop)
+        .or_else(|err| allow_notsupp_remember(err, &mut *skip_xattrs))?;
+    }
+
+    Ok(())
+}
+
+fn apply_acls(
+    flags: u64,
+    c_proc_path: *const libc::c_char,
+    metadata: &Metadata,
+) -> Result<(), Error> {
+    if !flags_contain(flags, flags::WITH_ACL) || metadata.acl.is_empty() {
+        return Ok(());
+    }
+
+    let mut acl = acl::ACL::init(5)?;
+
+    // acl type access:
+    acl.add_entry_full(
+        acl::ACL_USER_OBJ,
+        None,
+        acl::mode_user_to_acl_permissions(metadata.stat.mode),
+    )?;
+
+    acl.add_entry_full(
+        acl::ACL_OTHER,
+        None,
+        acl::mode_other_to_acl_permissions(metadata.stat.mode),
+    )?;
+
+    match metadata.acl.group_obj.as_ref() {
+        Some(group_obj) => {
+            acl.add_entry_full(
+                acl::ACL_MASK,
+                None,
+                acl::mode_group_to_acl_permissions(metadata.stat.mode),
+            )?;
+            acl.add_entry_full(acl::ACL_GROUP_OBJ, None, group_obj.permissions.0)?;
+        }
+        None => {
+            acl.add_entry_full(
+                acl::ACL_GROUP_OBJ,
+                None,
+                acl::mode_group_to_acl_permissions(metadata.stat.mode),
+            )?;
+        }
+    }
+
+    for user in &metadata.acl.users {
+        acl.add_entry_full(acl::ACL_USER, Some(user.uid), user.permissions.0)?;
+    }
+
+    for group in &metadata.acl.groups {
+        acl.add_entry_full(acl::ACL_GROUP, Some(group.gid), group.permissions.0)?;
+    }
+
+    if !acl.is_valid() {
+        bail!("Error while restoring ACL - ACL invalid");
+    }
+
+    c_try!(unsafe { acl::acl_set_file(c_proc_path, acl::ACL_TYPE_ACCESS, acl.ptr,) });
+    drop(acl);
+
+    // acl type default:
+    if let Some(default) = metadata.acl.default.as_ref() {
+        let mut acl = acl::ACL::init(5)?;
+
+        acl.add_entry_full(acl::ACL_USER_OBJ, None, default.user_obj_permissions.0)?;
+
+        acl.add_entry_full(acl::ACL_GROUP_OBJ, None, default.group_obj_permissions.0)?;
+
+        acl.add_entry_full(acl::ACL_OTHER, None, default.other_permissions.0)?;
+
+        if default.mask_permissions != pxar::format::acl::Permissions::NO_MASK {
+            acl.add_entry_full(acl::ACL_MASK, None, default.mask_permissions.0)?;
+        }
+
+        for user in &metadata.acl.default_users {
+            acl.add_entry_full(acl::ACL_USER, Some(user.uid), user.permissions.0)?;
+        }
+
+        for group in &metadata.acl.default_groups {
+            acl.add_entry_full(acl::ACL_GROUP, Some(group.gid), group.permissions.0)?;
+        }
+
+        if !acl.is_valid() {
+            bail!("Error while restoring ACL - ACL invalid");
+        }
+
+        c_try!(unsafe { acl::acl_set_file(c_proc_path, acl::ACL_TYPE_DEFAULT, acl.ptr,) });
+    }
+
+    Ok(())
+}
+
+fn apply_quota_project_id(flags: u64, fd: RawFd, metadata: &Metadata) -> Result<(), Error> {
+    if !flags_contain(flags, flags::WITH_QUOTA_PROJID) {
+        return Ok(());
+    }
+
+    let projid = match metadata.quota_project_id {
+        Some(projid) => projid,
+        None => return Ok(()),
+    };
+
+    let mut fsxattr = fs::FSXAttr::default();
+    unsafe {
+        fs::fs_ioc_fsgetxattr(fd, &mut fsxattr).map_err(|err| {
+            format_err!(
+                "error while getting fsxattr to restore quota project id - {}",
+                err
+            )
+        })?;
+
+        fsxattr.fsx_projid = projid.projid as u32;
+
+        fs::fs_ioc_fssetxattr(fd, &fsxattr).map_err(|err| {
+            format_err!(
+                "error while setting fsxattr to restore quota project id - {}",
+                err
+            )
+        })?;
+    }
+
+    Ok(())
+}

src/pxar/tools.rs

@@ -0,0 +1,192 @@
+//! Some common methods used within the pxar code.
+
+use std::convert::TryFrom;
+use std::ffi::OsStr;
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use anyhow::{bail, format_err, Error};
+use nix::sys::stat::Mode;
+
+use pxar::{mode, Entry, EntryKind, Metadata};
+
+/// Get the file permissions as `nix::Mode`
+pub fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
+    let mode = meta.stat.get_permission_bits();
+    u32::try_from(mode)
+        .map_err(drop)
+        .and_then(|mode| Mode::from_bits(mode).ok_or(()))
+        .map_err(|_| format_err!("mode contains illegal bits: 0x{:x} (0o{:o})", mode, mode))
+}
+
+/// Make sure path is relative and not '.' or '..'.
+pub fn assert_relative_path<S: AsRef<OsStr> + ?Sized>(path: &S) -> Result<(), Error> {
+    assert_relative_path_do(Path::new(path))
+}
+
+fn assert_relative_path_do(path: &Path) -> Result<(), Error> {
+    if !path.is_relative() {
+        bail!("bad absolute file name in archive: {:?}", path);
+    }
+
+    let mut components = path.components();
+    match components.next() {
+        Some(std::path::Component::Normal(_)) => (),
+        _ => bail!("invalid path component in archive: {:?}", path),
+    }
+
+    if components.next().is_some() {
+        bail!(
+            "invalid path with multiple components in archive: {:?}",
+            path
+        );
+    }
+
+    Ok(())
+}
+
+#[rustfmt::skip]
+fn symbolic_mode(c: u64, special: bool, special_x: u8, special_no_x: u8) -> [u8; 3] {
+    [
+        if 0 != c & 4 { b'r' } else { b'-' },
+        if 0 != c & 2 { b'w' } else { b'-' },
+        match (c & 1, special) {
+            (0, false) => b'-',
+            (0, true) => special_no_x,
+            (_, false) => b'x',
+            (_, true) => special_x,
+        }
+    ]
+}
+
+fn mode_string(entry: &Entry) -> String {
+    // https://www.gnu.org/software/coreutils/manual/html_node/What-information-is-listed.html#What-information-is-listed
+    // additionally we use:
+    //     file type capital 'L' hard links
+    //     a second '+' after the mode to show non-acl xattr presence
+    //
+    // Trwxrwxrwx++ uid/gid size mtime filename [-> destination]
+
+    let meta = entry.metadata();
+    let mode = meta.stat.mode;
+    let type_char = if entry.is_hardlink() {
+        'L'
+    } else {
+        match mode & mode::IFMT {
+            mode::IFREG => '-',
+            mode::IFBLK => 'b',
+            mode::IFCHR => 'c',
+            mode::IFDIR => 'd',
+            mode::IFLNK => 'l',
+            mode::IFIFO => 'p',
+            mode::IFSOCK => 's',
+            _ => '?',
+        }
+    };
+
+    let fmt_u = symbolic_mode((mode >> 6) & 7, 0 != mode & mode::ISUID, b's', b'S');
+    let fmt_g = symbolic_mode((mode >> 3) & 7, 0 != mode & mode::ISGID, b's', b'S');
+    let fmt_o = symbolic_mode((mode >> 3) & 7, 0 != mode & mode::ISVTX, b't', b'T');
+
+    let has_acls = if meta.acl.is_empty() { ' ' } else { '+' };
+
+    let has_xattrs = if meta.xattrs.is_empty() { ' ' } else { '+' };
+
+    format!(
+        "{}{}{}{}{}{}",
+        type_char,
+        unsafe { std::str::from_utf8_unchecked(&fmt_u) },
+        unsafe { std::str::from_utf8_unchecked(&fmt_g) },
+        unsafe { std::str::from_utf8_unchecked(&fmt_o) },
+        has_acls,
+        has_xattrs,
+    )
+}
+
+pub fn format_single_line_entry(entry: &Entry) -> String {
+    use chrono::offset::TimeZone;
+
+    let mode_string = mode_string(entry);
+
+    let meta = entry.metadata();
+    let mtime = meta.mtime_as_duration();
+    let mtime = chrono::Local.timestamp(mtime.as_secs() as i64, mtime.subsec_nanos());
+
+    let (size, link) = match entry.kind() {
+        EntryKind::File { size, .. } => (format!("{}", *size), String::new()),
+        EntryKind::Symlink(link) => ("0".to_string(), format!(" -> {:?}", link.as_os_str())),
+        EntryKind::Hardlink(link) => ("0".to_string(), format!(" -> {:?}", link.as_os_str())),
+        EntryKind::Device(dev) => (format!("{},{}", dev.major, dev.minor), String::new()),
+        _ => ("0".to_string(), String::new()),
+    };
+
+    format!(
+        "{} {:<13} {} {:>8} {:?}{}",
+        mode_string,
+        format!("{}/{}", meta.stat.uid, meta.stat.gid),
+        mtime.format("%Y-%m-%d %H:%M:%S"),
+        size,
+        entry.path(),
+        link,
+    )
+}
+
+pub fn format_multi_line_entry(entry: &Entry) -> String {
+    use chrono::offset::TimeZone;
+
+    let mode_string = mode_string(entry);
+
+    let meta = entry.metadata();
+    let mtime = meta.mtime_as_duration();
+    let mtime = chrono::Local.timestamp(mtime.as_secs() as i64, mtime.subsec_nanos());
+
+    let (size, link, type_name) = match entry.kind() {
+        EntryKind::File { size, .. } => (format!("{}", *size), String::new(), "file"),
+        EntryKind::Symlink(link) => (
+            "0".to_string(),
+            format!(" -> {:?}", link.as_os_str()),
+            "symlink",
+        ),
+        EntryKind::Hardlink(link) => (
+            "0".to_string(),
+            format!(" -> {:?}", link.as_os_str()),
+            "symlink",
+        ),
+        EntryKind::Device(dev) => (
+            format!("{},{}", dev.major, dev.minor),
+            String::new(),
+            if meta.stat.is_chardev() {
+                "characters pecial file"
+            } else if meta.stat.is_blockdev() {
+                "block special file"
+            } else {
+                "device"
+            },
+        ),
+        EntryKind::Socket => ("0".to_string(), String::new(), "socket"),
+        EntryKind::Fifo => ("0".to_string(), String::new(), "fifo"),
+        EntryKind::Directory => ("0".to_string(), String::new(), "directory"),
+        EntryKind::GoodbyeTable => ("0".to_string(), String::new(), "bad entry"),
+    };
+
+    let file_name = match std::str::from_utf8(entry.path().as_os_str().as_bytes()) {
+        Ok(name) => std::borrow::Cow::Borrowed(name),
+        Err(_) => std::borrow::Cow::Owned(format!("{:?}", entry.path())),
+    };
+
+    format!(
+        "  File: {}{}\n  \
+           Size: {:<13} Type: {}\n\
+         Access: ({:o}/{})  Uid: {:<5} Gid: {:<5}\n\
+         Modify: {}\n",
+        file_name,
+        link,
+        size,
+        type_name,
+        meta.file_mode(),
+        mode_string,
+        meta.stat.uid,
+        meta.stat.gid,
+        mtime.format("%Y-%m-%d %H:%M:%S"),
+    )
+}

src/tools/acl.rs

@@ -49,7 +49,8 @@ pub const ACL_EA_VERSION: u32 = 0x0002;
 #[link(name = "acl")]
 extern "C" {
     fn acl_get_file(path: *const c_char, acl_type: ACLType) -> *mut c_void;
-    fn acl_set_file(path: *const c_char, acl_type: ACLType, acl: *mut c_void) -> c_int;
+    // FIXME: remove 'pub' after the cleanup
+    pub(crate) fn acl_set_file(path: *const c_char, acl_type: ACLType, acl: *mut c_void) -> c_int;
     fn acl_get_fd(fd: RawFd) -> *mut c_void;
     fn acl_get_entry(acl: *const c_void, entry_id: c_int, entry: *mut *mut c_void) -> c_int;
     fn acl_create_entry(acl: *mut *mut c_void, entry: *mut *mut c_void) -> c_int;
@@ -68,7 +69,8 @@ extern "C" {
 
 #[derive(Debug)]
 pub struct ACL {
-    ptr: *mut c_void,
+    // FIXME: remove 'pub' after the cleanup
+    pub(crate) ptr: *mut c_void,
 }
 
 impl Drop for ACL {

src/tools/xattr.rs

@@ -16,6 +16,22 @@ pub fn xattr_name_fcaps() -> &'static CStr {
     c_str!("security.capability")
 }
 
+/// `"system.posix_acl_access"` as a CStr to avoid typos.
+///
+/// This cannot be `const` until `const_cstr_unchecked` is stable.
+#[inline]
+pub fn xattr_acl_access() -> &'static CStr {
+    c_str!("system.posix_acl_access")
+}
+
+/// `"system.posix_acl_default"` as a CStr to avoid typos.
+///
+/// This cannot be `const` until `const_cstr_unchecked` is stable.
+#[inline]
+pub fn xattr_acl_default() -> &'static CStr {
+    c_str!("system.posix_acl_default")
+}
+
 /// Result of `flistxattr`, allows iterating over the attributes as a list of `&CStr`s.
 ///
 /// Listing xattrs produces a list separated by zeroes, inherently making them available as `&CStr`
@@ -139,6 +155,11 @@ pub fn is_security_capability(name: &CStr) -> bool {
     name.to_bytes() == xattr_name_fcaps().to_bytes()
 }
 
+pub fn is_acl(name: &CStr) -> bool {
+    name.to_bytes() == xattr_acl_access().to_bytes()
+    || name.to_bytes() == xattr_acl_default().to_bytes()
+}
+
 /// Check if the passed name buffer starts with a valid xattr namespace prefix
 /// and is within the length limit of 255 bytes
 pub fn is_valid_xattr_name(c_name: &CStr) -> bool {

tests/catar.rs

@@ -14,30 +14,27 @@ fn run_test(dir_name: &str) -> Result<(), Error> {
         .status()
         .expect("failed to execute casync");
 
-    let mut writer = std::fs::OpenOptions::new()
+    let writer = std::fs::OpenOptions::new()
         .create(true)
         .write(true)
         .truncate(true)
         .open("test-proxmox.catar")?;
+    let writer = pxar::encoder::sync::StandardWriter::new(writer);
 
-    let mut dir = nix::dir::Dir::open(
+    let dir = nix::dir::Dir::open(
         dir_name, nix::fcntl::OFlag::O_NOFOLLOW,
         nix::sys::stat::Mode::empty())?;
 
-    let path = std::path::PathBuf::from(dir_name);
+    create_archive(
-
+        dir,
-    let catalog = None::<&mut catalog::DummyCatalogWriter>;
+        writer,
-    Encoder::encode(
+        Vec::new(),
-        path,
+        flags::DEFAULT,
-        &mut dir,
-        &mut writer,
-        catalog,
         None,
         false,
-        false,
+        |_| Ok(()),
-        flags::DEFAULT,
-        Vec::new(),
         ENCODER_MAX_ENTRIES,
+        None,
     )?;
 
     Command::new("cmp")