src/pxar/encoder.rs: use BackupCatalogWriter

src/bin/proxmox-backup-client.rs

@@ -7,6 +7,8 @@ use chrono::{Local, Utc, TimeZone};
 use std::path::{Path, PathBuf};
 use std::collections::{HashSet, HashMap};
 use std::io::Write;
+use std::os::unix::fs::OpenOptionsExt;
+
 use proxmox::tools::fs::{file_get_contents, file_get_json, file_set_contents, image_size};
 
 use proxmox_backup::tools;
@@ -25,7 +27,7 @@ use proxmox_backup::pxar;
 
 use serde_json::{json, Value};
 //use hyper::Body;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use regex::Regex;
 use xdg::BaseDirectories;
 
@@ -157,9 +159,10 @@ fn backup_directory<P: AsRef<Path>>(
     verbose: bool,
     skip_lost_and_found: bool,
     crypt_config: Option<Arc<CryptConfig>>,
+    catalog: Arc<Mutex<pxar::catalog::SimpleCatalog>>,
 ) -> Result<BackupStats, Error> {
 
-    let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose, skip_lost_and_found)?;
+    let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose, skip_lost_and_found, catalog)?;
     let chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
 
     let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
@@ -595,6 +598,10 @@ fn create_backup(
 
     let mut file_list = vec![];
 
+    let catalog_filename = format!("/tmp/pbs-catalog-{}.cat", std::process::id());
+    let catalog = Arc::new(Mutex::new(pxar::catalog::SimpleCatalog::new(&catalog_filename)?));
+    let mut upload_catalog = false;
+
     for (backup_type, filename, target, size) in upload_list {
         match backup_type {
             BackupType::CONFIG => {
@@ -608,6 +615,7 @@ fn create_backup(
                 file_list.push((target, stats));
             }
             BackupType::PXAR => {
+                upload_catalog = true;
                 println!("Upload directory '{}' to '{:?}' as {}", filename, repo, target);
                 let stats = backup_directory(
                     &client,
@@ -618,6 +626,7 @@ fn create_backup(
                     verbose,
                     skip_lost_and_found,
                     crypt_config.clone(),
+                    catalog.clone(),
                 )?;
                 file_list.push((target, stats));
             }
@@ -637,6 +646,19 @@ fn create_backup(
         }
     }
 
+    // finalize and upload catalog
+    if upload_catalog {
+        let mutex = Arc::try_unwrap(catalog)
+            .map_err(|_| format_err!("unable to get catalog (still used)"))?;
+        drop(mutex); // close catalog
+
+        let target = "catalog.blob";
+        let stats = client.upload_blob_from_file(&catalog_filename, target, crypt_config.clone(), true).wait()?;
+        file_list.push((target.to_owned(), stats));
+
+        let _ = std::fs::remove_file(&catalog_filename);
+    }
+
     if let Some(rsa_encrypted_key) = rsa_encrypted_key {
         let target = "rsa-encrypted.key";
         println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
@@ -772,8 +794,6 @@ fn restore(
 
     let client = client.start_backup_reader(repo.store(), &backup_type, &backup_id, backup_time, true).wait()?;
 
-    use std::os::unix::fs::OpenOptionsExt;
-
     let tmpfile = std::fs::OpenOptions::new()
         .write(true)
         .read(true)

src/bin/pxar.rs

@@ -208,7 +208,8 @@ fn create_archive(
         feature_flags ^= pxar::flags::WITH_SOCKETS;
     }
 
-    pxar::Encoder::encode(source, &mut dir, &mut writer, devices, verbose, false, feature_flags)?;
+    let catalog = None::<&mut pxar::catalog::SimpleCatalog>;
+    pxar::Encoder::encode(source, &mut dir, &mut writer, catalog, devices, verbose, false, feature_flags)?;
 
     writer.flush()?;
 

src/client/pxar_backup_stream.rs

@@ -37,7 +37,14 @@ impl Drop for PxarBackupStream {
 
 impl PxarBackupStream {
 
-    pub fn new(mut dir: Dir, path: PathBuf, device_set: Option<HashSet<u64>>, verbose: bool, skip_lost_and_found: bool) -> Result<Self, Error> {
+    pub fn new(
+        mut dir: Dir,
+        path: PathBuf,
+        device_set: Option<HashSet<u64>>,
+        verbose: bool,
+        skip_lost_and_found: bool,
+        catalog: Arc<Mutex<crate::pxar::catalog::SimpleCatalog>>,
+    ) -> Result<Self, Error> {
 
         let (rx, tx) = nix::unistd::pipe()?;
 
@@ -47,9 +54,11 @@ impl PxarBackupStream {
         let error = Arc::new(Mutex::new(None));
         let error2 = error.clone();
 
-        let child = thread::spawn(move|| {
+        let catalog = catalog.clone();
+        let child = thread::spawn(move || {
+            let mut guard = catalog.lock().unwrap();
             let mut writer = unsafe { std::fs::File::from_raw_fd(tx) };
-            if let Err(err) = pxar::Encoder::encode(path, &mut dir, &mut writer, device_set, verbose, skip_lost_and_found, pxar::flags::DEFAULT) {
+            if let Err(err) = pxar::Encoder::encode(path, &mut dir, &mut writer, Some(&mut *guard), device_set, verbose, skip_lost_and_found, pxar::flags::DEFAULT) {
                 let mut error = error2.lock().unwrap();
                 *error = Some(err.to_string());
             }
@@ -65,12 +74,18 @@ impl PxarBackupStream {
         })
     }
 
-    pub fn open(dirname: &Path, device_set: Option<HashSet<u64>>, verbose: bool, skip_lost_and_found: bool) -> Result<Self, Error> {
+    pub fn open(
+        dirname: &Path,
+        device_set: Option<HashSet<u64>>,
+        verbose: bool,
+        skip_lost_and_found: bool,
+        catalog: Arc<Mutex<crate::pxar::catalog::SimpleCatalog>>,
+    ) -> Result<Self, Error> {
 
         let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
         let path = std::path::PathBuf::from(dirname);
 
-        Self::new(dir, path, device_set, verbose, skip_lost_and_found)
+        Self::new(dir, path, device_set, verbose, skip_lost_and_found, catalog)
     }
 }
 

src/pxar/encoder.rs

@@ -11,6 +11,8 @@ use super::format_definition::*;
 use super::binary_search_tree::*;
 use super::helper::*;
 use super::match_pattern::*;
+use super::catalog::BackupCatalogWriter;
+
 use crate::tools::fs;
 use crate::tools::acl;
 use crate::tools::xattr;
@@ -42,11 +44,12 @@ struct HardLinkInfo {
     st_ino: u64,
 }
 
-pub struct Encoder<'a, W: Write> {
+pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
     base_path: PathBuf,
     relative_path: PathBuf,
     writer: &'a mut W,
     writer_pos: usize,
+    catalog: Option<&'a mut C>,
     _size: usize,
     file_copy_buffer: Vec<u8>,
     device_set: Option<HashSet<u64>>,
@@ -58,7 +61,7 @@ pub struct Encoder<'a, W: Write> {
     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
 }
 
-impl <'a, W: Write> Encoder<'a, W> {
+impl <'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
 
     // used for error reporting
     fn full_path(&self) ->  PathBuf {
@@ -78,6 +81,7 @@ impl <'a, W: Write> Encoder<'a, W> {
         path: PathBuf,
         dir: &mut nix::dir::Dir,
         writer: &'a mut W,
+        catalog: Option<&'a mut C>,
         device_set: Option<HashSet<u64>>,
         verbose: bool,
         skip_lost_and_found: bool, // fixme: should be a feature flag ??
@@ -118,6 +122,7 @@ impl <'a, W: Write> Encoder<'a, W> {
             relative_path: PathBuf::new(),
             writer: writer,
             writer_pos: 0,
+            catalog,
             _size: 0,
             file_copy_buffer,
             device_set,
@@ -758,7 +763,13 @@ impl <'a, W: Write> Encoder<'a, W> {
                 };
 
                 self.write_filename(&filename)?;
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.start_directory(&filename)?;
+                }
                 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
+                if let Some(ref mut catalog) = self.catalog {
+                    catalog.end_directory()?;
+                }
 
             } else if is_reg_file(&stat) {
 
@@ -777,7 +788,9 @@ impl <'a, W: Write> Encoder<'a, W> {
                 }
 
                 if let Some((target, offset)) = hardlink_target {
-
+                    if let Some(ref mut catalog) = self.catalog {
+                        catalog.add_hardlink(&filename)?;
+                    }
                     self.write_filename(&filename)?;
                     self.encode_hardlink(target.as_bytes(), offset)?;
 
@@ -792,6 +805,9 @@ impl <'a, W: Write> Encoder<'a, W> {
                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
                     };
 
+                    if let Some(ref mut catalog) = self.catalog {
+                        catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
+                    }
                     let child_magic = if dir_stat.st_dev != stat.st_dev {
                         detect_fs_type(filefd)?
                     } else {
@@ -805,6 +821,7 @@ impl <'a, W: Write> Encoder<'a, W> {
                 }
 
             } else if is_symlink(&stat) {
+
                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 
                 let res = filename.with_nix_path(|cstr| {
@@ -813,6 +830,9 @@ impl <'a, W: Write> Encoder<'a, W> {
 
                 match Errno::result(res) {
                     Ok(len) => {
+                        if let Some(ref mut catalog) = self.catalog {
+                            catalog.add_symlink(&filename)?;
+                        }
                         buffer[len as usize] = 0u8; // add Nul byte
                         self.write_filename(&filename)?;
                         self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
@@ -825,6 +845,13 @@ impl <'a, W: Write> Encoder<'a, W> {
                 }
             } else if is_block_dev(&stat) || is_char_dev(&stat) {
                 if self.has_features(flags::WITH_DEVICE_NODES) {
+                    if let Some(ref mut catalog) = self.catalog {
+                        if is_block_dev(&stat) {
+                            catalog.add_block_device(&filename)?;
+                        } else {
+                            catalog.add_char_device(&filename)?;
+                        }
+                    }
                     self.write_filename(&filename)?;
                     self.encode_device(&stat)?;
                 } else {
@@ -832,6 +859,9 @@ impl <'a, W: Write> Encoder<'a, W> {
                 }
             } else if is_fifo(&stat) {
                 if self.has_features(flags::WITH_FIFOS) {
+                    if let Some(ref mut catalog) = self.catalog {
+                        catalog.add_fifo(&filename)?;
+                    }
                     self.write_filename(&filename)?;
                     self.encode_special(&stat)?;
                 } else {
@@ -839,6 +869,9 @@ impl <'a, W: Write> Encoder<'a, W> {
                 }
             } else if is_socket(&stat) {
                 if self.has_features(flags::WITH_SOCKETS) {
+                    if let Some(ref mut catalog) = self.catalog {
+                        catalog.add_socket(&filename)?;
+                    }
                     self.write_filename(&filename)?;
                     self.encode_special(&stat)?;
                 } else {

tests/catar.rs

@@ -26,7 +26,8 @@ fn run_test(dir_name: &str) -> Result<(), Error> {
 
     let path = std::path::PathBuf::from(dir_name);
 
-    Encoder::encode(path, &mut dir, &mut writer, None, false, false, flags::DEFAULT)?;
+    let catalog = None::<&mut catalog::SimpleCatalog>;
+    Encoder::encode(path, &mut dir, &mut writer, catalog, None, false, false, flags::DEFAULT)?;
 
     Command::new("cmp")
         .arg("--verbose")