bump version to 0.8.12-1

We need to do the check before we load the chunk.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "proxmox-backup"
-version = "0.8.11"
+version = "0.8.12"
 authors = ["Dietmar Maurer <dietmar@proxmox.com>"]
 edition = "2018"
 license = "AGPL-3"
@@ -43,7 +43,7 @@ proxmox = { version = "0.3.3", features = [ "sortable-macro", "api-macro", "webs
 #proxmox = { git = "ssh://gitolite3@proxdev.maurer-it.com/rust/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
 #proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro", "websocket" ] }
 proxmox-fuse = "0.1.0"
-pxar = { version = "0.3.0", features = [ "tokio-io", "futures-io" ] }
+pxar = { version = "0.4.0", features = [ "tokio-io", "futures-io" ] }
 #pxar = { path = "../pxar", features = [ "tokio-io", "futures-io" ] }
 regex = "1.2"
 rustyline = "6"

debian/changelog

@@ -1,3 +1,13 @@
+rust-proxmox-backup (0.8.12-1) unstable; urgency=medium
+
+  * verify: speedup - only verify chunks once
+
+  * verify: sort backup groups
+
+  * bump pxar dep to 0.4.0
+
+ -- Proxmox Support Team <support@proxmox.com>  Tue, 25 Aug 2020 08:55:52 +0200
+
 rust-proxmox-backup (0.8.11-1) unstable; urgency=medium
 
   * improve sync jobs, allow to stop them and better logging

docs/epilog.rst

@@ -13,7 +13,8 @@
 .. _Proxmox: https://www.proxmox.com
 .. _Proxmox Community Forum: https://forum.proxmox.com
 .. _Proxmox Virtual Environment: https://www.proxmox.com/proxmox-ve
-.. _Proxmox Backup: https://pbs.proxmox.com/wiki/index.php/Main_Page  // FIXME
+  // FIXME
+.. _Proxmox Backup: https://pbs.proxmox.com/wiki/index.php/Main_Page
 .. _PBS Development List: https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
 .. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html
 .. _Rust: https://www.rust-lang.org/

src/backup/backup_info.rs

@@ -45,6 +45,31 @@ pub struct BackupGroup {
     backup_id: String,
 }
 
+impl std::cmp::Ord for BackupGroup {
+
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        let type_order = self.backup_type.cmp(&other.backup_type);
+        if type_order != std::cmp::Ordering::Equal {
+            return type_order;
+        }
+        // try to compare IDs numerically
+        let id_self = self.backup_id.parse::<u64>();
+        let id_other = other.backup_id.parse::<u64>();
+        match (id_self, id_other) {
+            (Ok(id_self), Ok(id_other)) => id_self.cmp(&id_other),
+            (Ok(_), Err(_)) => std::cmp::Ordering::Less,
+            (Err(_), Ok(_)) => std::cmp::Ordering::Greater,
+             _ => self.backup_id.cmp(&other.backup_id),
+        }
+    }
+}
+
+impl std::cmp::PartialOrd for BackupGroup {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 impl BackupGroup {
 
     pub fn new<T: Into<String>, U: Into<String>>(backup_type: T, backup_id: U) -> Self {

src/backup/verify.rs

@@ -50,7 +50,17 @@ fn verify_index_chunks(
         worker.fail_on_abort()?;
 
         let info = index.chunk_info(pos).unwrap();
-        let size = info.range.end - info.range.start;
+
+        if verified_chunks.contains(&info.digest) {
+            continue; // already verified
+        }
+
+        if corrupt_chunks.contains(&info.digest) {
+            let digest_str = proxmox::tools::digest_to_hex(&info.digest);
+            worker.log(format!("chunk {} was marked as corrupt", digest_str));
+            errors += 1;
+            continue;
+        }
 
         let chunk = match datastore.load_chunk(&info.digest) {
             Err(err) => {
@@ -81,20 +91,14 @@ fn verify_index_chunks(
             errors += 1;
         }
 
-        if !verified_chunks.contains(&info.digest) {
+        let size = info.range.end - info.range.start;
-            if !corrupt_chunks.contains(&info.digest) {
+
-                if let Err(err) = chunk.verify_unencrypted(size as usize, &info.digest) {
+        if let Err(err) = chunk.verify_unencrypted(size as usize, &info.digest) {
-                    corrupt_chunks.insert(info.digest);
+            corrupt_chunks.insert(info.digest);
-                    worker.log(format!("{}", err));
+            worker.log(format!("{}", err));
-                    errors += 1;
+            errors += 1;
-                } else {
+        } else {
-                    verified_chunks.insert(info.digest);
+            verified_chunks.insert(info.digest);
-                }
-            } else {
-                let digest_str = proxmox::tools::digest_to_hex(&info.digest);
-                worker.log(format!("chunk {} was marked as corrupt", digest_str));
-                errors += 1;
-            }
         }
     }
 
@@ -267,7 +271,7 @@ pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<
 
     let mut errors = Vec::new();
 
-    let list = match BackupGroup::list_groups(&datastore.base_path()) {
+    let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
@@ -275,6 +279,8 @@ pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<
         }
     };
 
+    list.sort_unstable();
+
     worker.log(format!("verify datastore {}", datastore.name()));
 
     for group in list {