allow to abort verify jobs

And improve job description rendering on gui.

src/backup/verify.rs

@@ -34,7 +34,7 @@ fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -
     Ok(())
 }
 
-fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
+fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
     path.push(&info.filename);
@@ -51,6 +51,10 @@ fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &File
     }
 
     for pos in 0..index.index_count() {
+
+        worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
+
         let (start, end, digest) = index.chunk_info(pos).unwrap();
         let size = end - start;
         datastore.verify_stored_chunk(&digest, size)?;
@@ -59,7 +63,7 @@ fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &File
     Ok(())
 }
 
-fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
+fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> {
     let mut path = backup_dir.relative_path();
     path.push(&info.filename);
 
@@ -75,6 +79,10 @@ fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &Fi
     }
 
     for pos in 0..index.index_count() {
+
+        worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
+
         let chunk_info = index.chunk_info(pos).unwrap();
         let size = chunk_info.range.end - chunk_info.range.start;
         datastore.verify_stored_chunk(&chunk_info.digest, size)?;
@@ -88,14 +96,17 @@ fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &Fi
 /// This checks all archives inside a backup snapshot.
 /// Errors are logged to the worker log.
 ///
-/// Return true if verify is successful
-pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker: &WorkerTask) -> bool {
+/// Returns
+/// - Ok(true) if verify is successful
+/// - Ok(false) if there were verification errors
+/// - Err(_) if task was aborted
+pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker: &WorkerTask) -> Result<bool, Error> {
 
     let manifest = match datastore.load_manifest(&backup_dir) {
         Ok((manifest, _)) => manifest,
         Err(err) => {
             worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
-            return false;
+            return Ok(false);
         }
     };
 
@@ -107,27 +118,39 @@ pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker:
         let result = proxmox::try_block!({
             worker.log(format!("  check {}", info.filename));
             match archive_type(&info.filename)? {
-                ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info),
-                ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info),
+                ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info, worker),
+                ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info, worker),
                 ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
             }
         });
+
+        worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
+
         if let Err(err) = result {
             worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
             error_count += 1;
         }
     }
 
-    error_count == 0
+    Ok(error_count == 0)
 }
 
-pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> bool {
+/// Verify all backups inside a backup group
+///
+/// Errors are logged to the worker log.
+///
+/// Returns
+/// - Ok(true) if verify is successful
+/// - Ok(false) if there were verification errors
+/// - Err(_) if task was aborted
+pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<bool, Error> {
 
     let mut list = match group.list_backups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
             worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
-            return false;
+            return Ok(false);
         }
     };
 
@@ -137,21 +160,29 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &
 
     BackupInfo::sort_list(&mut list, false); // newest first
     for info in list {
-        if !verify_backup_dir(datastore, &info.backup_dir, worker) {
+        if !verify_backup_dir(datastore, &info.backup_dir, worker)? {
             error_count += 1;
         }
     }
 
-    error_count == 0
+    Ok(error_count == 0)
 }
 
-pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> bool {
+/// Verify all backups inside a datastore
+///
+/// Errors are logged to the worker log.
+///
+/// Returns
+/// - Ok(true) if verify is successful
+/// - Ok(false) if there were verification errors
+/// - Err(_) if task was aborted
+pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<bool, Error> {
 
     let list = match BackupGroup::list_groups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
-            return false;
+            return Ok(false);
         }
     };
 
@@ -159,10 +190,10 @@ pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> bool {
 
     let mut error_count = 0;
     for group in list {
-        if !verify_backup_group(datastore, &group, worker) {
+        if !verify_backup_group(datastore, &group, worker)? {
             error_count += 1;
         }
     }
 
-    error_count == 0
+    Ok(error_count == 0)
 }