allow to abort verify jobs

And improve job description rendering on gui.

src/api2/admin/datastore.rs

@@ -434,7 +434,7 @@ pub fn verify(
 ) -> Result<Value, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
 
-    let what;
+    let worker_id;
 
     let mut backup_dir = None;
     let mut backup_group = None;
@@ -442,16 +442,16 @@ pub fn verify(
     match (backup_type, backup_id, backup_time) {
         (Some(backup_type), Some(backup_id), Some(backup_time)) => {
             let dir = BackupDir::new(backup_type, backup_id, backup_time);
-            what = format!("{}:{}", store, dir);
+            worker_id = format!("{}_{}", store, dir);
             backup_dir = Some(dir);
         }
         (Some(backup_type), Some(backup_id), None) => {
             let group = BackupGroup::new(backup_type, backup_id);
-            what = format!("{}:{}", store, group);
+            worker_id = format!("{}_{}", store, group);
             backup_group = Some(group);
         }
         (None, None, None) => {
-            what = store.clone();
+            worker_id = store.clone();
         }
         _ => bail!("parameters do not spefify a backup group or snapshot"),
     }
@@ -460,14 +460,14 @@ pub fn verify(
     let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false };
 
     let upid_str = WorkerTask::new_thread(
-        "verify", Some(what.clone()), &username, to_stdout, move |worker|
+        "verify", Some(worker_id.clone()), &username, to_stdout, move |worker|
         {
             let success = if let Some(backup_dir) = backup_dir {
-                verify_backup_dir(&datastore, &backup_dir, &worker)
+                verify_backup_dir(&datastore, &backup_dir, &worker)?
             } else if let Some(backup_group) = backup_group {
-                verify_backup_group(&datastore, &backup_group, &worker)
+                verify_backup_group(&datastore, &backup_group, &worker)?
             } else {
-                verify_all_backups(&datastore, &worker)
+                verify_all_backups(&datastore, &worker)?
             };
             if !success {
                 bail!("verfication failed - please check the log for details");

src/backup/verify.rs

@@ -34,7 +34,7 @@ fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -
     Ok(())
 }
 
-fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
+fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> {
 
     let mut path = backup_dir.relative_path();
     path.push(&info.filename);
@@ -51,6 +51,10 @@ fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &File
     }
 
     for pos in 0..index.index_count() {
+
+        worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
+
         let (start, end, digest) = index.chunk_info(pos).unwrap();
         let size = end - start;
         datastore.verify_stored_chunk(&digest, size)?;
@@ -59,7 +63,7 @@ fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &File
     Ok(())
 }
 
-fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
+fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> {
     let mut path = backup_dir.relative_path();
     path.push(&info.filename);
 
@@ -75,6 +79,10 @@ fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &Fi
     }
 
     for pos in 0..index.index_count() {
+
+        worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
+
         let chunk_info = index.chunk_info(pos).unwrap();
         let size = chunk_info.range.end - chunk_info.range.start;
         datastore.verify_stored_chunk(&chunk_info.digest, size)?;
@@ -88,14 +96,17 @@ fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &Fi
 /// This checks all archives inside a backup snapshot.
 /// Errors are logged to the worker log.
 ///
-/// Return true if verify is successful
+/// Returns
-pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker: &WorkerTask) -> bool {
+/// - Ok(true) if verify is successful
+/// - Ok(false) if there were verification errors
+/// - Err(_) if task was aborted
+pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker: &WorkerTask) -> Result<bool, Error> {
 
     let manifest = match datastore.load_manifest(&backup_dir) {
         Ok((manifest, _)) => manifest,
         Err(err) => {
             worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
-            return false;
+            return Ok(false);
         }
     };
 
@@ -107,27 +118,39 @@ pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker:
         let result = proxmox::try_block!({
             worker.log(format!("  check {}", info.filename));
             match archive_type(&info.filename)? {
-                ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info),
+                ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info, worker),
-                ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info),
+                ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info, worker),
                 ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
             }
         });
+
+        worker.fail_on_abort()?;
+        crate::tools::fail_on_shutdown()?;
+
         if let Err(err) = result {
             worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
             error_count += 1;
         }
     }
 
-    error_count == 0
+    Ok(error_count == 0)
 }
 
-pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> bool {
+/// Verify all backups inside a backup group
+///
+/// Errors are logged to the worker log.
+///
+/// Returns
+/// - Ok(true) if verify is successful
+/// - Ok(false) if there were verification errors
+/// - Err(_) if task was aborted
+pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<bool, Error> {
 
     let mut list = match group.list_backups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
             worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
-            return false;
+            return Ok(false);
         }
     };
 
@@ -137,21 +160,29 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &
 
     BackupInfo::sort_list(&mut list, false); // newest first
     for info in list {
-        if !verify_backup_dir(datastore, &info.backup_dir, worker) {
+        if !verify_backup_dir(datastore, &info.backup_dir, worker)? {
             error_count += 1;
         }
     }
 
-    error_count == 0
+    Ok(error_count == 0)
 }
 
-pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> bool {
+/// Verify all backups inside a datastore
+///
+/// Errors are logged to the worker log.
+///
+/// Returns
+/// - Ok(true) if verify is successful
+/// - Ok(false) if there were verification errors
+/// - Err(_) if task was aborted
+pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<bool, Error> {
 
     let list = match BackupGroup::list_groups(&datastore.base_path()) {
         Ok(list) => list,
         Err(err) => {
             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
-            return false;
+            return Ok(false);
         }
     };
 
@@ -159,10 +190,10 @@ pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> bool {
 
     let mut error_count = 0;
     for group in list {
-        if !verify_backup_group(datastore, &group, worker) {
+        if !verify_backup_group(datastore, &group, worker)? {
             error_count += 1;
         }
     }
 
-    error_count == 0
+    Ok(error_count == 0)
 }

www/Utils.js

@@ -38,8 +38,9 @@ Ext.define('PBS.Utils', {
 	    let datastore = result[1], type = result[2], id = result[3];
 	    return `Datastore ${datastore} - ${what} ${type}/${id}`;
 	}
-	return what;
+	return `Datastore ${id} - ${what}`;
     },
+
     render_datastore_time_worker_id: function(id, what) {
 	const res = id.match(/^(\S+)_([^_\s]+)_([^_\s]+)_([^_\s]+)$/);
 	if (res) {
@@ -62,6 +63,9 @@ Ext.define('PBS.Utils', {
 	    prune: (type, id) => {
 		return PBS.Utils.render_datastore_worker_id(id, gettext('Prune'));
 	    },
+	    verify: (type, id) => {
+		return PBS.Utils.render_datastore_worker_id(id, gettext('Verify'));
+	    },
 	    backup: (type, id) => {
 		return PBS.Utils.render_datastore_worker_id(id, gettext('Backup'));
 	    },