From 8ea00f6e4923d3d3f42184f40c514ad31b1d0e5d Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 25 Jun 2020 12:55:34 +0200 Subject: [PATCH] allow to abort verify jobs And improve job description rendering on gui. --- src/api2/admin/datastore.rs | 16 +++++----- src/backup/verify.rs | 63 +++++++++++++++++++++++++++---------- www/Utils.js | 6 +++- 3 files changed, 60 insertions(+), 25 deletions(-) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index 460bf2ff..dae045b1 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -434,7 +434,7 @@ pub fn verify( ) -> Result { let datastore = DataStore::lookup_datastore(&store)?; - let what; + let worker_id; let mut backup_dir = None; let mut backup_group = None; @@ -442,16 +442,16 @@ pub fn verify( match (backup_type, backup_id, backup_time) { (Some(backup_type), Some(backup_id), Some(backup_time)) => { let dir = BackupDir::new(backup_type, backup_id, backup_time); - what = format!("{}:{}", store, dir); + worker_id = format!("{}_{}", store, dir); backup_dir = Some(dir); } (Some(backup_type), Some(backup_id), None) => { let group = BackupGroup::new(backup_type, backup_id); - what = format!("{}:{}", store, group); + worker_id = format!("{}_{}", store, group); backup_group = Some(group); } (None, None, None) => { - what = store.clone(); + worker_id = store.clone(); } _ => bail!("parameters do not spefify a backup group or snapshot"), } @@ -460,14 +460,14 @@ pub fn verify( let to_stdout = if rpcenv.env_type() == RpcEnvironmentType::CLI { true } else { false }; let upid_str = WorkerTask::new_thread( - "verify", Some(what.clone()), &username, to_stdout, move |worker| + "verify", Some(worker_id.clone()), &username, to_stdout, move |worker| { let success = if let Some(backup_dir) = backup_dir { - verify_backup_dir(&datastore, &backup_dir, &worker) + verify_backup_dir(&datastore, &backup_dir, &worker)? } else if let Some(backup_group) = backup_group { - verify_backup_group(&datastore, &backup_group, &worker) + verify_backup_group(&datastore, &backup_group, &worker)? } else { - verify_all_backups(&datastore, &worker) + verify_all_backups(&datastore, &worker)? }; if !success { bail!("verfication failed - please check the log for details"); diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 1a554e28..026c5e6b 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -34,7 +34,7 @@ fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) - Ok(()) } -fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> { +fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> { let mut path = backup_dir.relative_path(); path.push(&info.filename); @@ -51,6 +51,10 @@ fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &File } for pos in 0..index.index_count() { + + worker.fail_on_abort()?; + crate::tools::fail_on_shutdown()?; + let (start, end, digest) = index.chunk_info(pos).unwrap(); let size = end - start; datastore.verify_stored_chunk(&digest, size)?; @@ -59,7 +63,7 @@ fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &File Ok(()) } -fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> { +fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> { let mut path = backup_dir.relative_path(); path.push(&info.filename); @@ -75,6 +79,10 @@ fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &Fi } for pos in 0..index.index_count() { + + worker.fail_on_abort()?; + crate::tools::fail_on_shutdown()?; + let chunk_info = index.chunk_info(pos).unwrap(); let size = chunk_info.range.end - chunk_info.range.start; datastore.verify_stored_chunk(&chunk_info.digest, size)?; @@ -88,14 +96,17 @@ fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &Fi /// This checks all archives inside a backup snapshot. /// Errors are logged to the worker log. /// -/// Return true if verify is successful -pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker: &WorkerTask) -> bool { +/// Returns +/// - Ok(true) if verify is successful +/// - Ok(false) if there were verification errors +/// - Err(_) if task was aborted +pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker: &WorkerTask) -> Result { let manifest = match datastore.load_manifest(&backup_dir) { Ok((manifest, _)) => manifest, Err(err) => { worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err)); - return false; + return Ok(false); } }; @@ -107,27 +118,39 @@ pub fn verify_backup_dir(datastore: &DataStore, backup_dir: &BackupDir, worker: let result = proxmox::try_block!({ worker.log(format!(" check {}", info.filename)); match archive_type(&info.filename)? { - ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info), - ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info), + ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info, worker), + ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info, worker), ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info), } }); + + worker.fail_on_abort()?; + crate::tools::fail_on_shutdown()?; + if let Err(err) = result { worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err)); error_count += 1; } } - error_count == 0 + Ok(error_count == 0) } -pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> bool { +/// Verify all backups inside a backup group +/// +/// Errors are logged to the worker log. +/// +/// Returns +/// - Ok(true) if verify is successful +/// - Ok(false) if there were verification errors +/// - Err(_) if task was aborted +pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result { let mut list = match group.list_backups(&datastore.base_path()) { Ok(list) => list, Err(err) => { worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err)); - return false; + return Ok(false); } }; @@ -137,21 +160,29 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: & BackupInfo::sort_list(&mut list, false); // newest first for info in list { - if !verify_backup_dir(datastore, &info.backup_dir, worker) { + if !verify_backup_dir(datastore, &info.backup_dir, worker)? { error_count += 1; } } - error_count == 0 + Ok(error_count == 0) } -pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> bool { +/// Verify all backups inside a datastore +/// +/// Errors are logged to the worker log. +/// +/// Returns +/// - Ok(true) if verify is successful +/// - Ok(false) if there were verification errors +/// - Err(_) if task was aborted +pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result { let list = match BackupGroup::list_groups(&datastore.base_path()) { Ok(list) => list, Err(err) => { worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err)); - return false; + return Ok(false); } }; @@ -159,10 +190,10 @@ pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> bool { let mut error_count = 0; for group in list { - if !verify_backup_group(datastore, &group, worker) { + if !verify_backup_group(datastore, &group, worker)? { error_count += 1; } } - error_count == 0 + Ok(error_count == 0) } diff --git a/www/Utils.js b/www/Utils.js index e6c281bd..662b7ce2 100644 --- a/www/Utils.js +++ b/www/Utils.js @@ -38,8 +38,9 @@ Ext.define('PBS.Utils', { let datastore = result[1], type = result[2], id = result[3]; return `Datastore ${datastore} - ${what} ${type}/${id}`; } - return what; + return `Datastore ${id} - ${what}`; }, + render_datastore_time_worker_id: function(id, what) { const res = id.match(/^(\S+)_([^_\s]+)_([^_\s]+)_([^_\s]+)$/); if (res) { @@ -62,6 +63,9 @@ Ext.define('PBS.Utils', { prune: (type, id) => { return PBS.Utils.render_datastore_worker_id(id, gettext('Prune')); }, + verify: (type, id) => { + return PBS.Utils.render_datastore_worker_id(id, gettext('Verify')); + }, backup: (type, id) => { return PBS.Utils.render_datastore_worker_id(id, gettext('Backup')); },