diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 4e49615c..1f349c8c 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::sync::{Arc}; use std::path::{Path, PathBuf}; use anyhow::{bail, format_err, Error}; @@ -196,6 +196,7 @@ async fn schedule_tasks() -> Result<(), Error> { schedule_datastore_garbage_collection().await; schedule_datastore_prune().await; + schedule_datastore_verification().await; schedule_datastore_sync_jobs().await; Ok(()) @@ -463,6 +464,107 @@ async fn schedule_datastore_prune() { } } +async fn schedule_datastore_verification() { + use proxmox_backup::backup::{DataStore, verify_all_backups}; + use proxmox_backup::server::{WorkerTask}; + use proxmox_backup::config::datastore::{self, DataStoreConfig}; + use proxmox_backup::tools::systemd::time::{ + parse_calendar_event, compute_next_event}; + + let config = match datastore::config() { + Err(err) => { + eprintln!("unable to read datastore config - {}", err); + return; + } + Ok((config, _digest)) => config, + }; + + for (store, (_, store_config)) in config.sections { + let datastore = match DataStore::lookup_datastore(&store) { + Ok(datastore) => datastore, + Err(err) => { + eprintln!("lookup_datastore failed - {}", err); + continue; + } + }; + + let store_config: DataStoreConfig = match serde_json::from_value(store_config) { + Ok(c) => c, + Err(err) => { + eprintln!("datastore config from_value failed - {}", err); + continue; + } + }; + + let event_str = match store_config.verify_schedule { + Some(event_str) => event_str, + None => continue, + }; + + let event = match parse_calendar_event(&event_str) { + Ok(event) => event, + Err(err) => { + eprintln!("unable to parse schedule '{}' - {}", event_str, err); + continue; + } + }; + + let worker_type = "verify"; + + let last = match lookup_last_worker(worker_type, &store) { + Ok(Some(upid)) => { + if proxmox_backup::server::worker_is_active_local(&upid) { + continue; + } + upid.starttime + } + Ok(None) => 0, + Err(err) => { + eprintln!("lookup_last_job_start failed: {}", err); + continue; + } + }; + + let next = match compute_next_event(&event, last, false) { + Ok(Some(next)) => next, + Ok(None) => continue, + Err(err) => { + eprintln!("compute_next_event for '{}' failed - {}", event_str, err); + continue; + } + }; + + let now = proxmox::tools::time::epoch_i64(); + + if next > now { continue; } + + let worker_id = store.clone(); + let store2 = store.clone(); + if let Err(err) = WorkerTask::new_thread( + worker_type, + Some(worker_id), + Userid::backup_userid().clone(), + false, + move |worker| { + worker.log(format!("starting verification on store {}", store2)); + worker.log(format!("task triggered by schedule '{}'", event_str)); + if let Ok(failed_dirs) = verify_all_backups(datastore, worker.clone()) { + if failed_dirs.len() > 0 { + worker.log("Failed to verify following snapshots:"); + for dir in failed_dirs { + worker.log(format!("\t{}", dir)); + } + bail!("verification failed - please check the log for details"); + } + } + Ok(()) + }, + ) { + eprintln!("unable to start verification on store {} - {}", store, err); + } + } +} + async fn schedule_datastore_sync_jobs() { use proxmox_backup::{