add verification scheduling to proxmox-backup-proxy

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>

src/bin/proxmox-backup-proxy.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::sync::{Arc};
 use std::path::{Path, PathBuf};
 
 use anyhow::{bail, format_err, Error};
@@ -196,6 +196,7 @@ async fn schedule_tasks() -> Result<(), Error> {
 
     schedule_datastore_garbage_collection().await;
     schedule_datastore_prune().await;
+    schedule_datastore_verification().await;
     schedule_datastore_sync_jobs().await;
 
     Ok(())
@@ -463,6 +464,107 @@ async fn schedule_datastore_prune() {
     }
 }
 
+async fn schedule_datastore_verification() {
+    use proxmox_backup::backup::{DataStore, verify_all_backups};
+    use proxmox_backup::server::{WorkerTask};
+    use proxmox_backup::config::datastore::{self, DataStoreConfig};
+    use proxmox_backup::tools::systemd::time::{
+        parse_calendar_event, compute_next_event};
+
+    let config = match datastore::config() {
+        Err(err) => {
+            eprintln!("unable to read datastore config - {}", err);
+            return;
+        }
+        Ok((config, _digest)) => config,
+    };
+
+    for (store, (_, store_config)) in config.sections {
+        let datastore = match DataStore::lookup_datastore(&store) {
+            Ok(datastore) => datastore,
+            Err(err) => {
+                eprintln!("lookup_datastore failed - {}", err);
+                continue;
+            }
+        };
+
+        let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("datastore config from_value failed - {}", err);
+                continue;
+            }
+        };
+
+        let event_str = match store_config.verify_schedule {
+            Some(event_str) => event_str,
+            None => continue,
+        };
+
+        let event = match parse_calendar_event(&event_str) {
+            Ok(event) => event,
+            Err(err) => {
+                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let worker_type = "verify";
+
+        let last = match lookup_last_worker(worker_type, &store) {
+            Ok(Some(upid)) => {
+                if proxmox_backup::server::worker_is_active_local(&upid) {
+                    continue;
+                }
+                upid.starttime
+            }
+            Ok(None) => 0,
+            Err(err) => {
+                eprintln!("lookup_last_job_start failed: {}", err);
+                continue;
+            }
+        };
+
+        let next = match compute_next_event(&event, last, false) {
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
+            Err(err) => {
+                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let now = proxmox::tools::time::epoch_i64();
+
+        if next > now { continue; }
+
+        let worker_id = store.clone();
+        let store2 = store.clone();
+        if let Err(err) = WorkerTask::new_thread(
+            worker_type,
+            Some(worker_id),
+            Userid::backup_userid().clone(),
+            false,
+            move |worker| {
+                worker.log(format!("starting verification on store {}", store2));
+                worker.log(format!("task triggered by schedule '{}'", event_str));
+                if let Ok(failed_dirs) = verify_all_backups(datastore, worker.clone()) {
+                    if failed_dirs.len() > 0 {
+                        worker.log("Failed to verify following snapshots:");
+                        for dir in failed_dirs {
+                            worker.log(format!("\t{}", dir));
+                        }
+                        bail!("verification failed - please check the log for details");
+                    }
+                }
+                Ok(())
+            },
+        ) {
+            eprintln!("unable to start verification on store {} - {}", store, err);
+        }
+    }
+}
+
 async fn schedule_datastore_sync_jobs() {
 
     use proxmox_backup::{