api2: add verification admin endpoint and do_verification_job function

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>

src/api2/admin.rs

@@ -3,10 +3,12 @@ use proxmox::list_subdirs_api_method;
 
 pub mod datastore;
 pub mod sync;
+pub mod verify;
 
 const SUBDIRS: SubdirMap = &[
     ("datastore", &datastore::ROUTER),
-    ("sync", &sync::ROUTER)
+    ("sync", &sync::ROUTER),
+    ("verify", &verify::ROUTER)
 ];
 
 pub const ROUTER: Router = Router::new()

src/api2/admin/verify.rs

@@ -0,0 +1,107 @@
+use anyhow::{format_err, Error};
+
+use proxmox::api::router::SubdirMap;
+use proxmox::{list_subdirs_api_method, sortable};
+use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+
+use crate::api2::types::*;
+use crate::backup::do_verification_job;
+use crate::config::jobstate::{Job, JobState};
+use crate::config::verify;
+use crate::config::verify::{VerificationJobConfig, VerificationJobStatus};
+use serde_json::Value;
+use crate::tools::systemd::time::{parse_calendar_event, compute_next_event};
+use crate::server::UPID;
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "List configured jobs and their status.",
+        type: Array,
+        items: { type: verify::VerificationJobStatus },
+    },
+)]
+/// List all verification jobs
+pub fn list_verification_jobs(
+    _param: Value,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<VerificationJobStatus>, Error> {
+
+    let (config, digest) = verify::config()?;
+
+    let mut list: Vec<VerificationJobStatus> = config.convert_to_typed_array("verification")?;
+
+    for job in &mut list {
+        let last_state = JobState::load("verificationjob", &job.id)
+            .map_err(|err| format_err!("could not open statefile for {}: {}", &job.id, err))?;
+
+        let (upid, endtime, state, starttime) = match last_state {
+            JobState::Created { time } => (None, None, None, time),
+            JobState::Started { upid } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), None, None, parsed_upid.starttime)
+            },
+            JobState::Finished { upid, state } => {
+                let parsed_upid: UPID = upid.parse()?;
+                (Some(upid), Some(state.endtime()), Some(state.to_string()), parsed_upid.starttime)
+            },
+        };
+
+        job.last_run_upid = upid;
+        job.last_run_state = state;
+        job.last_run_endtime = endtime;
+
+        let last = job.last_run_endtime.unwrap_or_else(|| starttime);
+
+        job.next_run = (|| -> Option<i64> {
+            let schedule = job.schedule.as_ref()?;
+            let event = parse_calendar_event(&schedule).ok()?;
+            // ignore errors
+            compute_next_event(&event, last, false).unwrap_or_else(|_| None)
+        })();
+    }
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            }
+        }
+    }
+)]
+/// Runs a verification job manually.
+fn run_verification_job(
+    id: String,
+    _info: &ApiMethod,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+    let (config, _digest) = verify::config()?;
+    let verification_job: VerificationJobConfig = config.lookup("verification", &id)?;
+
+    let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
+    let job = Job::new("verificationjob", &id)?;
+
+    let upid_str = do_verification_job(job, verification_job, &userid, None)?;
+
+    Ok(upid_str)
+}
+
+#[sortable]
+const VERIFICATION_INFO_SUBDIRS: SubdirMap = &[("run", &Router::new().post(&API_METHOD_RUN_VERIFICATION_JOB))];
+
+const VERIFICATION_INFO_ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(VERIFICATION_INFO_SUBDIRS))
+    .subdirs(VERIFICATION_INFO_SUBDIRS);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_VERIFICATION_JOBS)
+    .match_all("id", &VERIFICATION_INFO_ROUTER);

src/backup/verify.rs

@@ -7,7 +7,10 @@ use nix::dir::Dir;
 use anyhow::{bail, format_err, Error};
 
 use crate::{
+    server::WorkerTask,
     api2::types::*,
+    config::jobstate::Job,
+    config::verify::VerificationJobConfig,
     backup::{
         DataStore,
         DataBlob,
@@ -526,3 +529,96 @@ pub fn verify_all_backups(
 
     Ok(errors)
 }
+
+/// Runs a verification job.
+pub fn do_verification_job(
+    mut job: Job,
+    verification_job: VerificationJobConfig,
+    userid: &Userid,
+    schedule: Option<String>,
+) -> Result<String, Error> {
+    let datastore = DataStore::lookup_datastore(&verification_job.store)?;
+
+    let mut backups_to_verify = BackupInfo::list_backups(&datastore.base_path())?;
+    if verification_job.ignore_verified.unwrap_or(true) {
+        backups_to_verify.retain(|backup_info| {
+            let manifest = match datastore.load_manifest(&backup_info.backup_dir) {
+                Ok((manifest, _)) => manifest,
+                Err(_) => return false,
+            };
+
+            let raw_verify_state = manifest.unprotected["verify_state"].clone();
+            let last_state = match serde_json::from_value::<SnapshotVerifyState>(raw_verify_state) {
+                Ok(last_state) => last_state,
+                Err(_) => return true,
+            };
+
+            let now = proxmox::tools::time::epoch_i64();
+            let days_since_last_verify = (now - last_state.upid.starttime) / 86400;
+            verification_job.outdated_after.is_some()
+                && days_since_last_verify > verification_job.outdated_after.unwrap()
+        })
+    }
+
+    let job_id = job.jobname().to_string();
+    let worker_type = job.jobtype().to_string();
+    let upid_str = WorkerTask::new_thread(
+        &worker_type,
+        Some(job.jobname().to_string()),
+        userid.clone(),
+        false,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+
+            task_log!(worker,"Starting datastore verify job '{}'", job_id);
+            task_log!(worker,"verifying {} backups", backups_to_verify.len());
+            if let Some(event_str) = schedule {
+                task_log!(worker,"task triggered by schedule '{}'", event_str);
+            }
+
+            let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024 * 16)));
+            let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+            let result = proxmox::try_block!({
+                let mut failed_dirs: Vec<String> = Vec::new();
+
+                for backup_info in backups_to_verify {
+                    let verification_result = verify_backup_dir(
+                        datastore.clone(),
+                        &backup_info.backup_dir,
+                        verified_chunks.clone(),
+                        corrupt_chunks.clone(),
+                        worker.clone(),
+                        worker.upid().clone()
+                    );
+
+                    if let Ok(false) = verification_result {
+                        failed_dirs.push(backup_info.backup_dir.to_string());
+                    } // otherwise successful or aborted
+                }
+
+                if !failed_dirs.is_empty() {
+                    task_log!(worker,"Failed to verify following snapshots:",);
+                    for dir in failed_dirs {
+                        task_log!(worker, "\t{}", dir)
+                    }
+                    bail!("verification failed - please check the log for details");
+                }
+                Ok(())
+            });
+
+            let status = worker.create_state(&result);
+
+            match job.finish(status) {
+                Err(err) => eprintln!(
+                    "could not finish job state for {}: {}",
+                    job.jobtype().to_string(),
+                    err
+                ),
+                Ok(_) => (),
+            }
+
+            result
+        },
+    )?;
+    Ok(upid_str)
+}