From 8d1beca7e876548bf430811977e833aab9226df7 Mon Sep 17 00:00:00 2001 From: Hannes Laimer Date: Tue, 20 Oct 2020 11:10:05 +0200 Subject: [PATCH] api2: add verification admin endpoint and do_verification_job function Signed-off-by: Hannes Laimer --- src/api2/admin.rs | 4 +- src/api2/admin/verify.rs | 107 +++++++++++++++++++++++++++++++++++++++ src/backup/verify.rs | 96 +++++++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 src/api2/admin/verify.rs diff --git a/src/api2/admin.rs b/src/api2/admin.rs index b927ce1e..79ce29f3 100644 --- a/src/api2/admin.rs +++ b/src/api2/admin.rs @@ -3,10 +3,12 @@ use proxmox::list_subdirs_api_method; pub mod datastore; pub mod sync; +pub mod verify; const SUBDIRS: SubdirMap = &[ ("datastore", &datastore::ROUTER), - ("sync", &sync::ROUTER) + ("sync", &sync::ROUTER), + ("verify", &verify::ROUTER) ]; pub const ROUTER: Router = Router::new() diff --git a/src/api2/admin/verify.rs b/src/api2/admin/verify.rs new file mode 100644 index 00000000..f61373a0 --- /dev/null +++ b/src/api2/admin/verify.rs @@ -0,0 +1,107 @@ +use anyhow::{format_err, Error}; + +use proxmox::api::router::SubdirMap; +use proxmox::{list_subdirs_api_method, sortable}; +use proxmox::api::{api, ApiMethod, Router, RpcEnvironment}; + +use crate::api2::types::*; +use crate::backup::do_verification_job; +use crate::config::jobstate::{Job, JobState}; +use crate::config::verify; +use crate::config::verify::{VerificationJobConfig, VerificationJobStatus}; +use serde_json::Value; +use crate::tools::systemd::time::{parse_calendar_event, compute_next_event}; +use crate::server::UPID; + +#[api( + input: { + properties: {}, + }, + returns: { + description: "List configured jobs and their status.", + type: Array, + items: { type: verify::VerificationJobStatus }, + }, +)] +/// List all verification jobs +pub fn list_verification_jobs( + _param: Value, + mut rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + + let (config, digest) = verify::config()?; + + let mut list: Vec = config.convert_to_typed_array("verification")?; + + for job in &mut list { + let last_state = JobState::load("verificationjob", &job.id) + .map_err(|err| format_err!("could not open statefile for {}: {}", &job.id, err))?; + + let (upid, endtime, state, starttime) = match last_state { + JobState::Created { time } => (None, None, None, time), + JobState::Started { upid } => { + let parsed_upid: UPID = upid.parse()?; + (Some(upid), None, None, parsed_upid.starttime) + }, + JobState::Finished { upid, state } => { + let parsed_upid: UPID = upid.parse()?; + (Some(upid), Some(state.endtime()), Some(state.to_string()), parsed_upid.starttime) + }, + }; + + job.last_run_upid = upid; + job.last_run_state = state; + job.last_run_endtime = endtime; + + let last = job.last_run_endtime.unwrap_or_else(|| starttime); + + job.next_run = (|| -> Option { + let schedule = job.schedule.as_ref()?; + let event = parse_calendar_event(&schedule).ok()?; + // ignore errors + compute_next_event(&event, last, false).unwrap_or_else(|_| None) + })(); + } + + rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into(); + + Ok(list) +} + +#[api( + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + } + } + } +)] +/// Runs a verification job manually. +fn run_verification_job( + id: String, + _info: &ApiMethod, + rpcenv: &mut dyn RpcEnvironment, +) -> Result { + let (config, _digest) = verify::config()?; + let verification_job: VerificationJobConfig = config.lookup("verification", &id)?; + + let userid: Userid = rpcenv.get_user().unwrap().parse()?; + + let job = Job::new("verificationjob", &id)?; + + let upid_str = do_verification_job(job, verification_job, &userid, None)?; + + Ok(upid_str) +} + +#[sortable] +const VERIFICATION_INFO_SUBDIRS: SubdirMap = &[("run", &Router::new().post(&API_METHOD_RUN_VERIFICATION_JOB))]; + +const VERIFICATION_INFO_ROUTER: Router = Router::new() + .get(&list_subdirs_api_method!(VERIFICATION_INFO_SUBDIRS)) + .subdirs(VERIFICATION_INFO_SUBDIRS); + +pub const ROUTER: Router = Router::new() + .get(&API_METHOD_LIST_VERIFICATION_JOBS) + .match_all("id", &VERIFICATION_INFO_ROUTER); diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 2103da40..abba38a5 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -7,7 +7,10 @@ use nix::dir::Dir; use anyhow::{bail, format_err, Error}; use crate::{ + server::WorkerTask, api2::types::*, + config::jobstate::Job, + config::verify::VerificationJobConfig, backup::{ DataStore, DataBlob, @@ -526,3 +529,96 @@ pub fn verify_all_backups( Ok(errors) } + +/// Runs a verification job. +pub fn do_verification_job( + mut job: Job, + verification_job: VerificationJobConfig, + userid: &Userid, + schedule: Option, +) -> Result { + let datastore = DataStore::lookup_datastore(&verification_job.store)?; + + let mut backups_to_verify = BackupInfo::list_backups(&datastore.base_path())?; + if verification_job.ignore_verified.unwrap_or(true) { + backups_to_verify.retain(|backup_info| { + let manifest = match datastore.load_manifest(&backup_info.backup_dir) { + Ok((manifest, _)) => manifest, + Err(_) => return false, + }; + + let raw_verify_state = manifest.unprotected["verify_state"].clone(); + let last_state = match serde_json::from_value::(raw_verify_state) { + Ok(last_state) => last_state, + Err(_) => return true, + }; + + let now = proxmox::tools::time::epoch_i64(); + let days_since_last_verify = (now - last_state.upid.starttime) / 86400; + verification_job.outdated_after.is_some() + && days_since_last_verify > verification_job.outdated_after.unwrap() + }) + } + + let job_id = job.jobname().to_string(); + let worker_type = job.jobtype().to_string(); + let upid_str = WorkerTask::new_thread( + &worker_type, + Some(job.jobname().to_string()), + userid.clone(), + false, + move |worker| { + job.start(&worker.upid().to_string())?; + + task_log!(worker,"Starting datastore verify job '{}'", job_id); + task_log!(worker,"verifying {} backups", backups_to_verify.len()); + if let Some(event_str) = schedule { + task_log!(worker,"task triggered by schedule '{}'", event_str); + } + + let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024 * 16))); + let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64))); + let result = proxmox::try_block!({ + let mut failed_dirs: Vec = Vec::new(); + + for backup_info in backups_to_verify { + let verification_result = verify_backup_dir( + datastore.clone(), + &backup_info.backup_dir, + verified_chunks.clone(), + corrupt_chunks.clone(), + worker.clone(), + worker.upid().clone() + ); + + if let Ok(false) = verification_result { + failed_dirs.push(backup_info.backup_dir.to_string()); + } // otherwise successful or aborted + } + + if !failed_dirs.is_empty() { + task_log!(worker,"Failed to verify following snapshots:",); + for dir in failed_dirs { + task_log!(worker, "\t{}", dir) + } + bail!("verification failed - please check the log for details"); + } + Ok(()) + }); + + let status = worker.create_state(&result); + + match job.finish(status) { + Err(err) => eprintln!( + "could not finish job state for {}: {}", + job.jobtype().to_string(), + err + ), + Ok(_) => (), + } + + result + }, + )?; + Ok(upid_str) +}