tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

move jobstate to server

Browse Source
This commit is contained in:
Dietmar Maurer 2020-10-28 07:33:05 +01:00
parent a12388d177
commit 1298618a83
12 changed files with 63 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/api2/admin/sync.rs
View File

@ -9,7 +9,7 @@ use crate::api2::types::*;
use crate::api2::pull::do_sync_job;
use crate::config::sync::{self, SyncJobStatus, SyncJobConfig};
use crate::server::UPID;
use crate::config::jobstate::{Job, JobState};
use crate::server::jobstate::{Job, JobState};
use crate::tools::systemd::time::{
parse_calendar_event, compute_next_event};

4
src/api2/admin/verify.rs
View File

@ -5,8 +5,8 @@ use proxmox::{list_subdirs_api_method, sortable};
use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
use crate::api2::types::*;
use crate::backup::do_verification_job;
use crate::config::jobstate::{Job, JobState};
use crate::server::do_verification_job;
use crate::server::jobstate::{Job, JobState};
use crate::config::verify;
use crate::config::verify::{VerificationJobConfig, VerificationJobStatus};
use serde_json::Value;

13
src/api2/config/datastore.rs
View File

@ -12,6 +12,7 @@ use crate::backup::*;
use crate::config::cached_user_info::CachedUserInfo;
use crate::config::datastore::{self, DataStoreConfig, DIR_NAME_SCHEMA};
use crate::config::acl::{PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
use crate::server::jobstate;
#[api(
input: {
@ -127,8 +128,8 @@ pub fn create_datastore(param: Value) -> Result<(), Error> {
datastore::save_config(&config)?;
crate::config::jobstate::create_state_file("prune", &datastore.name)?;
crate::config::jobstate::create_state_file("garbage_collection", &datastore.name)?;
jobstate::create_state_file("prune", &datastore.name)?;
jobstate::create_state_file("garbage_collection", &datastore.name)?;
Ok(())
}
@ -328,11 +329,11 @@ pub fn update_datastore(
// we want to reset the statefiles, to avoid an immediate action in some cases
// (e.g. going from monthly to weekly in the second week of the month)
if gc_schedule_changed {
crate::config::jobstate::create_state_file("garbage_collection", &name)?;
jobstate::create_state_file("garbage_collection", &name)?;
}
if prune_schedule_changed {
crate::config::jobstate::create_state_file("prune", &name)?;
jobstate::create_state_file("prune", &name)?;
}
Ok(())
@ -375,8 +376,8 @@ pub fn delete_datastore(name: String, digest: Option<String>) -> Result<(), Erro
datastore::save_config(&config)?;
// ignore errors
let _ = crate::config::jobstate::remove_state_file("prune", &name);
let _ = crate::config::jobstate::remove_state_file("garbage_collection", &name);
let _ = jobstate::remove_state_file("prune", &name);
let _ = jobstate::remove_state_file("garbage_collection", &name);
Ok(())
}

4
src/api2/config/sync.rs
View File

@ -83,7 +83,7 @@ pub fn create_sync_job(param: Value) -> Result<(), Error> {
sync::save_config(&config)?;
crate::config::jobstate::create_state_file("syncjob", &sync_job.id)?;
crate::server::jobstate::create_state_file("syncjob", &sync_job.id)?;
Ok(())
}
@ -266,7 +266,7 @@ pub fn delete_sync_job(id: String, digest: Option<String>) -> Result<(), Error>
sync::save_config(&config)?;
crate::config::jobstate::remove_state_file("syncjob", &id)?;
crate::server::jobstate::remove_state_file("syncjob", &id)?;
Ok(())
}

6
src/api2/config/verify.rs
View File

@ -80,7 +80,7 @@ pub fn create_verification_job(param: Value) -> Result<(), Error> {
verify::save_config(&config)?;
crate::config::jobstate::create_state_file("verificationjob", &verification_job.id)?;
crate::server::jobstate::create_state_file("verificationjob", &verification_job.id)?;
Ok(())
}
@ -258,7 +258,7 @@ pub fn delete_verification_job(id: String, digest: Option<String>) -> Result<(),
verify::save_config(&config)?;
crate::config::jobstate::remove_state_file("verificationjob", &id)?;
crate::server::jobstate::remove_state_file("verificationjob", &id)?;
Ok(())
}
@ -271,4 +271,4 @@ const ITEM_ROUTER: Router = Router::new()
pub const ROUTER: Router = Router::new()
.get(&API_METHOD_LIST_VERIFICATION_JOBS)
.post(&API_METHOD_CREATE_VERIFICATION_JOB)
.match_all("id", &ITEM_ROUTER);
.match_all("id", &ITEM_ROUTER);

3
src/api2/pull.rs
View File

@ -7,14 +7,13 @@ use futures::{select, future::FutureExt};
use proxmox::api::api;
use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
use crate::server::{WorkerTask};
use crate::server::{WorkerTask, jobstate::Job};
use crate::backup::DataStore;
use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pull::pull_store};
use crate::api2::types::*;
use crate::config::{
remote,
sync::SyncJobConfig,
jobstate::Job,
acl::{PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ},
cached_user_info::CachedUserInfo,
};

96
src/backup/verify.rs
View File

@ -7,10 +7,7 @@ use nix::dir::Dir;
use anyhow::{bail, format_err, Error};
use crate::{
server::WorkerTask,
api2::types::*,
config::jobstate::Job,
config::verify::VerificationJobConfig,
backup::{
DataStore,
DataBlob,
@ -529,96 +526,3 @@ pub fn verify_all_backups(
Ok(errors)
}
/// Runs a verification job.
pub fn do_verification_job(
mut job: Job,
verification_job: VerificationJobConfig,
userid: &Userid,
schedule: Option<String>,
) -> Result<String, Error> {
let datastore = DataStore::lookup_datastore(&verification_job.store)?;
let mut backups_to_verify = BackupInfo::list_backups(&datastore.base_path())?;
if verification_job.ignore_verified.unwrap_or(true) {
backups_to_verify.retain(|backup_info| {
let manifest = match datastore.load_manifest(&backup_info.backup_dir) {
Ok((manifest, _)) => manifest,
Err(_) => return false,
};
let raw_verify_state = manifest.unprotected["verify_state"].clone();
let last_state = match serde_json::from_value::<SnapshotVerifyState>(raw_verify_state) {
Ok(last_state) => last_state,
Err(_) => return true,
};
let now = proxmox::tools::time::epoch_i64();
let days_since_last_verify = (now - last_state.upid.starttime) / 86400;
verification_job.outdated_after.is_some()
&& days_since_last_verify > verification_job.outdated_after.unwrap()
})
}
let job_id = job.jobname().to_string();
let worker_type = job.jobtype().to_string();
let upid_str = WorkerTask::new_thread(
&worker_type,
Some(job.jobname().to_string()),
userid.clone(),
false,
move |worker| {
job.start(&worker.upid().to_string())?;
task_log!(worker,"Starting datastore verify job '{}'", job_id);
task_log!(worker,"verifying {} backups", backups_to_verify.len());
if let Some(event_str) = schedule {
task_log!(worker,"task triggered by schedule '{}'", event_str);
}
let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024 * 16)));
let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
let result = proxmox::try_block!({
let mut failed_dirs: Vec<String> = Vec::new();
for backup_info in backups_to_verify {
let verification_result = verify_backup_dir(
datastore.clone(),
&backup_info.backup_dir,
verified_chunks.clone(),
corrupt_chunks.clone(),
worker.clone(),
worker.upid().clone()
);
if let Ok(false) = verification_result {
failed_dirs.push(backup_info.backup_dir.to_string());
} // otherwise successful or aborted
}
if !failed_dirs.is_empty() {
task_log!(worker,"Failed to verify following snapshots:",);
for dir in failed_dirs {
task_log!(worker, "\t{}", dir)
}
bail!("verification failed - please check the log for details");
}
Ok(())
});
let status = worker.create_state(&result);
match job.finish(status) {
Err(err) => eprintln!(
"could not finish job state for {}: {}",
job.jobtype().to_string(),
err
),
Ok(_) => (),
}
result
},
)?;
Ok(upid_str)
}

2
src/bin/proxmox-backup-api.rs
View File

@ -37,7 +37,7 @@ async fn run() -> Result<(), Error> {
config::update_self_signed_cert(false)?;
proxmox_backup::rrd::create_rrdb_dir()?;
proxmox_backup::config::jobstate::create_jobstate_dir()?;
proxmox_backup::server::jobstate::create_jobstate_dir()?;
if let Err(err) = generate_auth_key() {
bail!("unable to generate auth key - {}", err);

72
src/bin/proxmox-backup-proxy.rs
View File

@ -10,11 +10,30 @@ use openssl::ssl::{SslMethod, SslAcceptor, SslFiletype};
use proxmox::try_block;
use proxmox::api::RpcEnvironmentType;
use proxmox_backup::{
backup::DataStore,
server::{
UPID,
WorkerTask,
ApiConfig,
rest::*,
jobstate::{
self,
Job,
},
rotate_task_log_archive,
},
tools::systemd::time::{
parse_calendar_event,
compute_next_event,
},
};
use proxmox_backup::api2::types::Userid;
use proxmox_backup::configdir;
use proxmox_backup::buildcfg;
use proxmox_backup::server;
use proxmox_backup::server::{ApiConfig, rest::*};
use proxmox_backup::auth_helpers::*;
use proxmox_backup::tools::{
daemon,
@ -29,7 +48,7 @@ use proxmox_backup::tools::{
};
use proxmox_backup::api2::pull::do_sync_job;
use proxmox_backup::backup::do_verification_job;
use proxmox_backup::server::do_verification_job;
fn main() -> Result<(), Error> {
proxmox_backup::tools::setup_safe_path_env();
@ -221,14 +240,10 @@ async fn schedule_tasks() -> Result<(), Error> {
async fn schedule_datastore_garbage_collection() {
use proxmox_backup::backup::DataStore;
use proxmox_backup::server::{UPID, WorkerTask};
use proxmox_backup::config::{
jobstate::{self, Job},
datastore::{self, DataStoreConfig}
use proxmox_backup::config::datastore::{
self,
DataStoreConfig,
};
use proxmox_backup::tools::systemd::time::{
parse_calendar_event, compute_next_event};
let config = match datastore::config() {
Err(err) => {
@ -340,15 +355,17 @@ async fn schedule_datastore_garbage_collection() {
async fn schedule_datastore_prune() {
use proxmox_backup::backup::{
PruneOptions, DataStore, BackupGroup, compute_prune_info};
use proxmox_backup::server::{WorkerTask};
use proxmox_backup::config::{
jobstate::{self, Job},
datastore::{self, DataStoreConfig}
use proxmox_backup::{
backup::{
PruneOptions,
BackupGroup,
compute_prune_info,
},
config::datastore::{
self,
DataStoreConfig,
},
};
use proxmox_backup::tools::systemd::time::{
parse_calendar_event, compute_next_event};
let config = match datastore::config() {
Err(err) => {
@ -487,9 +504,9 @@ async fn schedule_datastore_prune() {
async fn schedule_datastore_sync_jobs() {
use proxmox_backup::{
config::{ sync::{self, SyncJobConfig}, jobstate::{self, Job} },
tools::systemd::time::{ parse_calendar_event, compute_next_event },
use proxmox_backup::config::sync::{
self,
SyncJobConfig,
};
let config = match sync::config() {
@ -559,10 +576,12 @@ async fn schedule_datastore_sync_jobs() {
}
async fn schedule_datastore_verify_jobs() {
use proxmox_backup::{
config::{verify::{self, VerificationJobConfig}, jobstate::{self, Job}},
tools::systemd::time::{parse_calendar_event, compute_next_event},
use proxmox_backup::config::verify::{
self,
VerificationJobConfig,
};
let config = match verify::config() {
Err(err) => {
eprintln!("unable to read verification job config - {}", err);
@ -619,13 +638,6 @@ async fn schedule_datastore_verify_jobs() {
}
async fn schedule_task_log_rotate() {
use proxmox_backup::{
config::jobstate::{self, Job},
server::rotate_task_log_archive,
};
use proxmox_backup::server::WorkerTask;
use proxmox_backup::tools::systemd::time::{
parse_calendar_event, compute_next_event};
let worker_type = "logrotate";
let job_id = "task_archive";

1
src/config.rs
View File

@ -18,7 +18,6 @@ use crate::buildcfg;
pub mod acl;
pub mod cached_user_info;
pub mod datastore;
pub mod jobstate;
pub mod network;
pub mod remote;
pub mod sync;

4
src/server.rs
View File

@ -30,3 +30,7 @@ pub mod formatter;
#[macro_use]
pub mod rest;
pub mod jobstate;
mod verify_job;
pub use verify_job::*;

0
src/config/jobstate.rs → src/server/jobstate.rs
View File