tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

manager: hidden command to move datastore prune opts into jobs

Browse Source 
Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Wolfgang Bumiller 2022-05-24 12:54:42 +02:00
parent 9ce2f903fb
commit 134779664e
8 changed files with 142 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
pbs-api-types/src/datastore.rs
View File

@ -218,29 +218,8 @@ pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore
optional: true,
schema: PRUNE_SCHEDULE_SCHEMA,
},
"keep-last": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_LAST,
},
"keep-hourly": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_HOURLY,
},
"keep-daily": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_DAILY,
},
"keep-weekly": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_WEEKLY,
},
"keep-monthly": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_MONTHLY,
},
"keep-yearly": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_YEARLY,
keep: {
type: crate::KeepOptions,
},
"verify-new": {
description: "If enabled, all new backups will be verified right after completion.",
@ -277,18 +256,8 @@ pub struct DataStoreConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub prune_schedule: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub keep_last: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub keep_hourly: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub keep_daily: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub keep_weekly: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub keep_monthly: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub keep_yearly: Option<u64>,
#[serde(flatten)]
pub keep: crate::KeepOptions,
/// If enabled, all backups will be verified right after completion.
#[serde(skip_serializing_if = "Option::is_none")]
@ -319,12 +288,7 @@ impl DataStoreConfig {
comment: None,
gc_schedule: None,
prune_schedule: None,
keep_last: None,
keep_hourly: None,
keep_daily: None,
keep_weekly: None,
keep_monthly: None,
keep_yearly: None,
keep: Default::default(),
verify_new: None,
notify_user: None,
notify: None,

9
pbs-config/src/datastore.rs
View File

@ -2,7 +2,7 @@ use anyhow::Error;
use lazy_static::lazy_static;
use std::collections::HashMap;
use proxmox_schema::{ApiType, Schema};
use proxmox_schema::{AllOfSchema, ApiType, Schema};
use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
use pbs_api_types::{DataStoreConfig, DATASTORE_SCHEMA};
@ -14,15 +14,12 @@ lazy_static! {
}
fn init() -> SectionConfig {
let obj_schema = match DataStoreConfig::API_SCHEMA {
Schema::Object(ref obj_schema) => obj_schema,
_ => unreachable!(),
};
const OBJ_SCHEMA: &AllOfSchema = DataStoreConfig::API_SCHEMA.unwrap_all_of_schema();
let plugin = SectionConfigPlugin::new(
"datastore".to_string(),
Some(String::from("name")),
obj_schema,
OBJ_SCHEMA,
);
let mut config = SectionConfig::new(&DATASTORE_SCHEMA);
config.register_plugin(plugin);

1
pbs-config/src/prune.rs
View File

@ -39,6 +39,7 @@ pub fn config() -> Result<(SectionConfigData, [u8; 32]), Error> {
let digest = openssl::sha::sha256(content.as_bytes());
let data = CONFIG.parse(PRUNE_CFG_FILENAME, &content)?;
Ok((data, digest))
}

36
src/api2/config/datastore.rs
View File

@ -251,22 +251,22 @@ pub fn update_datastore(
data.prune_schedule = None;
}
DeletableProperty::keep_last => {
data.keep_last = None;
data.keep.keep_last = None;
}
DeletableProperty::keep_hourly => {
data.keep_hourly = None;
data.keep.keep_hourly = None;
}
DeletableProperty::keep_daily => {
data.keep_daily = None;
data.keep.keep_daily = None;
}
DeletableProperty::keep_weekly => {
data.keep_weekly = None;
data.keep.keep_weekly = None;
}
DeletableProperty::keep_monthly => {
data.keep_monthly = None;
data.keep.keep_monthly = None;
}
DeletableProperty::keep_yearly => {
data.keep_yearly = None;
data.keep.keep_yearly = None;
}
DeletableProperty::verify_new => {
data.verify_new = None;
@ -308,23 +308,23 @@ pub fn update_datastore(
data.prune_schedule = update.prune_schedule;
}
if update.keep_last.is_some() {
data.keep_last = update.keep_last;
if update.keep.keep_last.is_some() {
data.keep.keep_last = update.keep.keep_last;
}
if update.keep_hourly.is_some() {
data.keep_hourly = update.keep_hourly;
if update.keep.keep_hourly.is_some() {
data.keep.keep_hourly = update.keep.keep_hourly;
}
if update.keep_daily.is_some() {
data.keep_daily = update.keep_daily;
if update.keep.keep_daily.is_some() {
data.keep.keep_daily = update.keep.keep_daily;
}
if update.keep_weekly.is_some() {
data.keep_weekly = update.keep_weekly;
if update.keep.keep_weekly.is_some() {
data.keep.keep_weekly = update.keep.keep_weekly;
}
if update.keep_monthly.is_some() {
data.keep_monthly = update.keep_monthly;
if update.keep.keep_monthly.is_some() {
data.keep.keep_monthly = update.keep.keep_monthly;
}
if update.keep_yearly.is_some() {
data.keep_yearly = update.keep_yearly;
if update.keep.keep_yearly.is_some() {
data.keep.keep_yearly = update.keep.keep_yearly;
}
if let Some(notify_str) = update.notify {

4
src/bin/proxmox-backup-manager.rs
View File

@ -453,6 +453,9 @@ async fn run() -> Result<(), Error> {
.insert("versions", CliCommand::new(&API_METHOD_GET_VERSIONS));
let args: Vec<String> = std::env::args().take(2).collect();
if args.len() >= 2 && args[1] == "update-to-prune-jobs-config" {
return update_to_prune_jobs_config();
}
let avoid_init = args.len() >= 2 && (args[1] == "bashcomplete" || args[1] == "printdoc");
if !avoid_init {
@ -460,6 +463,7 @@ async fn run() -> Result<(), Error> {
let file_opts = CreateOptions::new()
.owner(backup_user.uid)
.group(backup_user.gid);
proxmox_rest_server::init_worker_tasks(
pbs_buildcfg::PROXMOX_BACKUP_LOG_DIR_M!().into(),
file_opts,

13
src/bin/proxmox-backup-proxy.rs
View File

@ -47,8 +47,8 @@ use pbs_buildcfg::configdir;
use proxmox_time::CalendarEvent;
use pbs_api_types::{
Authid, DataStoreConfig, KeepOptions, Operation, PruneJobConfig, PruneJobOptions,
SyncJobConfig, TapeBackupJobConfig, VerificationJobConfig,
Authid, DataStoreConfig, Operation, PruneJobConfig, PruneJobOptions, SyncJobConfig,
TapeBackupJobConfig, VerificationJobConfig,
};
use proxmox_rest_server::daemon;
@ -692,14 +692,7 @@ async fn schedule_datastore_prune() {
};
let prune_options = PruneJobOptions {
keep: KeepOptions {
keep_last: store_config.keep_last,
keep_hourly: store_config.keep_hourly,
keep_daily: store_config.keep_daily,
keep_weekly: store_config.keep_weekly,
keep_monthly: store_config.keep_monthly,
keep_yearly: store_config.keep_yearly,
},
keep: store_config.keep,
..Default::default()
};

77
src/bin/proxmox_backup_manager/prune.rs
View File

@ -1,12 +1,13 @@
use std::collections::HashMap;
use anyhow::Error;
use serde::Deserialize;
use serde_json::Value;
use proxmox_router::{cli::*, ApiHandler, RpcEnvironment};
use proxmox_schema::api;
use pbs_api_types::{PruneJobConfig, JOB_ID_SCHEMA};
use pbs_api_types::{DataStoreConfig, PruneJobConfig, PruneJobOptions, JOB_ID_SCHEMA};
use pbs_config::prune;
use proxmox_backup::api2;
@ -155,3 +156,77 @@ fn get_prune_job(id: &str) -> Result<PruneJobConfig, Error> {
config.lookup("prune", id)
}
pub(crate) fn update_to_prune_jobs_config() -> Result<(), Error> {
use pbs_config::datastore;
let _prune_lock = prune::lock_config()?;
let _datastore_lock = datastore::lock_config()?;
let (mut data, _digest) = prune::config()?;
let (mut storeconfig, _digest) = datastore::config()?;
for (store, entry) in storeconfig.sections.iter_mut() {
let ty = &entry.0;
if ty != "datastore" {
continue;
}
let mut config = match DataStoreConfig::deserialize(&entry.1) {
Ok(c) => c,
Err(err) => {
eprintln!("failed to parse config of store {store}: {err}");
continue;
}
};
let options = PruneJobOptions {
keep: std::mem::take(&mut config.keep),
..Default::default()
};
let schedule = config.prune_schedule.take();
entry.1 = serde_json::to_value(config)?;
let schedule = match schedule {
Some(s) => s,
None => {
eprintln!("dropping disabled prune job in datastore.cfg");
continue;
}
};
let mut id = format!("storeconfig-{store}");
id.truncate(32);
if data.sections.contains_key(&id) {
eprintln!("skipping existing converted prune job: {id}");
continue;
}
if !options.keeps_something() {
eprintln!("dropping empty prune job data in datastore.cfg");
continue;
}
let prune_config = PruneJobConfig {
id: id.clone(),
store: store.clone(),
disable: false,
comment: None,
schedule,
options,
};
let prune_config = serde_json::to_value(prune_config)?;
data.sections
.insert(id, ("prune".to_string(), prune_config));
}
prune::save_config(&data)?;
datastore::save_config(&storeconfig)?;
Ok(())
}

64
tests/prune.rs
View File

@ -12,7 +12,7 @@ fn get_prune_list(
return_kept: bool,
options: &PruneJobOptions,
) -> Vec<PathBuf> {
let mut prune_info = compute_prune_info(list, options).unwrap();
let mut prune_info = compute_prune_info(list, &options.keep).unwrap();
prune_info.reverse();
@ -68,13 +68,13 @@ fn test_prune_protected() -> Result<(), Error> {
eprintln!("{:?}", orig_list);
let mut options = PruneJobOptions::default();
options.keep_last = Some(1);
options.keep.keep_last = Some(1);
let remove_list = get_prune_list(orig_list.clone(), false, &options);
let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-11-15T10:39:15Z")];
assert_eq!(remove_list, expect);
let mut options = PruneJobOptions::default();
options.keep_hourly = Some(1);
options.keep.keep_hourly = Some(1);
let remove_list = get_prune_list(orig_list.clone(), false, &options);
let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-11-15T10:39:15Z")];
assert_eq!(remove_list, expect);
@ -94,7 +94,7 @@ fn test_prune_hourly() -> Result<(), Error> {
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_hourly = Some(3);
options.keep.keep_hourly = Some(3);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-11-15T10:49:15Z"),
@ -105,7 +105,7 @@ fn test_prune_hourly() -> Result<(), Error> {
let list = orig_list;
let mut options = PruneJobOptions::default();
options.keep_hourly = Some(2);
options.keep.keep_hourly = Some(2);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-11-15T10:59:15Z"),
@ -132,15 +132,15 @@ fn test_prune_simple2() -> Result<(), Error> {
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_daily = Some(1);
options.keep.keep_daily = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")];
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_last = Some(1);
options.keep_daily = Some(1);
options.keep.keep_last = Some(1);
options.keep.keep_daily = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-03T11:59:15Z"),
@ -150,8 +150,8 @@ fn test_prune_simple2() -> Result<(), Error> {
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_daily = Some(1);
options.keep_weekly = Some(1);
options.keep.keep_daily = Some(1);
options.keep.keep_weekly = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-01T11:59:15Z"),
@ -161,9 +161,9 @@ fn test_prune_simple2() -> Result<(), Error> {
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_daily = Some(1);
options.keep_weekly = Some(1);
options.keep_monthly = Some(1);
options.keep.keep_daily = Some(1);
options.keep.keep_weekly = Some(1);
options.keep.keep_monthly = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-11-22T11:59:15Z"),
@ -174,8 +174,8 @@ fn test_prune_simple2() -> Result<(), Error> {
let list = orig_list;
let mut options = PruneJobOptions::default();
options.keep_monthly = Some(1);
options.keep_yearly = Some(1);
options.keep.keep_monthly = Some(1);
options.keep.keep_yearly = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2018-11-15T11:59:15Z"),
@ -199,21 +199,21 @@ fn test_prune_simple() -> Result<(), Error> {
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_last = Some(4);
options.keep.keep_last = Some(4);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = Vec::new();
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_last = Some(3);
options.keep.keep_last = Some(3);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-12-02T11:59:15Z")];
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_last = Some(2);
options.keep.keep_last = Some(2);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -223,7 +223,7 @@ fn test_prune_simple() -> Result<(), Error> {
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_last = Some(1);
options.keep.keep_last = Some(1);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -234,7 +234,7 @@ fn test_prune_simple() -> Result<(), Error> {
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_last = Some(0);
options.keep.keep_last = Some(0);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -247,8 +247,8 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-last, keep-daily mixed
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_last = Some(2);
options.keep_daily = Some(2);
options.keep.keep_last = Some(2);
options.keep.keep_daily = Some(2);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![];
assert_eq!(remove_list, expect);
@ -256,7 +256,7 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-daily test
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_daily = Some(3);
options.keep.keep_daily = Some(3);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")];
assert_eq!(remove_list, expect);
@ -264,7 +264,7 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-daily test
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_daily = Some(2);
options.keep.keep_daily = Some(2);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -275,7 +275,7 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-weekly
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_weekly = Some(5);
options.keep.keep_weekly = Some(5);
let remove_list = get_prune_list(list, false, &options);
// all backup are within the same week, so we only keep a single file
let expect: Vec<PathBuf> = vec![
@ -288,8 +288,8 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-daily + keep-weekly
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_daily = Some(1);
options.keep_weekly = Some(5);
options.keep.keep_daily = Some(1);
options.keep.keep_weekly = Some(5);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -301,7 +301,7 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-monthly
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_monthly = Some(6);
options.keep.keep_monthly = Some(6);
let remove_list = get_prune_list(list, false, &options);
// all backup are within the same month, so we only keep a single file
let expect: Vec<PathBuf> = vec![
@ -314,7 +314,7 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-yearly
let list = orig_list.clone();
let mut options = PruneJobOptions::default();
options.keep_yearly = Some(7);
options.keep.keep_yearly = Some(7);
let remove_list = get_prune_list(list, false, &options);
// all backup are within the same year, so we only keep a single file
let expect: Vec<PathBuf> = vec![
@ -327,9 +327,9 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-weekly + keep-monthly + keep-yearly
let list = orig_list;
let mut options = PruneJobOptions::default();
options.keep_weekly = Some(5);
options.keep_monthly = Some(6);
options.keep_yearly = Some(7);
options.keep.keep_weekly = Some(5);
options.keep.keep_monthly = Some(6);
options.keep.keep_yearly = Some(7);
let remove_list = get_prune_list(list, false, &options);
// all backup are within one week, so we only keep a single file
let expect: Vec<PathBuf> = vec![