manager: hidden command to move datastore prune opts into jobs

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>

pbs-api-types/src/datastore.rs

@@ -218,29 +218,8 @@ pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore
             optional: true,
             schema: PRUNE_SCHEDULE_SCHEMA,
         },
-        "keep-last": {
+        keep: {
-            optional: true,
+            type: crate::KeepOptions,
-            schema: PRUNE_SCHEMA_KEEP_LAST,
-        },
-        "keep-hourly": {
-            optional: true,
-            schema: PRUNE_SCHEMA_KEEP_HOURLY,
-        },
-        "keep-daily": {
-            optional: true,
-            schema: PRUNE_SCHEMA_KEEP_DAILY,
-        },
-        "keep-weekly": {
-            optional: true,
-            schema: PRUNE_SCHEMA_KEEP_WEEKLY,
-        },
-        "keep-monthly": {
-            optional: true,
-            schema: PRUNE_SCHEMA_KEEP_MONTHLY,
-        },
-        "keep-yearly": {
-            optional: true,
-            schema: PRUNE_SCHEMA_KEEP_YEARLY,
         },
         "verify-new": {
             description: "If enabled, all new backups will be verified right after completion.",
@@ -277,18 +256,8 @@ pub struct DataStoreConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub prune_schedule: Option<String>,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(flatten)]
-    pub keep_last: Option<u64>,
+    pub keep: crate::KeepOptions,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub keep_hourly: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub keep_daily: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub keep_weekly: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub keep_monthly: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub keep_yearly: Option<u64>,
 
     /// If enabled, all backups will be verified right after completion.
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -319,12 +288,7 @@ impl DataStoreConfig {
             comment: None,
             gc_schedule: None,
             prune_schedule: None,
-            keep_last: None,
+            keep: Default::default(),
-            keep_hourly: None,
-            keep_daily: None,
-            keep_weekly: None,
-            keep_monthly: None,
-            keep_yearly: None,
             verify_new: None,
             notify_user: None,
             notify: None,

pbs-config/src/datastore.rs

@@ -2,7 +2,7 @@ use anyhow::Error;
 use lazy_static::lazy_static;
 use std::collections::HashMap;
 
-use proxmox_schema::{ApiType, Schema};
+use proxmox_schema::{AllOfSchema, ApiType, Schema};
 use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
 
 use pbs_api_types::{DataStoreConfig, DATASTORE_SCHEMA};
@@ -14,15 +14,12 @@ lazy_static! {
 }
 
 fn init() -> SectionConfig {
-    let obj_schema = match DataStoreConfig::API_SCHEMA {
+    const OBJ_SCHEMA: &AllOfSchema = DataStoreConfig::API_SCHEMA.unwrap_all_of_schema();
-        Schema::Object(ref obj_schema) => obj_schema,
-        _ => unreachable!(),
-    };
 
     let plugin = SectionConfigPlugin::new(
         "datastore".to_string(),
         Some(String::from("name")),
-        obj_schema,
+        OBJ_SCHEMA,
     );
     let mut config = SectionConfig::new(&DATASTORE_SCHEMA);
     config.register_plugin(plugin);

pbs-config/src/prune.rs

@@ -39,6 +39,7 @@ pub fn config() -> Result<(SectionConfigData, [u8; 32]), Error> {
 
     let digest = openssl::sha::sha256(content.as_bytes());
     let data = CONFIG.parse(PRUNE_CFG_FILENAME, &content)?;
+
     Ok((data, digest))
 }
 

src/api2/config/datastore.rs

@@ -251,22 +251,22 @@ pub fn update_datastore(
                     data.prune_schedule = None;
                 }
                 DeletableProperty::keep_last => {
-                    data.keep_last = None;
+                    data.keep.keep_last = None;
                 }
                 DeletableProperty::keep_hourly => {
-                    data.keep_hourly = None;
+                    data.keep.keep_hourly = None;
                 }
                 DeletableProperty::keep_daily => {
-                    data.keep_daily = None;
+                    data.keep.keep_daily = None;
                 }
                 DeletableProperty::keep_weekly => {
-                    data.keep_weekly = None;
+                    data.keep.keep_weekly = None;
                 }
                 DeletableProperty::keep_monthly => {
-                    data.keep_monthly = None;
+                    data.keep.keep_monthly = None;
                 }
                 DeletableProperty::keep_yearly => {
-                    data.keep_yearly = None;
+                    data.keep.keep_yearly = None;
                 }
                 DeletableProperty::verify_new => {
                     data.verify_new = None;
@@ -308,23 +308,23 @@ pub fn update_datastore(
         data.prune_schedule = update.prune_schedule;
     }
 
-    if update.keep_last.is_some() {
+    if update.keep.keep_last.is_some() {
-        data.keep_last = update.keep_last;
+        data.keep.keep_last = update.keep.keep_last;
     }
-    if update.keep_hourly.is_some() {
+    if update.keep.keep_hourly.is_some() {
-        data.keep_hourly = update.keep_hourly;
+        data.keep.keep_hourly = update.keep.keep_hourly;
     }
-    if update.keep_daily.is_some() {
+    if update.keep.keep_daily.is_some() {
-        data.keep_daily = update.keep_daily;
+        data.keep.keep_daily = update.keep.keep_daily;
     }
-    if update.keep_weekly.is_some() {
+    if update.keep.keep_weekly.is_some() {
-        data.keep_weekly = update.keep_weekly;
+        data.keep.keep_weekly = update.keep.keep_weekly;
     }
-    if update.keep_monthly.is_some() {
+    if update.keep.keep_monthly.is_some() {
-        data.keep_monthly = update.keep_monthly;
+        data.keep.keep_monthly = update.keep.keep_monthly;
     }
-    if update.keep_yearly.is_some() {
+    if update.keep.keep_yearly.is_some() {
-        data.keep_yearly = update.keep_yearly;
+        data.keep.keep_yearly = update.keep.keep_yearly;
     }
 
     if let Some(notify_str) = update.notify {

src/bin/proxmox-backup-manager.rs

@@ -453,6 +453,9 @@ async fn run() -> Result<(), Error> {
         .insert("versions", CliCommand::new(&API_METHOD_GET_VERSIONS));
 
     let args: Vec<String> = std::env::args().take(2).collect();
+    if args.len() >= 2 && args[1] == "update-to-prune-jobs-config" {
+        return update_to_prune_jobs_config();
+    }
     let avoid_init = args.len() >= 2 && (args[1] == "bashcomplete" || args[1] == "printdoc");
 
     if !avoid_init {
@@ -460,6 +463,7 @@ async fn run() -> Result<(), Error> {
         let file_opts = CreateOptions::new()
             .owner(backup_user.uid)
             .group(backup_user.gid);
+
         proxmox_rest_server::init_worker_tasks(
             pbs_buildcfg::PROXMOX_BACKUP_LOG_DIR_M!().into(),
             file_opts,

src/bin/proxmox-backup-proxy.rs

@@ -47,8 +47,8 @@ use pbs_buildcfg::configdir;
 use proxmox_time::CalendarEvent;
 
 use pbs_api_types::{
-    Authid, DataStoreConfig, KeepOptions, Operation, PruneJobConfig, PruneJobOptions,
+    Authid, DataStoreConfig, Operation, PruneJobConfig, PruneJobOptions, SyncJobConfig,
-    SyncJobConfig, TapeBackupJobConfig, VerificationJobConfig,
+    TapeBackupJobConfig, VerificationJobConfig,
 };
 
 use proxmox_rest_server::daemon;
@@ -692,14 +692,7 @@ async fn schedule_datastore_prune() {
         };
 
         let prune_options = PruneJobOptions {
-            keep: KeepOptions {
+            keep: store_config.keep,
-                keep_last: store_config.keep_last,
-                keep_hourly: store_config.keep_hourly,
-                keep_daily: store_config.keep_daily,
-                keep_weekly: store_config.keep_weekly,
-                keep_monthly: store_config.keep_monthly,
-                keep_yearly: store_config.keep_yearly,
-            },
             ..Default::default()
         };
 

src/bin/proxmox_backup_manager/prune.rs

@@ -1,12 +1,13 @@
 use std::collections::HashMap;
 
 use anyhow::Error;
+use serde::Deserialize;
 use serde_json::Value;
 
 use proxmox_router::{cli::*, ApiHandler, RpcEnvironment};
 use proxmox_schema::api;
 
-use pbs_api_types::{PruneJobConfig, JOB_ID_SCHEMA};
+use pbs_api_types::{DataStoreConfig, PruneJobConfig, PruneJobOptions, JOB_ID_SCHEMA};
 use pbs_config::prune;
 
 use proxmox_backup::api2;
@@ -155,3 +156,77 @@ fn get_prune_job(id: &str) -> Result<PruneJobConfig, Error> {
 
     config.lookup("prune", id)
 }
+
+pub(crate) fn update_to_prune_jobs_config() -> Result<(), Error> {
+    use pbs_config::datastore;
+
+    let _prune_lock = prune::lock_config()?;
+    let _datastore_lock = datastore::lock_config()?;
+
+    let (mut data, _digest) = prune::config()?;
+    let (mut storeconfig, _digest) = datastore::config()?;
+
+    for (store, entry) in storeconfig.sections.iter_mut() {
+        let ty = &entry.0;
+
+        if ty != "datastore" {
+            continue;
+        }
+
+        let mut config = match DataStoreConfig::deserialize(&entry.1) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("failed to parse config of store {store}: {err}");
+                continue;
+            }
+        };
+
+        let options = PruneJobOptions {
+            keep: std::mem::take(&mut config.keep),
+            ..Default::default()
+        };
+
+        let schedule = config.prune_schedule.take();
+
+        entry.1 = serde_json::to_value(config)?;
+
+        let schedule = match schedule {
+            Some(s) => s,
+            None => {
+                eprintln!("dropping disabled prune job in datastore.cfg");
+                continue;
+            }
+        };
+
+        let mut id = format!("storeconfig-{store}");
+        id.truncate(32);
+        if data.sections.contains_key(&id) {
+            eprintln!("skipping existing converted prune job: {id}");
+            continue;
+        }
+
+        if !options.keeps_something() {
+            eprintln!("dropping empty prune job data in datastore.cfg");
+            continue;
+        }
+
+        let prune_config = PruneJobConfig {
+            id: id.clone(),
+            store: store.clone(),
+            disable: false,
+            comment: None,
+            schedule,
+            options,
+        };
+
+        let prune_config = serde_json::to_value(prune_config)?;
+
+        data.sections
+            .insert(id, ("prune".to_string(), prune_config));
+    }
+
+    prune::save_config(&data)?;
+    datastore::save_config(&storeconfig)?;
+
+    Ok(())
+}

tests/prune.rs

@@ -12,7 +12,7 @@ fn get_prune_list(
     return_kept: bool,
     options: &PruneJobOptions,
 ) -> Vec<PathBuf> {
-    let mut prune_info = compute_prune_info(list, options).unwrap();
+    let mut prune_info = compute_prune_info(list, &options.keep).unwrap();
 
     prune_info.reverse();
 
@@ -68,13 +68,13 @@ fn test_prune_protected() -> Result<(), Error> {
     eprintln!("{:?}", orig_list);
 
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(1);
+    options.keep.keep_last = Some(1);
     let remove_list = get_prune_list(orig_list.clone(), false, &options);
     let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-11-15T10:39:15Z")];
     assert_eq!(remove_list, expect);
 
     let mut options = PruneJobOptions::default();
-    options.keep_hourly = Some(1);
+    options.keep.keep_hourly = Some(1);
     let remove_list = get_prune_list(orig_list.clone(), false, &options);
     let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-11-15T10:39:15Z")];
     assert_eq!(remove_list, expect);
@@ -94,7 +94,7 @@ fn test_prune_hourly() -> Result<(), Error> {
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_hourly = Some(3);
+    options.keep.keep_hourly = Some(3);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-11-15T10:49:15Z"),
@@ -105,7 +105,7 @@ fn test_prune_hourly() -> Result<(), Error> {
 
     let list = orig_list;
     let mut options = PruneJobOptions::default();
-    options.keep_hourly = Some(2);
+    options.keep.keep_hourly = Some(2);
     let remove_list = get_prune_list(list, true, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-11-15T10:59:15Z"),
@@ -132,15 +132,15 @@ fn test_prune_simple2() -> Result<(), Error> {
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_daily = Some(1);
+    options.keep.keep_daily = Some(1);
     let remove_list = get_prune_list(list, true, &options);
     let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")];
     assert_eq!(remove_list, expect);
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(1);
+    options.keep.keep_last = Some(1);
-    options.keep_daily = Some(1);
+    options.keep.keep_daily = Some(1);
     let remove_list = get_prune_list(list, true, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-12-03T11:59:15Z"),
@@ -150,8 +150,8 @@ fn test_prune_simple2() -> Result<(), Error> {
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_daily = Some(1);
+    options.keep.keep_daily = Some(1);
-    options.keep_weekly = Some(1);
+    options.keep.keep_weekly = Some(1);
     let remove_list = get_prune_list(list, true, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-12-01T11:59:15Z"),
@@ -161,9 +161,9 @@ fn test_prune_simple2() -> Result<(), Error> {
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_daily = Some(1);
+    options.keep.keep_daily = Some(1);
-    options.keep_weekly = Some(1);
+    options.keep.keep_weekly = Some(1);
-    options.keep_monthly = Some(1);
+    options.keep.keep_monthly = Some(1);
     let remove_list = get_prune_list(list, true, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-11-22T11:59:15Z"),
@@ -174,8 +174,8 @@ fn test_prune_simple2() -> Result<(), Error> {
 
     let list = orig_list;
     let mut options = PruneJobOptions::default();
-    options.keep_monthly = Some(1);
+    options.keep.keep_monthly = Some(1);
-    options.keep_yearly = Some(1);
+    options.keep.keep_yearly = Some(1);
     let remove_list = get_prune_list(list, true, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2018-11-15T11:59:15Z"),
@@ -199,21 +199,21 @@ fn test_prune_simple() -> Result<(), Error> {
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(4);
+    options.keep.keep_last = Some(4);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = Vec::new();
     assert_eq!(remove_list, expect);
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(3);
+    options.keep.keep_last = Some(3);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-12-02T11:59:15Z")];
     assert_eq!(remove_list, expect);
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(2);
+    options.keep.keep_last = Some(2);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@@ -223,7 +223,7 @@ fn test_prune_simple() -> Result<(), Error> {
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(1);
+    options.keep.keep_last = Some(1);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@@ -234,7 +234,7 @@ fn test_prune_simple() -> Result<(), Error> {
 
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(0);
+    options.keep.keep_last = Some(0);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@@ -247,8 +247,8 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-last, keep-daily mixed
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_last = Some(2);
+    options.keep.keep_last = Some(2);
-    options.keep_daily = Some(2);
+    options.keep.keep_daily = Some(2);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![];
     assert_eq!(remove_list, expect);
@@ -256,7 +256,7 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-daily test
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_daily = Some(3);
+    options.keep.keep_daily = Some(3);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")];
     assert_eq!(remove_list, expect);
@@ -264,7 +264,7 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-daily test
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_daily = Some(2);
+    options.keep.keep_daily = Some(2);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@@ -275,7 +275,7 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-weekly
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_weekly = Some(5);
+    options.keep.keep_weekly = Some(5);
     let remove_list = get_prune_list(list, false, &options);
     // all backup are within the same week, so we only keep a single file
     let expect: Vec<PathBuf> = vec![
@@ -288,8 +288,8 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-daily + keep-weekly
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_daily = Some(1);
+    options.keep.keep_daily = Some(1);
-    options.keep_weekly = Some(5);
+    options.keep.keep_weekly = Some(5);
     let remove_list = get_prune_list(list, false, &options);
     let expect: Vec<PathBuf> = vec![
         PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@@ -301,7 +301,7 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-monthly
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_monthly = Some(6);
+    options.keep.keep_monthly = Some(6);
     let remove_list = get_prune_list(list, false, &options);
     // all backup are within the same month, so we only keep a single file
     let expect: Vec<PathBuf> = vec![
@@ -314,7 +314,7 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-yearly
     let list = orig_list.clone();
     let mut options = PruneJobOptions::default();
-    options.keep_yearly = Some(7);
+    options.keep.keep_yearly = Some(7);
     let remove_list = get_prune_list(list, false, &options);
     // all backup are within the same year, so we only keep a single file
     let expect: Vec<PathBuf> = vec![
@@ -327,9 +327,9 @@ fn test_prune_simple() -> Result<(), Error> {
     // keep-weekly + keep-monthly + keep-yearly
     let list = orig_list;
     let mut options = PruneJobOptions::default();
-    options.keep_weekly = Some(5);
+    options.keep.keep_weekly = Some(5);
-    options.keep_monthly = Some(6);
+    options.keep.keep_monthly = Some(6);
-    options.keep_yearly = Some(7);
+    options.keep.keep_yearly = Some(7);
     let remove_list = get_prune_list(list, false, &options);
     // all backup are within one week, so we only keep a single file
     let expect: Vec<PathBuf> = vec![