tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

move PruneOptions to pbs_api_types workspace

Browse Source
This commit is contained in:
Dietmar Maurer 2021-09-10 09:21:27 +02:00
parent e7d4be9d85
commit 89725197c0
7 changed files with 160 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
pbs-api-types/src/datastore.rs
View File

@ -119,6 +119,52 @@ pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema =
.minimum(1)
.schema();
#[api(
properties: {
"keep-last": {
schema: PRUNE_SCHEMA_KEEP_LAST,
optional: true,
},
"keep-hourly": {
schema: PRUNE_SCHEMA_KEEP_HOURLY,
optional: true,
},
"keep-daily": {
schema: PRUNE_SCHEMA_KEEP_DAILY,
optional: true,
},
"keep-weekly": {
schema: PRUNE_SCHEMA_KEEP_WEEKLY,
optional: true,
},
"keep-monthly": {
schema: PRUNE_SCHEMA_KEEP_MONTHLY,
optional: true,
},
"keep-yearly": {
schema: PRUNE_SCHEMA_KEEP_YEARLY,
optional: true,
},
}
)]
#[derive(Serialize, Deserialize, Default)]
#[serde(rename_all = "kebab-case")]
/// Common pruning options
pub struct PruneOptions {
#[serde(skip_serializing_if="Option::is_none")]
pub keep_last: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_hourly: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_daily: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_weekly: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_monthly: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_yearly: Option<u64>,
}
#[api(
properties: {
name: {

182
pbs-datastore/src/prune.rs
View File

@ -2,18 +2,8 @@ use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use anyhow::{Error};
use serde::{Deserialize, Serialize};
use proxmox::api::api;
use pbs_api_types::{
PRUNE_SCHEMA_KEEP_LAST,
PRUNE_SCHEMA_KEEP_HOURLY,
PRUNE_SCHEMA_KEEP_DAILY,
PRUNE_SCHEMA_KEEP_WEEKLY,
PRUNE_SCHEMA_KEEP_MONTHLY,
PRUNE_SCHEMA_KEEP_YEARLY,
};
use pbs_api_types::PruneOptions;
use super::BackupInfo;
@ -80,142 +70,52 @@ fn remove_incomplete_snapshots(
}
}
#[api(
properties: {
"keep-last": {
schema: PRUNE_SCHEMA_KEEP_LAST,
optional: true,
},
"keep-hourly": {
schema: PRUNE_SCHEMA_KEEP_HOURLY,
optional: true,
},
"keep-daily": {
schema: PRUNE_SCHEMA_KEEP_DAILY,
optional: true,
},
"keep-weekly": {
schema: PRUNE_SCHEMA_KEEP_WEEKLY,
optional: true,
},
"keep-monthly": {
schema: PRUNE_SCHEMA_KEEP_MONTHLY,
optional: true,
},
"keep-yearly": {
schema: PRUNE_SCHEMA_KEEP_YEARLY,
optional: true,
},
}
)]
#[derive(Serialize, Deserialize, Default)]
#[serde(rename_all = "kebab-case")]
/// Common pruning options
pub struct PruneOptions {
#[serde(skip_serializing_if="Option::is_none")]
pub keep_last: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_hourly: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_daily: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_weekly: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_monthly: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_yearly: Option<u64>,
pub fn keeps_something(options: &PruneOptions) -> bool {
let mut keep_something = false;
if let Some(count) = options.keep_last { if count > 0 { keep_something = true; } }
if let Some(count) = options.keep_hourly { if count > 0 { keep_something = true; } }
if let Some(count) = options.keep_daily { if count > 0 { keep_something = true; } }
if let Some(count) = options.keep_weekly { if count > 0 { keep_something = true; } }
if let Some(count) = options.keep_monthly { if count > 0 { keep_something = true; } }
if let Some(count) = options.keep_yearly { if count > 0 { keep_something = true; } }
keep_something
}
impl PruneOptions {
pub fn cli_options_string(options: &PruneOptions) -> String {
let mut opts = Vec::new();
pub fn new() -> Self {
Self {
keep_last: None,
keep_hourly: None,
keep_daily: None,
keep_weekly: None,
keep_monthly: None,
keep_yearly: None,
if let Some(count) = options.keep_last {
if count > 0 {
opts.push(format!("--keep-last {}", count));
}
}
if let Some(count) = options.keep_hourly {
if count > 0 {
opts.push(format!("--keep-hourly {}", count));
}
}
if let Some(count) = options.keep_daily {
if count > 0 {
opts.push(format!("--keep-daily {}", count));
}
}
if let Some(count) = options.keep_weekly {
if count > 0 {
opts.push(format!("--keep-weekly {}", count));
}
}
if let Some(count) = options.keep_monthly {
if count > 0 {
opts.push(format!("--keep-monthly {}", count));
}
}
if let Some(count) = options.keep_yearly {
if count > 0 {
opts.push(format!("--keep-yearly {}", count));
}
}
pub fn keep_hourly(mut self, value: Option<u64>) -> Self {
self.keep_hourly = value;
self
}
pub fn keep_last(mut self, value: Option<u64>) -> Self {
self.keep_last = value;
self
}
pub fn keep_daily(mut self, value: Option<u64>) -> Self {
self.keep_daily = value;
self
}
pub fn keep_weekly(mut self, value: Option<u64>) -> Self {
self.keep_weekly = value;
self
}
pub fn keep_monthly(mut self, value: Option<u64>) -> Self {
self.keep_monthly = value;
self
}
pub fn keep_yearly(mut self, value: Option<u64>) -> Self {
self.keep_yearly = value;
self
}
pub fn keeps_something(&self) -> bool {
let mut keep_something = false;
if let Some(count) = self.keep_last { if count > 0 { keep_something = true; } }
if let Some(count) = self.keep_hourly { if count > 0 { keep_something = true; } }
if let Some(count) = self.keep_daily { if count > 0 { keep_something = true; } }
if let Some(count) = self.keep_weekly { if count > 0 { keep_something = true; } }
if let Some(count) = self.keep_monthly { if count > 0 { keep_something = true; } }
if let Some(count) = self.keep_yearly { if count > 0 { keep_something = true; } }
keep_something
}
pub fn cli_options_string(&self) -> String {
let mut opts = Vec::new();
if let Some(count) = self.keep_last {
if count > 0 {
opts.push(format!("--keep-last {}", count));
}
}
if let Some(count) = self.keep_hourly {
if count > 0 {
opts.push(format!("--keep-hourly {}", count));
}
}
if let Some(count) = self.keep_daily {
if count > 0 {
opts.push(format!("--keep-daily {}", count));
}
}
if let Some(count) = self.keep_weekly {
if count > 0 {
opts.push(format!("--keep-weekly {}", count));
}
}
if let Some(count) = self.keep_monthly {
if count > 0 {
opts.push(format!("--keep-monthly {}", count));
}
}
if let Some(count) = self.keep_yearly {
if count > 0 {
opts.push(format!("--keep-yearly {}", count));
}
}
opts.join(" ")
}
opts.join(" ")
}
pub fn compute_prune_info(

3
proxmox-backup-client/src/main.rs
View File

@ -29,7 +29,7 @@ use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
use pbs_api_types::{
BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, Authid, CryptMode, GroupListItem,
PruneListItem, SnapshotListItem, StorageStatus, Fingerprint,
PruneListItem, SnapshotListItem, StorageStatus, Fingerprint, PruneOptions,
};
use pbs_client::{
BACKUP_SOURCE_SCHEMA,
@ -72,7 +72,6 @@ use pbs_datastore::manifest::{
ENCRYPTED_KEY_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type,
};
use pbs_datastore::read_chunk::AsyncReadChunk;
use pbs_datastore::prune::PruneOptions;
use pbs_tools::sync::StdChannelWriter;
use pbs_tools::tokio::TokioWriterAdapter;
use pbs_tools::json;

20
src/api2/admin/datastore.rs
View File

@ -26,12 +26,14 @@ use proxmox::{http_err, identity, list_subdirs_api_method, sortable};
use pxar::accessor::aio::Accessor;
use pxar::EntryKind;
use pbs_api_types::{
Authid, BackupContent, Counts, CryptMode, DataStoreListItem, GarbageCollectionStatus,
GroupListItem, SnapshotListItem, SnapshotVerifyState, BACKUP_ARCHIVE_NAME_SCHEMA,
BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA,
IGNORE_VERIFIED_BACKUPS_SCHEMA, UPID_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA,
PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY, PRIV_DATASTORE_READ, PRIV_DATASTORE_PRUNE,
use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode,
DataStoreListItem, GarbageCollectionStatus, GroupListItem,
SnapshotListItem, SnapshotVerifyState, PruneOptions,
BACKUP_ARCHIVE_NAME_SCHEMA, BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA,
BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA,
IGNORE_VERIFIED_BACKUPS_SCHEMA, UPID_SCHEMA,
VERIFICATION_OUTDATED_AFTER_SCHEMA, PRIV_DATASTORE_AUDIT,
PRIV_DATASTORE_MODIFY, PRIV_DATASTORE_READ, PRIV_DATASTORE_PRUNE,
PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_VERIFY,
};
@ -46,7 +48,7 @@ use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader, Lo
use pbs_datastore::fixed_index::{FixedIndexReader};
use pbs_datastore::index::IndexFile;
use pbs_datastore::manifest::{BackupManifest, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME};
use pbs_datastore::prune::{compute_prune_info, PruneOptions};
use pbs_datastore::prune::compute_prune_info;
use pbs_tools::blocking::WrappedReaderStream;
use pbs_tools::stream::{AsyncReaderStream, AsyncChannelWriter};
use pbs_tools::json::{required_integer_param, required_string_param};
@ -838,7 +840,7 @@ pub fn prune(
prune_info.reverse(); // delete older snapshots first
let keep_all = !prune_options.keeps_something();
let keep_all = !pbs_datastore::prune::keeps_something(&prune_options);
if dry_run {
for (info, mut keep) in prune_info {
@ -864,7 +866,7 @@ pub fn prune(
if keep_all {
worker.log("No prune selection - keeping all files.");
} else {
worker.log(format!("retention options: {}", prune_options.cli_options_string()));
worker.log(format!("retention options: {}", pbs_datastore::prune::cli_options_string(&prune_options)));
worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
store, backup_type, backup_id));
}

8
src/bin/proxmox-backup-proxy.rs
View File

@ -32,8 +32,10 @@ use pbs_buildcfg::configdir;
use pbs_systemd::time::{compute_next_event, parse_calendar_event};
use pbs_tools::logrotate::LogRotate;
use pbs_api_types::{Authid, TapeBackupJobConfig, VerificationJobConfig, SyncJobConfig, DataStoreConfig};
use pbs_datastore::prune::PruneOptions;
use pbs_api_types::{
Authid, TapeBackupJobConfig, VerificationJobConfig, SyncJobConfig, DataStoreConfig,
PruneOptions,
};
use proxmox_backup::server;
use proxmox_backup::auth_helpers::*;
@ -487,7 +489,7 @@ async fn schedule_datastore_prune() {
keep_yearly: store_config.keep_yearly,
};
if !prune_options.keeps_something() { // no prune settings - keep all
if !pbs_datastore::prune::keeps_something(&prune_options) { // no prune settings - keep all
continue;
}

8
src/server/prune_job.rs
View File

@ -4,8 +4,8 @@ use anyhow::Error;
use pbs_datastore::{task_log, task_warn};
use pbs_datastore::backup_info::BackupInfo;
use pbs_datastore::prune::{compute_prune_info, PruneOptions};
use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY};
use pbs_datastore::prune::compute_prune_info;
use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY, PruneOptions};
use pbs_config::CachedUserInfo;
use crate::{
@ -28,7 +28,7 @@ pub fn prune_datastore(
task_log!(worker, "(dry test run)");
}
let keep_all = !prune_options.keeps_something();
let keep_all = !pbs_datastore::prune::keeps_something(&prune_options);
if keep_all {
task_log!(worker, "No prune selection - keeping all files.");
@ -36,7 +36,7 @@ pub fn prune_datastore(
task_log!(
worker,
"retention options: {}",
prune_options.cli_options_string()
pbs_datastore::prune::cli_options_string(&prune_options)
);
}

74
tests/prune.rs
View File

@ -1,8 +1,9 @@
use anyhow::{Error};
use std::path::PathBuf;
use pbs_api_types::PruneOptions;
use pbs_datastore::manifest::MANIFEST_BLOB_NAME;
use pbs_datastore::prune::{compute_prune_info, PruneOptions};
use pbs_datastore::prune::compute_prune_info;
use pbs_datastore::{BackupDir, BackupInfo};
fn get_prune_list(
@ -56,7 +57,8 @@ fn test_prune_hourly() -> Result<(), Error> {
orig_list.push(create_info("host/elsa/2019-11-15T11:59:15Z", false));
let list = orig_list.clone();
let options = PruneOptions::new().keep_hourly(Some(3));
let mut options = PruneOptions::default();
options.keep_hourly = Some(3);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-11-15T10:49:15Z"),
@ -66,7 +68,8 @@ fn test_prune_hourly() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list;
let options = PruneOptions::new().keep_hourly(Some(2));
let mut options = PruneOptions::default();
options.keep_hourly = Some(2);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-11-15T10:59:15Z"),
@ -77,7 +80,7 @@ fn test_prune_hourly() -> Result<(), Error> {
Ok(())
}
#[test]
#[test]
fn test_prune_simple2() -> Result<(), Error> {
let mut orig_list = Vec::new();
@ -93,7 +96,8 @@ fn test_prune_simple2() -> Result<(), Error> {
orig_list.push(create_info("host/elsa/2019-12-04T11:59:15Z", false));
let list = orig_list.clone();
let options = PruneOptions::new().keep_daily(Some(1));
let mut options = PruneOptions::default();
options.keep_daily = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-04T11:59:15Z"),
@ -101,7 +105,9 @@ fn test_prune_simple2() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let options = PruneOptions::new().keep_last(Some(1)).keep_daily(Some(1));
let mut options = PruneOptions::default();
options.keep_last = Some(1);
options.keep_daily = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-03T11:59:15Z"),
@ -110,7 +116,9 @@ fn test_prune_simple2() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let options = PruneOptions::new().keep_daily(Some(1)).keep_weekly(Some(1));
let mut options = PruneOptions::default();
options.keep_daily = Some(1);
options.keep_weekly = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-01T11:59:15Z"),
@ -119,7 +127,10 @@ fn test_prune_simple2() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let options = PruneOptions::new().keep_daily(Some(1)).keep_weekly(Some(1)).keep_monthly(Some(1));
let mut options = PruneOptions::default();
options.keep_daily = Some(1);
options.keep_weekly = Some(1);
options.keep_monthly = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-11-22T11:59:15Z"),
@ -129,7 +140,9 @@ fn test_prune_simple2() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list;
let options = PruneOptions::new().keep_monthly(Some(1)).keep_yearly(Some(1));
let mut options = PruneOptions::default();
options.keep_monthly = Some(1);
options.keep_yearly = Some(1);
let remove_list = get_prune_list(list, true, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2018-11-15T11:59:15Z"),
@ -153,13 +166,15 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-last tests
let list = orig_list.clone();
let options = PruneOptions::new().keep_last(Some(4));
let mut options = PruneOptions::default();
options.keep_last = Some(4);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = Vec::new();
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let options = PruneOptions::new().keep_last(Some(3));
let mut options = PruneOptions::default();
options.keep_last = Some(3);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -167,7 +182,8 @@ fn test_prune_simple() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let options = PruneOptions::new().keep_last(Some(2));
let mut options = PruneOptions::default();
options.keep_last = Some(2);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -176,7 +192,8 @@ fn test_prune_simple() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let options = PruneOptions::new().keep_last(Some(1));
let mut options = PruneOptions::default();
options.keep_last = Some(1);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -186,7 +203,8 @@ fn test_prune_simple() -> Result<(), Error> {
assert_eq!(remove_list, expect);
let list = orig_list.clone();
let options = PruneOptions::new().keep_last(Some(0));
let mut options = PruneOptions::default();
options.keep_last = Some(0);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -198,21 +216,25 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-last, keep-daily mixed
let list = orig_list.clone();
let options = PruneOptions::new().keep_last(Some(2)).keep_daily(Some(2));
let mut options = PruneOptions::default();
options.keep_last = Some(2);
options.keep_daily = Some(2);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![];
assert_eq!(remove_list, expect);
// keep-daily test
let list = orig_list.clone();
let options = PruneOptions::new().keep_daily(Some(3));
let mut options = PruneOptions::default();
options.keep_daily = Some(3);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")];
assert_eq!(remove_list, expect);
// keep-daily test
let list = orig_list.clone();
let options = PruneOptions::new().keep_daily(Some(2));
let mut options = PruneOptions::default();
options.keep_daily = Some(2);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -222,7 +244,8 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-weekly
let list = orig_list.clone();
let options = PruneOptions::new().keep_weekly(Some(5));
let mut options = PruneOptions::default();
options.keep_weekly = Some(5);
let remove_list = get_prune_list(list, false, &options);
// all backup are within the same week, so we only keep a single file
let expect: Vec<PathBuf> = vec![
@ -234,7 +257,9 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-daily + keep-weekly
let list = orig_list.clone();
let options = PruneOptions::new().keep_daily(Some(1)).keep_weekly(Some(5));
let mut options = PruneOptions::default();
options.keep_daily = Some(1);
options.keep_weekly = Some(5);
let remove_list = get_prune_list(list, false, &options);
let expect: Vec<PathBuf> = vec![
PathBuf::from("host/elsa/2019-12-02T11:59:15Z"),
@ -245,7 +270,8 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-monthly
let list = orig_list.clone();
let options = PruneOptions::new().keep_monthly(Some(6));
let mut options = PruneOptions::default();
options.keep_monthly = Some(6);
let remove_list = get_prune_list(list, false, &options);
// all backup are within the same month, so we only keep a single file
let expect: Vec<PathBuf> = vec![
@ -257,7 +283,8 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-yearly
let list = orig_list.clone();
let options = PruneOptions::new().keep_yearly(Some(7));
let mut options = PruneOptions::default();
options.keep_yearly = Some(7);
let remove_list = get_prune_list(list, false, &options);
// all backup are within the same year, so we only keep a single file
let expect: Vec<PathBuf> = vec![
@ -269,7 +296,10 @@ fn test_prune_simple() -> Result<(), Error> {
// keep-weekly + keep-monthly + keep-yearly
let list = orig_list;
let options = PruneOptions::new().keep_weekly(Some(5)).keep_monthly(Some(6)).keep_yearly(Some(7));
let mut options = PruneOptions::default();
options.keep_weekly = Some(5);
options.keep_monthly = Some(6);
options.keep_yearly = Some(7);
let remove_list = get_prune_list(list, false, &options);
// all backup are within one week, so we only keep a single file
let expect: Vec<PathBuf> = vec![