tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

split the namespace out of BackupGroup/Dir api types

Browse Source 
We decided to go this route because it'll most likely be
safer in the API as we need to explicitly add namespaces
support to the various API endpoints this way.

For example, 'pull' should have 2 namespaces: local and
remote, and the GroupFilter (which would otherwise contain
exactly *one* namespace parameter) needs to be applied for
both sides (to decide what to pull from the remote, and what
to *remove* locally as cleanup).

The *datastore* types still contain the namespace and have a
`.backup_ns()` getter.

Note that the datastore's `Display` implementations are no
longer safe to use as a deserializable string.

Additionally, some datastore based methods now have been
exposed via the BackupGroup/BackupDir types to avoid a
"round trip" in code.

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
This commit is contained in:
Wolfgang Bumiller
2022-05-09 15:39:29 +02:00
committed by Thomas Lamprecht
parent 1baf9030ad
commit 133d718fe4
25 changed files with 800 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
pbs-datastore/src/backup_info.rs
View File

@ -1,3 +1,4 @@
use std::convert::TryFrom;
use std::fmt;
use std::os::unix::io::RawFd;
use std::path::PathBuf;
@ -8,11 +9,11 @@ use anyhow::{bail, format_err, Error};
use proxmox_sys::fs::lock_dir_noblock;
use pbs_api_types::{
BackupNamespace, BackupType, GroupFilter, BACKUP_DATE_REGEX, BACKUP_FILE_REGEX,
Authid, BackupNamespace, BackupType, GroupFilter, BACKUP_DATE_REGEX, BACKUP_FILE_REGEX,
};
use pbs_config::{open_backup_lockfile, BackupLockGuard};
use crate::manifest::{MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME};
use crate::manifest::{BackupManifest, MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME};
use crate::{DataBlob, DataStore};
/// BackupGroup is a directory containing a list of BackupDir
@ -20,6 +21,7 @@ use crate::{DataBlob, DataStore};
pub struct BackupGroup {
store: Arc<DataStore>,
ns: BackupNamespace,
group: pbs_api_types::BackupGroup,
}
@ -33,8 +35,12 @@ impl fmt::Debug for BackupGroup {
}
impl BackupGroup {
pub(crate) fn new(store: Arc<DataStore>, group: pbs_api_types::BackupGroup) -> Self {
Self { store, group }
pub(crate) fn new(
store: Arc<DataStore>,
ns: BackupNamespace,
group: pbs_api_types::BackupGroup,
) -> Self {
Self { store, ns, group }
}
/// Access the underlying [`BackupGroup`](pbs_api_types::BackupGroup).
@ -45,7 +51,7 @@ impl BackupGroup {
#[inline]
pub fn backup_ns(&self) -> &BackupNamespace {
&self.group.ns
&self.ns
}
#[inline]
@ -59,11 +65,14 @@ impl BackupGroup {
}
pub fn full_group_path(&self) -> PathBuf {
self.store.base_path().join(self.group.to_string())
self.store.group_path(&self.ns, &self.group)
}
pub fn relative_group_path(&self) -> PathBuf {
self.group.to_string().into()
let mut path = self.store.namespace_path(&self.ns);
path.push(self.group.ty.as_str());
path.push(&self.group.id);
path
}
pub fn list_backups(&self) -> Result<Vec<BackupInfo>, Error> {
@ -205,6 +214,26 @@ impl BackupGroup {
Ok(removed_all_snaps)
}
/// Returns the backup owner.
///
/// The backup owner is the entity who first created the backup group.
pub fn get_owner(&self) -> Result<Authid, Error> {
self.store.get_owner(&self.ns, self.as_ref())
}
/// Set the backup owner.
pub fn set_owner(&self, auth_id: &Authid, force: bool) -> Result<(), Error> {
self.store
.set_owner(&self.ns, &self.as_ref(), auth_id, force)
}
}
impl AsRef<pbs_api_types::BackupNamespace> for BackupGroup {
#[inline]
fn as_ref(&self) -> &pbs_api_types::BackupNamespace {
&self.ns
}
}
impl AsRef<pbs_api_types::BackupGroup> for BackupGroup {
@ -229,7 +258,11 @@ impl From<BackupGroup> for pbs_api_types::BackupGroup {
impl fmt::Display for BackupGroup {
#[inline]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.group, f)
if self.ns.is_root() {
fmt::Display::fmt(&self.group, f)
} else {
write!(f, "[{}]:{}", self.ns, self.group)
}
}
}
@ -237,6 +270,7 @@ impl From<BackupDir> for BackupGroup {
fn from(dir: BackupDir) -> BackupGroup {
BackupGroup {
store: dir.store,
ns: dir.ns,
group: dir.dir.group,
}
}
@ -246,6 +280,7 @@ impl From<&BackupDir> for BackupGroup {
fn from(dir: &BackupDir) -> BackupGroup {
BackupGroup {
store: Arc::clone(&dir.store),
ns: dir.ns.clone(),
group: dir.dir.group.clone(),
}
}
@ -257,6 +292,7 @@ impl From<&BackupDir> for BackupGroup {
#[derive(Clone)]
pub struct BackupDir {
store: Arc<DataStore>,
ns: BackupNamespace,
dir: pbs_api_types::BackupDir,
// backup_time as rfc3339
backup_time_string: String,
@ -279,6 +315,7 @@ impl BackupDir {
Self {
store: unsafe { DataStore::new_test() },
backup_time_string: Self::backup_time_to_string(dir.time).unwrap(),
ns: BackupNamespace::root(),
dir,
}
}
@ -287,6 +324,7 @@ impl BackupDir {
let backup_time_string = Self::backup_time_to_string(backup_time)?;
Ok(Self {
store: group.store,
ns: group.ns,
dir: (group.group, backup_time).into(),
backup_time_string,
})
@ -299,6 +337,7 @@ impl BackupDir {
let backup_time = proxmox_time::parse_rfc3339(&backup_time_string)?;
Ok(Self {
store: group.store,
ns: group.ns,
dir: (group.group, backup_time).into(),
backup_time_string,
})
@ -306,7 +345,7 @@ impl BackupDir {
#[inline]
pub fn backup_ns(&self) -> &BackupNamespace {
&self.dir.group.ns
&self.ns
}
#[inline]
@ -329,20 +368,16 @@ impl BackupDir {
}
pub fn relative_path(&self) -> PathBuf {
format!("{}/{}", self.dir.group, self.backup_time_string).into()
let mut path = self.store.namespace_path(&self.ns);
path.push(self.dir.group.ty.as_str());
path.push(&self.dir.group.id);
path.push(&self.backup_time_string);
path
}
/// Returns the absolute path for backup_dir, using the cached formatted time string.
pub fn full_path(&self) -> PathBuf {
let mut base_path = self.store.base_path();
for ns in self.dir.group.ns.components() {
base_path.push("ns");
base_path.push(ns);
}
base_path.push(self.dir.group.ty.as_str());
base_path.push(&self.dir.group.id);
base_path.push(&self.backup_time_string);
base_path
self.store.snapshot_path(&self.ns, &self.dir)
}
pub fn protected_file(&self) -> PathBuf {
@ -425,6 +460,46 @@ impl BackupDir {
Ok(())
}
/// Get the datastore.
pub fn datastore(&self) -> &Arc<DataStore> {
&self.store
}
/// Returns the backup owner.
///
/// The backup owner is the entity who first created the backup group.
pub fn get_owner(&self) -> Result<Authid, Error> {
self.store.get_owner(&self.ns, self.as_ref())
}
/// Lock the snapshot and open a reader.
pub fn locked_reader(&self) -> Result<crate::SnapshotReader, Error> {
crate::SnapshotReader::new_do(self.clone())
}
/// Load the manifest without a lock. Must not be written back.
pub fn load_manifest(&self) -> Result<(BackupManifest, u64), Error> {
let blob = self.load_blob(MANIFEST_BLOB_NAME)?;
let raw_size = blob.raw_size();
let manifest = BackupManifest::try_from(blob)?;
Ok((manifest, raw_size))
}
/// Update the manifest of the specified snapshot. Never write a manifest directly,
/// only use this method - anything else may break locking guarantees.
pub fn update_manifest(
&self,
update_fn: impl FnOnce(&mut BackupManifest),
) -> Result<(), Error> {
self.store.update_manifest(self, update_fn)
}
}
impl AsRef<pbs_api_types::BackupNamespace> for BackupDir {
fn as_ref(&self) -> &pbs_api_types::BackupNamespace {
&self.ns
}
}
impl AsRef<pbs_api_types::BackupDir> for BackupDir {
@ -465,7 +540,15 @@ impl From<BackupDir> for pbs_api_types::BackupDir {
impl fmt::Display for BackupDir {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}/{}", self.dir.group, self.backup_time_string)
if self.ns.is_root() {
write!(f, "{}/{}", self.dir.group, self.backup_time_string)
} else {
write!(
f,
"[{}]:{}/{}",
self.ns, self.dir.group, self.backup_time_string
)
}
}
}

116
pbs-datastore/src/datastore.rs
View File

@ -1,5 +1,4 @@
use std::collections::{HashMap, HashSet};
use std::convert::TryFrom;
use std::io::{self, Write};
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
@ -350,6 +349,7 @@ impl DataStore {
self.inner.chunk_store.base_path()
}
/// Returns the absolute path for a backup namespace on this datastore
pub fn namespace_path(&self, ns: &BackupNamespace) -> PathBuf {
let mut path = self.base_path();
path.reserve(ns.path_len());
@ -409,23 +409,24 @@ impl DataStore {
Ok(())
}
/// Returns the absolute path for a backup namespace on this datastore
pub fn ns_path(&self, ns: &BackupNamespace) -> PathBuf {
let mut full_path = self.base_path();
full_path.push(ns.path());
full_path
}
/// Returns the absolute path for a backup_group
pub fn group_path(&self, backup_group: &pbs_api_types::BackupGroup) -> PathBuf {
let mut full_path = self.base_path();
pub fn group_path(
&self,
ns: &BackupNamespace,
backup_group: &pbs_api_types::BackupGroup,
) -> PathBuf {
let mut full_path = self.namespace_path(ns);
full_path.push(backup_group.to_string());
full_path
}
/// Returns the absolute path for backup_dir
pub fn snapshot_path(&self, backup_dir: &pbs_api_types::BackupDir) -> PathBuf {
let mut full_path = self.base_path();
pub fn snapshot_path(
&self,
ns: &BackupNamespace,
backup_dir: &pbs_api_types::BackupDir,
) -> PathBuf {
let mut full_path = self.namespace_path(ns);
full_path.push(backup_dir.to_string());
full_path
}
@ -537,9 +538,10 @@ impl DataStore {
/// Returns true if all snapshots were removed, and false if some were protected
pub fn remove_backup_group(
self: &Arc<Self>,
ns: &BackupNamespace,
backup_group: &pbs_api_types::BackupGroup,
) -> Result<bool, Error> {
let backup_group = self.backup_group(backup_group.clone());
let backup_group = self.backup_group(ns.clone(), backup_group.clone());
backup_group.destroy()
}
@ -547,10 +549,11 @@ impl DataStore {
/// Remove a backup directory including all content
pub fn remove_backup_dir(
self: &Arc<Self>,
ns: &BackupNamespace,
backup_dir: &pbs_api_types::BackupDir,
force: bool,
) -> Result<(), Error> {
let backup_dir = self.backup_dir(backup_dir.clone())?;
let backup_dir = self.backup_dir(ns.clone(), backup_dir.clone())?;
backup_dir.destroy(force)
}
@ -560,9 +563,10 @@ impl DataStore {
/// Or None if there is no backup in the group (or the group dir does not exist).
pub fn last_successful_backup(
self: &Arc<Self>,
ns: &BackupNamespace,
backup_group: &pbs_api_types::BackupGroup,
) -> Result<Option<i64>, Error> {
let backup_group = self.backup_group(backup_group.clone());
let backup_group = self.backup_group(ns.clone(), backup_group.clone());
let group_path = backup_group.full_group_path();
@ -573,23 +577,31 @@ impl DataStore {
}
}
/// Return the path of the 'owner' file.
fn owner_path(&self, ns: &BackupNamespace, group: &pbs_api_types::BackupGroup) -> PathBuf {
self.group_path(ns, group).join("owner")
}
/// Returns the backup owner.
///
/// The backup owner is the entity who first created the backup group.
pub fn get_owner(&self, backup_group: &pbs_api_types::BackupGroup) -> Result<Authid, Error> {
let mut full_path = self.base_path();
full_path.push(backup_group.to_string());
full_path.push("owner");
pub fn get_owner(
&self,
ns: &BackupNamespace,
backup_group: &pbs_api_types::BackupGroup,
) -> Result<Authid, Error> {
let full_path = self.owner_path(ns, backup_group);
let owner = proxmox_sys::fs::file_read_firstline(full_path)?;
owner.trim_end().parse() // remove trailing newline
}
pub fn owns_backup(
&self,
ns: &BackupNamespace,
backup_group: &pbs_api_types::BackupGroup,
auth_id: &Authid,
) -> Result<bool, Error> {
let owner = self.get_owner(backup_group)?;
let owner = self.get_owner(ns, backup_group)?;
Ok(check_backup_owner(&owner, auth_id).is_ok())
}
@ -597,13 +609,12 @@ impl DataStore {
/// Set the backup owner.
pub fn set_owner(
&self,
ns: &BackupNamespace,
backup_group: &pbs_api_types::BackupGroup,
auth_id: &Authid,
force: bool,
) -> Result<(), Error> {
let mut path = self.base_path();
path.push(backup_group.to_string());
path.push("owner");
let path = self.owner_path(ns, backup_group);
let mut open_options = std::fs::OpenOptions::new();
open_options.write(true);
@ -633,12 +644,13 @@ impl DataStore {
/// This also acquires an exclusive lock on the directory and returns the lock guard.
pub fn create_locked_backup_group(
&self,
ns: &BackupNamespace,
backup_group: &pbs_api_types::BackupGroup,
auth_id: &Authid,
) -> Result<(Authid, DirLockGuard), Error> {
// create intermediate path first:
let mut full_path = self.base_path();
for ns in backup_group.ns.components() {
for ns in ns.components() {
full_path.push("ns");
full_path.push(ns);
}
@ -655,8 +667,8 @@ impl DataStore {
"backup group",
"another backup is already running",
)?;
self.set_owner(backup_group, auth_id, false)?;
let owner = self.get_owner(backup_group)?; // just to be sure
self.set_owner(ns, backup_group, auth_id, false)?;
let owner = self.get_owner(ns, backup_group)?; // just to be sure
Ok((owner, guard))
}
Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => {
@ -665,7 +677,7 @@ impl DataStore {
"backup group",
"another backup is already running",
)?;
let owner = self.get_owner(backup_group)?; // just to be sure
let owner = self.get_owner(ns, backup_group)?; // just to be sure
Ok((owner, guard))
}
Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err),
@ -677,11 +689,15 @@ impl DataStore {
/// The BackupGroup directory needs to exist.
pub fn create_locked_backup_dir(
&self,
ns: &BackupNamespace,
backup_dir: &pbs_api_types::BackupDir,
) -> Result<(PathBuf, bool, DirLockGuard), Error> {
let relative_path = PathBuf::from(backup_dir.to_string());
let mut full_path = self.base_path();
full_path.push(&relative_path);
let full_path = self.snapshot_path(ns, backup_dir);
let relative_path = full_path.strip_prefix(self.base_path()).map_err(|err| {
format_err!(
"failed to produce correct path for backup {backup_dir} in namespace {ns}: {err}"
)
})?;
let lock = || {
lock_dir_noblock(
@ -692,9 +708,9 @@ impl DataStore {
};
match std::fs::create_dir(&full_path) {
Ok(_) => Ok((relative_path, true, lock()?)),
Ok(_) => Ok((relative_path.to_owned(), true, lock()?)),
Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => {
Ok((relative_path, false, lock()?))
Ok((relative_path.to_owned(), false, lock()?))
}
Err(e) => Err(e.into()),
}
@ -1135,10 +1151,7 @@ impl DataStore {
/// Load the manifest without a lock. Must not be written back.
pub fn load_manifest(&self, backup_dir: &BackupDir) -> Result<(BackupManifest, u64), Error> {
let blob = backup_dir.load_blob(MANIFEST_BLOB_NAME)?;
let raw_size = blob.raw_size();
let manifest = BackupManifest::try_from(blob)?;
Ok((manifest, raw_size))
backup_dir.load_manifest()
}
/// Update the manifest of the specified snapshot. Never write a manifest directly,
@ -1240,8 +1253,12 @@ impl DataStore {
}
/// Open a backup group from this datastore.
pub fn backup_group(self: &Arc<Self>, group: pbs_api_types::BackupGroup) -> BackupGroup {
BackupGroup::new(Arc::clone(&self), group)
pub fn backup_group(
self: &Arc<Self>,
ns: BackupNamespace,
group: pbs_api_types::BackupGroup,
) -> BackupGroup {
BackupGroup::new(Arc::clone(&self), ns, group)
}
/// Open a backup group from this datastore.
@ -1254,19 +1271,25 @@ impl DataStore {
where
T: Into<String>,
{
self.backup_group((ns, ty, id.into()).into())
self.backup_group(ns, (ty, id.into()).into())
}
/*
/// Open a backup group from this datastore by backup group path such as `vm/100`.
///
/// Convenience method for `store.backup_group(path.parse()?)`
pub fn backup_group_from_path(self: &Arc<Self>, path: &str) -> Result<BackupGroup, Error> {
Ok(self.backup_group(path.parse()?))
todo!("split out the namespace");
}
*/
/// Open a snapshot (backup directory) from this datastore.
pub fn backup_dir(self: &Arc<Self>, dir: pbs_api_types::BackupDir) -> Result<BackupDir, Error> {
BackupDir::with_group(self.backup_group(dir.group), dir.time)
pub fn backup_dir(
self: &Arc<Self>,
ns: BackupNamespace,
dir: pbs_api_types::BackupDir,
) -> Result<BackupDir, Error> {
BackupDir::with_group(self.backup_group(ns, dir.group), dir.time)
}
/// Open a snapshot (backup directory) from this datastore.
@ -1280,7 +1303,7 @@ impl DataStore {
where
T: Into<String>,
{
self.backup_dir((ns, ty, id.into(), time).into())
self.backup_dir(ns, (ty, id.into(), time).into())
}
/// Open a snapshot (backup directory) from this datastore with a cached rfc3339 time string.
@ -1292,10 +1315,12 @@ impl DataStore {
BackupDir::with_rfc3339(group, time_string.into())
}
/*
/// Open a snapshot (backup directory) from this datastore by a snapshot path.
pub fn backup_dir_from_path(self: &Arc<Self>, path: &str) -> Result<BackupDir, Error> {
self.backup_dir(path.parse()?)
todo!("split out the namespace");
}
*/
}
/// A iterator for all BackupDir's (Snapshots) in a BackupGroup
@ -1391,7 +1416,8 @@ impl Iterator for ListGroups {
if BACKUP_ID_REGEX.is_match(name) {
return Some(Ok(BackupGroup::new(
Arc::clone(&self.store),
(self.ns.clone(), group_type, name.to_owned()).into(),
self.ns.clone(),
(group_type, name.to_owned()).into(),
)));
}
}

9
pbs-datastore/src/snapshot_reader.rs
View File

@ -8,13 +8,14 @@ use nix::dir::Dir;
use proxmox_sys::fs::lock_dir_noblock_shared;
use pbs_api_types::{BackupNamespace, Operation};
use crate::backup_info::BackupDir;
use crate::dynamic_index::DynamicIndexReader;
use crate::fixed_index::FixedIndexReader;
use crate::index::IndexFile;
use crate::manifest::{archive_type, ArchiveType, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME};
use crate::DataStore;
use pbs_api_types::Operation;
/// Helper to access the contents of a datastore backup snapshot
///
@ -30,10 +31,14 @@ impl SnapshotReader {
/// Lock snapshot, reads the manifest and returns a new instance
pub fn new(
datastore: Arc<DataStore>,
ns: BackupNamespace,
snapshot: pbs_api_types::BackupDir,
) -> Result<Self, Error> {
let snapshot = datastore.backup_dir(snapshot)?;
Self::new_do(datastore.backup_dir(ns, snapshot)?)
}
pub(crate) fn new_do(snapshot: BackupDir) -> Result<Self, Error> {
let datastore = snapshot.datastore();
let snapshot_path = snapshot.full_path();
let locked_dir =