tyler/proxmox-backup
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tape: improve locking (lock media-sets)

Browse Source 
- new helper: lock_media_set()

- MediaPool: lock media set

- Expose Inventory::new() to avoid double loading

- do not lock pool on restore (only lock media-set)

- change pool lock name to ".pool-{name}"
This commit is contained in:
Dietmar Maurer
2021-03-22 06:32:18 +01:00
parent e93263be1e
commit 30316192b3
6 changed files with 280 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/tape/inventory.rs
View File

@ -3,10 +3,30 @@
//! The Inventory persistently stores the list of known backup
//! media. A backup media is identified by its 'MediaId', which is the
//! MediaLabel/MediaSetLabel combination.
//!
//! Inventory Locking
//!
//! The inventory itself has several methods to update single entries,
//! but all of them can be considered atomic.
//!
//! Pool Locking
//!
//! To add/modify media assigned to a pool, we always do
//! lock_media_pool(). For unassigned media, we call
//! lock_unassigned_media_pool().
//!
//! MediaSet Locking
//!
//! To add/remove media from a media set, or to modify catalogs we
//! always do lock_media_set(). Also, we aquire this lock during
//! restore, to make sure it is not reused for backups.
//!
use std::collections::{HashMap, BTreeMap};
use std::path::{Path, PathBuf};
use std::os::unix::io::AsRawFd;
use std::fs::File;
use std::time::Duration;
use anyhow::{bail, Error};
use serde::{Serialize, Deserialize};
@ -78,7 +98,8 @@ impl Inventory {
pub const MEDIA_INVENTORY_FILENAME: &'static str = "inventory.json";
pub const MEDIA_INVENTORY_LOCKFILE: &'static str = ".inventory.lck";
fn new(base_path: &Path) -> Self {
/// Create empty instance, no data loaded
pub fn new(base_path: &Path) -> Self {
let mut inventory_path = base_path.to_owned();
inventory_path.push(Self::MEDIA_INVENTORY_FILENAME);
@ -127,7 +148,7 @@ impl Inventory {
}
/// Lock the database
pub fn lock(&self) -> Result<std::fs::File, Error> {
fn lock(&self) -> Result<std::fs::File, Error> {
let file = open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true)?;
if cfg!(test) {
// We cannot use chown inside test environment (no permissions)
@ -733,6 +754,52 @@ impl Inventory {
}
/// Lock a media pool
pub fn lock_media_pool(base_path: &Path, name: &str) -> Result<File, Error> {
let mut path = base_path.to_owned();
path.push(format!(".pool-{}", name));
path.set_extension("lck");
let timeout = std::time::Duration::new(10, 0);
let lock = proxmox::tools::fs::open_file_locked(&path, timeout, true)?;
let backup_user = crate::backup::backup_user()?;
fchown(lock.as_raw_fd(), Some(backup_user.uid), Some(backup_user.gid))?;
Ok(lock)
}
/// Lock for media not assigned to any pool
pub fn lock_unassigned_media_pool(base_path: &Path) -> Result<File, Error> {
// lock artificial "__UNASSIGNED__" pool to avoid races
lock_media_pool(base_path, "__UNASSIGNED__")
}
/// Lock a media set
///
/// Timeout is 10 seconds by default
pub fn lock_media_set(
base_path: &Path,
media_set_uuid: &Uuid,
timeout: Option<Duration>,
) -> Result<File, Error> {
let mut path = base_path.to_owned();
path.push(format!(".media-set-{}", media_set_uuid));
path.set_extension("lck");
let timeout = timeout.unwrap_or(Duration::new(10, 0));
let file = open_file_locked(&path, timeout, true)?;
if cfg!(test) {
// We cannot use chown inside test environment (no permissions)
return Ok(file);
}
let backup_user = crate::backup::backup_user()?;
fchown(file.as_raw_fd(), Some(backup_user.uid), Some(backup_user.gid))?;
Ok(file)
}
// shell completion helper
/// List of known media uuids

210
src/tape/media_pool.rs
View File

@ -8,6 +8,8 @@
//!
use std::path::{PathBuf, Path};
use std::fs::File;
use anyhow::{bail, Error};
use ::serde::{Deserialize, Serialize};
@ -27,6 +29,9 @@ use crate::{
MediaId,
MediaSet,
Inventory,
lock_media_set,
lock_media_pool,
lock_unassigned_media_pool,
file_formats::{
MediaLabel,
MediaSetLabel,
@ -34,9 +39,6 @@ use crate::{
}
};
/// Media Pool lock guard
pub struct MediaPoolLockGuard(std::fs::File);
/// Media Pool
pub struct MediaPool {
@ -49,11 +51,16 @@ pub struct MediaPool {
changer_name: Option<String>,
force_media_availability: bool,
// Set this if you do not need to allocate writeable media - this
// is useful for list_media()
no_media_set_locking: bool,
encrypt_fingerprint: Option<Fingerprint>,
inventory: Inventory,
current_media_set: MediaSet,
current_media_set_lock: Option<File>,
}
impl MediaPool {
@ -72,8 +79,15 @@ impl MediaPool {
retention: RetentionPolicy,
changer_name: Option<String>,
encrypt_fingerprint: Option<Fingerprint>,
no_media_set_locking: bool, // for list_media()
) -> Result<Self, Error> {
let _pool_lock = if no_media_set_locking {
None
} else {
Some(lock_media_pool(state_path, name)?)
};
let inventory = Inventory::load(state_path)?;
let current_media_set = match inventory.latest_media_set(name) {
@ -81,6 +95,12 @@ impl MediaPool {
None => MediaSet::new(),
};
let current_media_set_lock = if no_media_set_locking {
None
} else {
Some(lock_media_set(state_path, current_media_set.uuid(), None)?)
};
Ok(MediaPool {
name: String::from(name),
state_path: state_path.to_owned(),
@ -89,8 +109,10 @@ impl MediaPool {
changer_name,
inventory,
current_media_set,
current_media_set_lock,
encrypt_fingerprint,
force_media_availability: false,
no_media_set_locking,
})
}
@ -111,6 +133,7 @@ impl MediaPool {
state_path: &Path,
config: &MediaPoolConfig,
changer_name: Option<String>,
no_media_set_locking: bool, // for list_media()
) -> Result<Self, Error> {
let allocation = config.allocation.clone().unwrap_or_else(|| String::from("continue")).parse()?;
@ -129,6 +152,7 @@ impl MediaPool {
retention,
changer_name,
encrypt_fingerprint,
no_media_set_locking,
)
}
@ -239,9 +263,20 @@ impl MediaPool {
/// status, so this must not change persistent/saved state.
///
/// Returns the reason why we started a new media set (if we do)
pub fn start_write_session(&mut self, current_time: i64) -> Result<Option<String>, Error> {
pub fn start_write_session(
&mut self,
current_time: i64,
) -> Result<Option<String>, Error> {
let mut create_new_set = match self.current_set_usable() {
let _pool_lock = if self.no_media_set_locking {
None
} else {
Some(lock_media_pool(&self.state_path, &self.name)?)
};
self.inventory.reload()?;
let mut create_new_set = match self.current_set_usable() {
Err(err) => {
Some(err.to_string())
}
@ -268,6 +303,14 @@ impl MediaPool {
if create_new_set.is_some() {
let media_set = MediaSet::new();
let current_media_set_lock = if self.no_media_set_locking {
None
} else {
Some(lock_media_set(&self.state_path, media_set.uuid(), None)?)
};
self.current_media_set_lock = current_media_set_lock;
self.current_media_set = media_set;
}
@ -327,6 +370,10 @@ impl MediaPool {
fn add_media_to_current_set(&mut self, mut media_id: MediaId, current_time: i64) -> Result<(), Error> {
if self.current_media_set_lock.is_none() {
bail!("add_media_to_current_set: media set is not locked - internal error");
}
let seq_nr = self.current_media_set.media_list().len() as u64;
let pool = self.name.clone();
@ -357,6 +404,10 @@ impl MediaPool {
/// Allocates a writable media to the current media set
pub fn alloc_writable_media(&mut self, current_time: i64) -> Result<Uuid, Error> {
if self.current_media_set_lock.is_none() {
bail!("alloc_writable_media: media set is not locked - internal error");
}
let last_is_writable = self.current_set_usable()?;
if last_is_writable {
@ -367,81 +418,95 @@ impl MediaPool {
// try to find empty media in pool, add to media set
let media_list = self.list_media();
{ // limit pool lock scope
let _pool_lock = lock_media_pool(&self.state_path, &self.name)?;
let mut empty_media = Vec::new();
let mut used_media = Vec::new();
self.inventory.reload()?;
for media in media_list.into_iter() {
if !self.location_is_available(media.location()) {
continue;
}
// already part of a media set?
if media.media_set_label().is_some() {
used_media.push(media);
} else {
// only consider writable empty media
if media.status() == &MediaStatus::Writable {
empty_media.push(media);
}
}
}
let media_list = self.list_media();
// sort empty_media, newest first -> oldest last
empty_media.sort_unstable_by(|a, b| {
let mut res = b.label().ctime.cmp(&a.label().ctime);
if res == std::cmp::Ordering::Equal {
res = b.label().label_text.cmp(&a.label().label_text);
}
res
});
let mut empty_media = Vec::new();
let mut used_media = Vec::new();
if let Some(media) = empty_media.pop() {
// found empty media, add to media set an use it
let uuid = media.uuid().clone();
self.add_media_to_current_set(media.into_id(), current_time)?;
return Ok(uuid);
}
println!("no empty media in pool, try to reuse expired media");
let mut expired_media = Vec::new();
for media in used_media.into_iter() {
if let Some(set) = media.media_set_label() {
if &set.uuid == self.current_media_set.uuid() {
for media in media_list.into_iter() {
if !self.location_is_available(media.location()) {
continue;
}
} else {
continue;
// already part of a media set?
if media.media_set_label().is_some() {
used_media.push(media);
} else {
// only consider writable empty media
if media.status() == &MediaStatus::Writable {
empty_media.push(media);
}
}
}
if self.media_is_expired(&media, current_time) {
println!("found expired media on media '{}'", media.label_text());
expired_media.push(media);
}
}
// sort empty_media, newest first -> oldest last
empty_media.sort_unstable_by(|a, b| {
let mut res = b.label().ctime.cmp(&a.label().ctime);
if res == std::cmp::Ordering::Equal {
res = b.label().label_text.cmp(&a.label().label_text);
}
res
});
// sort expired_media, newest first -> oldest last
expired_media.sort_unstable_by(|a, b| {
let mut res = b.media_set_label().unwrap().ctime.cmp(&a.media_set_label().unwrap().ctime);
if res == std::cmp::Ordering::Equal {
res = b.label().label_text.cmp(&a.label().label_text);
if let Some(media) = empty_media.pop() {
// found empty media, add to media set an use it
let uuid = media.uuid().clone();
self.add_media_to_current_set(media.into_id(), current_time)?;
return Ok(uuid);
}
res
});
if let Some(media) = expired_media.pop() {
println!("reuse expired media '{}'", media.label_text());
let uuid = media.uuid().clone();
self.add_media_to_current_set(media.into_id(), current_time)?;
return Ok(uuid);
println!("no empty media in pool, try to reuse expired media");
let mut expired_media = Vec::new();
for media in used_media.into_iter() {
if let Some(set) = media.media_set_label() {
if &set.uuid == self.current_media_set.uuid() {
continue;
}
} else {
continue;
}
if self.media_is_expired(&media, current_time) {
println!("found expired media on media '{}'", media.label_text());
expired_media.push(media);
}
}
// sort expired_media, newest first -> oldest last
expired_media.sort_unstable_by(|a, b| {
let mut res = b.media_set_label().unwrap().ctime.cmp(&a.media_set_label().unwrap().ctime);
if res == std::cmp::Ordering::Equal {
res = b.label().label_text.cmp(&a.label().label_text);
}
res
});
while let Some(media) = expired_media.pop() {
// check if we can modify the media-set (i.e. skip
// media used by a restore job)
if let Ok(_media_set_lock) = lock_media_set(
&self.state_path,
&media.media_set_label().unwrap().uuid,
Some(std::time::Duration::new(0, 0)), // do not wait
) {
println!("reuse expired media '{}'", media.label_text());
let uuid = media.uuid().clone();
self.add_media_to_current_set(media.into_id(), current_time)?;
return Ok(uuid);
}
}
}
println!("no expired media in pool, try to find unassigned/free media");
// try unassigned media
let _lock = Self::lock_unassigned_media_pool(&self.state_path)?;
let _lock = lock_unassigned_media_pool(&self.state_path)?;
self.inventory.reload()?;
@ -561,23 +626,6 @@ impl MediaPool {
self.inventory.generate_media_set_name(media_set_uuid, template)
}
/// Lock the pool
pub fn lock(base_path: &Path, name: &str) -> Result<MediaPoolLockGuard, Error> {
let mut path = base_path.to_owned();
path.push(format!(".{}", name));
path.set_extension("lck");
let timeout = std::time::Duration::new(10, 0);
let lock = proxmox::tools::fs::open_file_locked(&path, timeout, true)?;
Ok(MediaPoolLockGuard(lock))
}
/// Lock for media not assigned to any pool
pub fn lock_unassigned_media_pool(base_path: &Path) -> Result<MediaPoolLockGuard, Error> {
// lock artificial "__UNASSIGNED__" pool to avoid races
MediaPool::lock(base_path, "__UNASSIGNED__")
}
}
/// Backup media