From 7320e9ff4bb38afe4656bdb1e8dd4960fdfac481 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 5 Dec 2020 12:53:08 +0100 Subject: [PATCH] tape: add media invenotry --- src/tape/inventory.rs | 644 ++++++++++++++++++++++++++++++++++++++++++ src/tape/mod.rs | 3 + 2 files changed, 647 insertions(+) create mode 100644 src/tape/inventory.rs diff --git a/src/tape/inventory.rs b/src/tape/inventory.rs new file mode 100644 index 00000000..63f85ba2 --- /dev/null +++ b/src/tape/inventory.rs @@ -0,0 +1,644 @@ +//! Backup media Inventory +//! +//! The Inventory persistently stores the list of known backup +//! media. A backup media is identified by its 'MediaId', which is the +//! DriveLabel/MediaSetLabel combination. + +use std::collections::{HashMap, BTreeMap}; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Error}; +use serde::{Serialize, Deserialize}; +use serde_json::json; + +use proxmox::tools::{ + Uuid, + fs::{ + open_file_locked, + replace_file, + file_get_json, + CreateOptions, + }, +}; + +use crate::{ + tools::systemd::time::compute_next_event, + api2::types::{ + MediaSetPolicy, + RetentionPolicy, + }, + tape::{ + MEDIA_POOL_STATUS_DIR, + file_formats::{ + DriveLabel, + MediaSetLabel, + }, + }, +}; + +/// Unique Media Identifier +/// +/// This combines the label and media set label. +#[derive(Debug,Serialize,Deserialize,Clone)] +pub struct MediaId { + pub label: DriveLabel, + #[serde(skip_serializing_if="Option::is_none")] + pub media_set_label: Option, +} + +/// Media Set +/// +/// A List of backup media +#[derive(Debug, Serialize, Deserialize)] +pub struct MediaSet { + /// Unique media set ID + uuid: Uuid, + /// List of BackupMedia + media_list: Vec>, +} + +impl MediaSet { + + pub const MEDIA_SET_MAX_SEQ_NR: u64 = 100; + + pub fn new() -> Self { + let uuid = Uuid::generate(); + Self { + uuid, + media_list: Vec::new(), + } + } + + pub fn with_data(uuid: Uuid, media_list: Vec>) -> Self { + Self { uuid, media_list } + } + + pub fn uuid(&self) -> &Uuid { + &self.uuid + } + + pub fn media_list(&self) -> &[Option] { + &self.media_list + } + + pub fn add_media(&mut self, uuid: Uuid) { + self.media_list.push(Some(uuid)); + } + + pub fn insert_media(&mut self, uuid: Uuid, seq_nr: u64) -> Result<(), Error> { + if seq_nr > Self::MEDIA_SET_MAX_SEQ_NR { + bail!("media set sequence number to large in media set {} ({} > {})", + self.uuid.to_string(), seq_nr, Self::MEDIA_SET_MAX_SEQ_NR); + } + let seq_nr = seq_nr as usize; + if self.media_list.len() > seq_nr { + if self.media_list[seq_nr].is_some() { + bail!("found duplicate squence number in media set '{}/{}'", + self.uuid.to_string(), seq_nr); + } + } else { + self.media_list.resize(seq_nr + 1, None); + } + self.media_list[seq_nr] = Some(uuid); + Ok(()) + } + + pub fn last_media_uuid(&self) -> Option<&Uuid> { + match self.media_list.last() { + None => None, + Some(None) => None, + Some(Some(ref last_uuid)) => Some(last_uuid), + } + } + + pub fn is_last_media(&self, uuid: &Uuid) -> bool { + match self.media_list.last() { + None => false, + Some(None) => false, + Some(Some(last_uuid)) => uuid == last_uuid, + } + } +} + +/// Media Inventory +pub struct Inventory { + map: BTreeMap, + + inventory_path: PathBuf, + lockfile_path: PathBuf, + + // helpers + media_set_start_times: HashMap +} + +impl Inventory { + + pub const MEDIA_INVENTORY_FILENAME: &'static str = "inventory.json"; + pub const MEDIA_INVENTORY_LOCKFILE: &'static str = ".inventory.lck"; + + fn new(base_path: &Path) -> Self { + + let mut inventory_path = base_path.to_owned(); + inventory_path.push(Self::MEDIA_INVENTORY_FILENAME); + + let mut lockfile_path = base_path.to_owned(); + lockfile_path.push(Self::MEDIA_INVENTORY_LOCKFILE); + + Self { + map: BTreeMap::new(), + media_set_start_times: HashMap::new(), + inventory_path, + lockfile_path, + } + } + + pub fn load(base_path: &Path) -> Result { + let mut me = Self::new(base_path); + me.reload()?; + Ok(me) + } + + /// Reload the database + pub fn reload(&mut self) -> Result<(), Error> { + self.map = Self::load_media_db(&self.inventory_path)?; + self.update_helpers(); + Ok(()) + } + + fn update_helpers(&mut self) { + + // recompute media_set_start_times + + let mut set_start_times = HashMap::new(); + + for media in self.map.values() { + let set = match &media.media_set_label { + None => continue, + Some(set) => set, + }; + if set.seq_nr == 0 { + set_start_times.insert(set.uuid.clone(), set.ctime); + } + } + + self.media_set_start_times = set_start_times; + } + + /// Lock the database + pub fn lock(&self) -> Result { + open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true) + } + + fn load_media_db(path: &Path) -> Result, Error> { + + let data = file_get_json(path, Some(json!([])))?; + let media_list: Vec = serde_json::from_value(data)?; + + let mut map = BTreeMap::new(); + for item in media_list.into_iter() { + map.insert(item.label.uuid.clone(), item); + } + + Ok(map) + } + + fn replace_file(&self) -> Result<(), Error> { + let list: Vec<&MediaId> = self.map.values().collect(); + let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?; + let options = CreateOptions::new(); + replace_file(&self.inventory_path, raw.as_bytes(), options)?; + Ok(()) + } + + /// Stores a single MediaID persistently + pub fn store(&mut self, mut media_id: MediaId) -> Result<(), Error> { + let _lock = self.lock()?; + self.map = Self::load_media_db(&self.inventory_path)?; + + // do not overwrite unsaved pool assignments + if media_id.media_set_label.is_none() { + if let Some(previous) = self.map.get(&media_id.label.uuid) { + if let Some(ref set) = previous.media_set_label { + if set.uuid.as_ref() == [0u8;16] { + media_id.media_set_label = Some(set.clone()); + } + } + } + } + + self.map.insert(media_id.label.uuid.clone(), media_id); + self.update_helpers(); + self.replace_file()?; + Ok(()) + } + + /* + /// Same a store, but extract MediaId form MediaLabelInfo + pub fn store_media_info(&mut self, info: &MediaLabelInfo) -> Result<(), Error> { + let media_id = MediaId { + label: info.label.clone(), + media_set_label: info.media_set_label.clone().map(|(l, _)| l), + }; + self.store(media_id) + } + */ + + /// Lookup media + pub fn lookup_media(&self, uuid: &Uuid) -> Option<&MediaId> { + self.map.get(uuid) + } + + /// find media by changer_id + pub fn find_media_by_changer_id(&self, changer_id: &str) -> Option<&MediaId> { + for (_uuid, media_id) in &self.map { + if media_id.label.changer_id == changer_id { + return Some(media_id); + } + } + None + } + + /// Lookup media pool + /// + /// Returns (pool, is_empty) + pub fn lookup_media_pool(&self, uuid: &Uuid) -> Option<(&str, bool)> { + match self.map.get(uuid) { + None => None, + Some(media_id) => { + match media_id.media_set_label { + None => None, // not assigned to any pool + Some(ref set) => { + let is_empty = set.uuid.as_ref() == [0u8;16]; + Some((&set.pool, is_empty)) + } + } + } + } + } + + /// List all media assigned to the pool + pub fn list_pool_media(&self, pool: &str) -> Vec { + let mut list = Vec::new(); + + for (_uuid, media_id) in &self.map { + match media_id.media_set_label { + None => continue, // not assigned to any pool + Some(ref set) => { + if set.pool != pool { + continue; // belong to another pool + } + + if set.uuid.as_ref() == [0u8;16] { // should we do this?? + list.push(MediaId { + label: media_id.label.clone(), + media_set_label: None, + }) + } else { + list.push(media_id.clone()); + } + } + } + + } + + list + } + + /// List all used media + pub fn list_used_media(&self) -> Vec { + let mut list = Vec::new(); + + for (_uuid, media_id) in &self.map { + match media_id.media_set_label { + None => continue, // not assigned to any pool + Some(ref set) => { + if set.uuid.as_ref() != [0u8;16] { + list.push(media_id.clone()); + } + } + } + } + + list + } + + /// List media not assigned to any pool + pub fn list_unassigned_media(&self) -> Vec { + let mut list = Vec::new(); + + for (_uuid, media_id) in &self.map { + if media_id.media_set_label.is_none() { + list.push(media_id.clone()); + } + } + + list + } + + pub fn media_set_start_time(&self, media_set_uuid: &Uuid) -> Option { + self.media_set_start_times.get(media_set_uuid).map(|t| *t) + } + + /// Compute a single media sets + pub fn compute_media_set_members(&self, media_set_uuid: &Uuid) -> Result { + + let mut set = MediaSet::with_data(media_set_uuid.clone(), Vec::new()); + + for media in self.map.values() { + match media.media_set_label { + None => continue, + Some(MediaSetLabel { seq_nr, ref uuid, .. }) => { + if uuid != media_set_uuid { + continue; + } + set.insert_media(media.label.uuid.clone(), seq_nr)?; + } + } + } + + Ok(set) + } + + /// Compute all media sets + pub fn compute_media_set_list(&self) -> Result, Error> { + + let mut set_map: HashMap = HashMap::new(); + + for media in self.map.values() { + match media.media_set_label { + None => continue, + Some(MediaSetLabel { seq_nr, ref uuid, .. }) => { + + let set = set_map.entry(uuid.clone()).or_insert_with(|| { + MediaSet::with_data(uuid.clone(), Vec::new()) + }); + + set.insert_media(media.label.uuid.clone(), seq_nr)?; + } + } + } + + Ok(set_map) + } + + /// Returns the latest media set for a pool + pub fn latest_media_set(&self, pool: &str) -> Option { + + let mut last_set: Option<(Uuid, i64)> = None; + + let set_list = self.map.values() + .filter_map(|media| media.media_set_label.as_ref()) + .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]); + + for set in set_list { + match last_set { + None => { + last_set = Some((set.uuid.clone(), set.ctime)); + } + Some((_, last_ctime)) => { + if set.ctime > last_ctime { + last_set = Some((set.uuid.clone(), set.ctime)); + } + } + } + } + + let (uuid, ctime) = match last_set { + None => return None, + Some((uuid, ctime)) => (uuid, ctime), + }; + + // consistency check - must be the only set with that ctime + let set_list = self.map.values() + .filter_map(|media| media.media_set_label.as_ref()) + .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]); + + for set in set_list { + if set.uuid != uuid && set.ctime >= ctime { // should not happen + eprintln!("latest_media_set: found set with equal ctime ({}, {})", set.uuid, uuid); + return None; + } + } + + Some(uuid) + } + + // Test if there is a media set (in the same pool) newer than this one. + // Return the ctime of the nearest media set + fn media_set_next_start_time(&self, media_set_uuid: &Uuid) -> Option { + + let (pool, ctime) = match self.map.values() + .filter_map(|media| media.media_set_label.as_ref()) + .find_map(|set| { + if &set.uuid == media_set_uuid { + Some((set.pool.clone(), set.ctime)) + } else { + None + } + }) { + Some((pool, ctime)) => (pool, ctime), + None => return None, + }; + + let set_list = self.map.values() + .filter_map(|media| media.media_set_label.as_ref()) + .filter(|set| (&set.uuid != media_set_uuid) && (&set.pool == &pool)); + + let mut next_ctime = None; + + for set in set_list { + if set.ctime > ctime { + match next_ctime { + None => { + next_ctime = Some(set.ctime); + } + Some(last_next_ctime) => { + if set.ctime < last_next_ctime { + next_ctime = Some(set.ctime); + } + } + } + } + } + + next_ctime + } + + pub fn media_expire_time( + &self, + media: &MediaId, + media_set_policy: &MediaSetPolicy, + retention_policy: &RetentionPolicy, + ) -> i64 { + + if let RetentionPolicy::KeepForever = retention_policy { + return i64::MAX; + } + + let set = match media.media_set_label { + None => return i64::MAX, + Some(ref set) => set, + }; + + let set_start_time = match self.media_set_start_time(&set.uuid) { + None => { + // missing information, use ctime from this + // set (always greater than ctime from seq_nr 0) + set.ctime + } + Some(time) => time, + }; + + let max_use_time = match media_set_policy { + MediaSetPolicy::ContinueCurrent => { + match self.media_set_next_start_time(&set.uuid) { + Some(next_start_time) => next_start_time, + None => return i64::MAX, + } + } + MediaSetPolicy::AlwaysCreate => { + set_start_time + 1 + } + MediaSetPolicy::CreateAt(ref event) => { + match compute_next_event(event, set_start_time, false) { + Ok(Some(next)) => next, + Ok(None) | Err(_) => return i64::MAX, + } + } + }; + + match retention_policy { + RetentionPolicy::KeepForever => i64::MAX, + RetentionPolicy::OverwriteAlways => max_use_time, + RetentionPolicy::ProtectFor(time_span) => { + let seconds = f64::from(time_span.clone()) as i64; + max_use_time + seconds + } + } + } + + /// Generate a human readable name for the media set + /// + /// The template can include strftime time format specifications. + pub fn generate_media_set_name( + &self, + media_set_uuid: &Uuid, + template: Option, + ) -> Result { + + if let Some(ctime) = self.media_set_start_time(media_set_uuid) { + let mut template = template.unwrap_or(String::from("%id%")); + template = template.replace("%id%", &media_set_uuid.to_string()); + proxmox::tools::time::strftime_local(&template, ctime) + } else { + // We don't know the set start time, so we cannot use the template + Ok(media_set_uuid.to_string()) + } + } + + // Helpers to simplify testing + + /// Genreate and insert a new free tape (test helper) + pub fn generate_free_tape(&mut self, changer_id: &str, ctime: i64) -> Uuid { + + let label = DriveLabel { + changer_id: changer_id.to_string(), + uuid: Uuid::generate(), + ctime, + }; + let uuid = label.uuid.clone(); + + self.store(MediaId { label, media_set_label: None }).unwrap(); + + uuid + } + + /// Genreate and insert a new tape assigned to a specific pool + /// (test helper) + pub fn generate_assigned_tape( + &mut self, + changer_id: &str, + pool: &str, + ctime: i64, + ) -> Uuid { + + let label = DriveLabel { + changer_id: changer_id.to_string(), + uuid: Uuid::generate(), + ctime, + }; + + let uuid = label.uuid.clone(); + + let set = MediaSetLabel::with_data(pool, [0u8; 16].into(), 0, ctime); + + self.store(MediaId { label, media_set_label: Some(set) }).unwrap(); + + uuid + } + + /// Genreate and insert a used tape (test helper) + pub fn generate_used_tape( + &mut self, + changer_id: &str, + set: MediaSetLabel, + ctime: i64, + ) -> Uuid { + let label = DriveLabel { + changer_id: changer_id.to_string(), + uuid: Uuid::generate(), + ctime, + }; + let uuid = label.uuid.clone(); + + self.store(MediaId { label, media_set_label: Some(set) }).unwrap(); + + uuid + } +} + +// shell completion helper + +/// List of known media uuids +pub fn complete_media_uuid( + _arg: &str, + _param: &HashMap, +) -> Vec { + + let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) { + Ok(inventory) => inventory, + Err(_) => return Vec::new(), + }; + + inventory.map.keys().map(|uuid| uuid.to_string()).collect() +} + +/// List of known media sets +pub fn complete_media_set_uuid( + _arg: &str, + _param: &HashMap, +) -> Vec { + + let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) { + Ok(inventory) => inventory, + Err(_) => return Vec::new(), + }; + + inventory.map.values() + .filter_map(|media| media.media_set_label.as_ref()) + .map(|set| set.uuid.to_string()).collect() +} + +/// List of known media labels (barcodes) +pub fn complete_media_changer_id( + _arg: &str, + _param: &HashMap, +) -> Vec { + + let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) { + Ok(inventory) => inventory, + Err(_) => return Vec::new(), + }; + + inventory.map.values().map(|media| media.label.changer_id.clone()).collect() +} diff --git a/src/tape/mod.rs b/src/tape/mod.rs index 20363329..05f4e2b4 100644 --- a/src/tape/mod.rs +++ b/src/tape/mod.rs @@ -6,6 +6,9 @@ pub use tape_write::*; mod tape_read; pub use tape_read::*; +mod inventory; +pub use inventory::*; + /// Directory path where we stora all status information pub const MEDIA_POOL_STATUS_DIR: &str = "/var/lib/proxmox-backup/mediapool";