tape: add media invenotry

src/tape/inventory.rs

@@ -0,0 +1,644 @@
+//! Backup media Inventory
+//!
+//! The Inventory persistently stores the list of known backup
+//! media. A backup media is identified by its 'MediaId', which is the
+//! DriveLabel/MediaSetLabel combination.
+
+use std::collections::{HashMap, BTreeMap};
+use std::path::{Path, PathBuf};
+
+use anyhow::{bail, Error};
+use serde::{Serialize, Deserialize};
+use serde_json::json;
+
+use proxmox::tools::{
+    Uuid,
+    fs::{
+        open_file_locked,
+        replace_file,
+        file_get_json,
+        CreateOptions,
+    },
+};
+
+use crate::{
+    tools::systemd::time::compute_next_event,
+    api2::types::{
+        MediaSetPolicy,
+        RetentionPolicy,
+    },
+    tape::{
+        MEDIA_POOL_STATUS_DIR,
+        file_formats::{
+            DriveLabel,
+            MediaSetLabel,
+        },
+    },
+};
+
+/// Unique Media Identifier
+///
+/// This combines the label and media set label.
+#[derive(Debug,Serialize,Deserialize,Clone)]
+pub struct MediaId {
+    pub label: DriveLabel,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub media_set_label: Option<MediaSetLabel>,
+}
+
+/// Media Set
+///
+/// A List of backup media
+#[derive(Debug, Serialize, Deserialize)]
+pub struct MediaSet {
+    /// Unique media set ID
+    uuid: Uuid,
+    /// List of BackupMedia
+    media_list: Vec<Option<Uuid>>,
+}
+
+impl MediaSet {
+
+    pub const MEDIA_SET_MAX_SEQ_NR: u64 = 100;
+
+    pub fn new() -> Self {
+        let uuid = Uuid::generate();
+        Self {
+            uuid,
+            media_list: Vec::new(),
+        }
+    }
+
+    pub fn with_data(uuid: Uuid, media_list: Vec<Option<Uuid>>) -> Self {
+        Self { uuid, media_list }
+    }
+
+    pub fn uuid(&self) -> &Uuid {
+        &self.uuid
+    }
+
+    pub fn media_list(&self) -> &[Option<Uuid>] {
+        &self.media_list
+    }
+
+    pub fn add_media(&mut self, uuid: Uuid) {
+        self.media_list.push(Some(uuid));
+    }
+
+    pub fn insert_media(&mut self, uuid: Uuid, seq_nr: u64) -> Result<(), Error> {
+        if seq_nr > Self::MEDIA_SET_MAX_SEQ_NR {
+            bail!("media set sequence number to large in media set {} ({} > {})",
+                  self.uuid.to_string(), seq_nr, Self::MEDIA_SET_MAX_SEQ_NR);
+        }
+        let seq_nr = seq_nr as usize;
+        if self.media_list.len() > seq_nr {
+            if self.media_list[seq_nr].is_some() {
+                bail!("found duplicate squence number in media set '{}/{}'",
+                      self.uuid.to_string(), seq_nr);
+            }
+        } else {
+            self.media_list.resize(seq_nr + 1, None);
+        }
+        self.media_list[seq_nr] = Some(uuid);
+        Ok(())
+    }
+
+    pub fn last_media_uuid(&self) -> Option<&Uuid> {
+        match self.media_list.last() {
+            None => None,
+            Some(None) => None,
+            Some(Some(ref last_uuid)) => Some(last_uuid),
+        }
+    }
+
+    pub fn is_last_media(&self, uuid: &Uuid) -> bool {
+        match self.media_list.last() {
+            None => false,
+            Some(None) => false,
+            Some(Some(last_uuid)) => uuid == last_uuid,
+        }
+    }
+}
+
+/// Media Inventory
+pub struct Inventory {
+    map: BTreeMap<Uuid, MediaId>,
+
+    inventory_path: PathBuf,
+    lockfile_path: PathBuf,
+
+    // helpers
+    media_set_start_times: HashMap<Uuid, i64>
+}
+
+impl Inventory {
+
+    pub const MEDIA_INVENTORY_FILENAME: &'static str = "inventory.json";
+    pub const MEDIA_INVENTORY_LOCKFILE: &'static str = ".inventory.lck";
+
+    fn new(base_path: &Path) -> Self {
+
+        let mut inventory_path = base_path.to_owned();
+        inventory_path.push(Self::MEDIA_INVENTORY_FILENAME);
+
+        let mut lockfile_path = base_path.to_owned();
+        lockfile_path.push(Self::MEDIA_INVENTORY_LOCKFILE);
+
+        Self {
+            map: BTreeMap::new(),
+            media_set_start_times: HashMap::new(),
+            inventory_path,
+            lockfile_path,
+        }
+    }
+
+    pub fn load(base_path: &Path) -> Result<Self, Error> {
+        let mut me = Self::new(base_path);
+        me.reload()?;
+        Ok(me)
+    }
+
+    /// Reload the database
+    pub fn reload(&mut self) -> Result<(), Error> {
+        self.map = Self::load_media_db(&self.inventory_path)?;
+        self.update_helpers();
+        Ok(())
+    }
+
+    fn update_helpers(&mut self) {
+
+        // recompute media_set_start_times
+
+        let mut set_start_times = HashMap::new();
+
+        for media in self.map.values() {
+            let set = match &media.media_set_label {
+                None => continue,
+                Some(set) => set,
+            };
+            if set.seq_nr == 0 {
+                set_start_times.insert(set.uuid.clone(), set.ctime);
+            }
+        }
+
+        self.media_set_start_times = set_start_times;
+    }
+
+    /// Lock the database
+    pub fn lock(&self) -> Result<std::fs::File, Error> {
+        open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true)
+    }
+
+    fn load_media_db(path: &Path) -> Result<BTreeMap<Uuid, MediaId>, Error> {
+
+        let data = file_get_json(path, Some(json!([])))?;
+        let media_list: Vec<MediaId> = serde_json::from_value(data)?;
+
+        let mut map = BTreeMap::new();
+        for item in media_list.into_iter() {
+            map.insert(item.label.uuid.clone(), item);
+        }
+
+        Ok(map)
+    }
+
+    fn replace_file(&self) -> Result<(), Error> {
+        let list: Vec<&MediaId> = self.map.values().collect();
+        let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?;
+        let options = CreateOptions::new();
+        replace_file(&self.inventory_path, raw.as_bytes(), options)?;
+        Ok(())
+    }
+
+    /// Stores a single MediaID persistently
+    pub fn store(&mut self, mut media_id: MediaId) -> Result<(), Error> {
+        let _lock = self.lock()?;
+        self.map = Self::load_media_db(&self.inventory_path)?;
+
+        // do not overwrite unsaved pool assignments
+        if media_id.media_set_label.is_none() {
+            if let Some(previous) = self.map.get(&media_id.label.uuid) {
+                if let Some(ref set) = previous.media_set_label {
+                    if set.uuid.as_ref() == [0u8;16] {
+                        media_id.media_set_label = Some(set.clone());
+                    }
+                }
+            }
+        }
+
+        self.map.insert(media_id.label.uuid.clone(), media_id);
+        self.update_helpers();
+        self.replace_file()?;
+        Ok(())
+    }
+
+    /*
+    /// Same a store, but extract MediaId form MediaLabelInfo
+    pub fn store_media_info(&mut self, info: &MediaLabelInfo) -> Result<(), Error> {
+        let media_id = MediaId {
+            label: info.label.clone(),
+            media_set_label: info.media_set_label.clone().map(|(l, _)| l),
+        };
+        self.store(media_id)
+    }
+    */
+
+    /// Lookup media
+    pub fn lookup_media(&self, uuid: &Uuid) -> Option<&MediaId> {
+        self.map.get(uuid)
+    }
+
+    /// find media by changer_id
+    pub fn find_media_by_changer_id(&self, changer_id: &str) -> Option<&MediaId> {
+        for (_uuid, media_id) in &self.map {
+            if media_id.label.changer_id == changer_id {
+                return Some(media_id);
+            }
+        }
+        None
+    }
+
+    /// Lookup media pool
+    ///
+    /// Returns (pool, is_empty)
+    pub fn lookup_media_pool(&self, uuid: &Uuid) -> Option<(&str, bool)> {
+        match self.map.get(uuid) {
+            None => None,
+            Some(media_id) => {
+                match media_id.media_set_label {
+                    None => None, // not assigned to any pool
+                    Some(ref set) => {
+                        let is_empty = set.uuid.as_ref() == [0u8;16];
+                        Some((&set.pool, is_empty))
+                    }
+                }
+            }
+        }
+    }
+
+    /// List all media assigned to the pool
+    pub fn list_pool_media(&self, pool: &str) -> Vec<MediaId> {
+        let mut list = Vec::new();
+
+        for (_uuid, media_id) in &self.map {
+            match media_id.media_set_label {
+                None => continue, // not assigned to any pool
+                Some(ref set) => {
+                    if set.pool != pool {
+                        continue; // belong to another pool
+                    }
+
+                    if set.uuid.as_ref() == [0u8;16] { // should we do this??
+                        list.push(MediaId {
+                            label: media_id.label.clone(),
+                            media_set_label: None,
+                        })
+                    } else {
+                        list.push(media_id.clone());
+                    }
+                }
+            }
+
+        }
+
+        list
+    }
+
+    /// List all used media
+    pub fn list_used_media(&self) -> Vec<MediaId> {
+        let mut list = Vec::new();
+
+        for (_uuid, media_id) in &self.map {
+            match media_id.media_set_label {
+                None => continue, // not assigned to any pool
+                Some(ref set) => {
+                    if set.uuid.as_ref() != [0u8;16] {
+                        list.push(media_id.clone());
+                    }
+                }
+            }
+        }
+
+        list
+    }
+
+    /// List media not assigned to any pool
+    pub fn list_unassigned_media(&self) -> Vec<MediaId> {
+        let mut list = Vec::new();
+
+        for (_uuid, media_id) in &self.map {
+            if media_id.media_set_label.is_none() {
+                list.push(media_id.clone());
+            }
+        }
+
+        list
+    }
+
+    pub fn media_set_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
+        self.media_set_start_times.get(media_set_uuid).map(|t| *t)
+    }
+
+    /// Compute a single media sets
+    pub fn compute_media_set_members(&self, media_set_uuid: &Uuid) -> Result<MediaSet, Error> {
+
+        let mut set = MediaSet::with_data(media_set_uuid.clone(), Vec::new());
+
+        for media in self.map.values() {
+            match media.media_set_label {
+                None => continue,
+                Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
+                    if  uuid != media_set_uuid {
+                        continue;
+                    }
+                    set.insert_media(media.label.uuid.clone(), seq_nr)?;
+                }
+            }
+        }
+
+        Ok(set)
+    }
+
+    /// Compute all media sets
+    pub fn compute_media_set_list(&self) -> Result<HashMap<Uuid, MediaSet>, Error> {
+
+        let mut set_map: HashMap<Uuid, MediaSet> = HashMap::new();
+
+        for media in self.map.values() {
+            match media.media_set_label {
+                None => continue,
+                Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
+
+                    let set = set_map.entry(uuid.clone()).or_insert_with(|| {
+                        MediaSet::with_data(uuid.clone(), Vec::new())
+                    });
+
+                    set.insert_media(media.label.uuid.clone(), seq_nr)?;
+                }
+            }
+        }
+
+        Ok(set_map)
+    }
+
+    /// Returns the latest media set for a pool
+    pub fn latest_media_set(&self, pool: &str) -> Option<Uuid> {
+
+        let mut last_set: Option<(Uuid, i64)> = None;
+
+        let set_list = self.map.values()
+            .filter_map(|media| media.media_set_label.as_ref())
+            .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
+
+        for set in set_list {
+            match last_set {
+                None => {
+                    last_set = Some((set.uuid.clone(), set.ctime));
+                }
+                Some((_, last_ctime)) => {
+                    if set.ctime > last_ctime {
+                        last_set = Some((set.uuid.clone(), set.ctime));
+                    }
+                }
+            }
+        }
+
+        let (uuid, ctime) = match last_set {
+            None => return None,
+            Some((uuid, ctime)) => (uuid, ctime),
+        };
+
+        // consistency check - must be the only set with that ctime
+        let set_list = self.map.values()
+            .filter_map(|media| media.media_set_label.as_ref())
+            .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
+
+        for set in set_list {
+            if set.uuid != uuid && set.ctime >= ctime { // should not happen
+                eprintln!("latest_media_set: found set with equal ctime ({}, {})", set.uuid, uuid);
+                return None;
+            }
+        }
+
+        Some(uuid)
+    }
+
+    // Test if there is a media set (in the same pool) newer than this one.
+    // Return the ctime of the nearest media set
+    fn media_set_next_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
+
+        let (pool, ctime) = match self.map.values()
+            .filter_map(|media| media.media_set_label.as_ref())
+            .find_map(|set| {
+                if &set.uuid == media_set_uuid {
+                    Some((set.pool.clone(), set.ctime))
+                } else {
+                    None
+                }
+            }) {
+                Some((pool, ctime)) => (pool, ctime),
+                None => return None,
+            };
+
+        let set_list = self.map.values()
+            .filter_map(|media| media.media_set_label.as_ref())
+            .filter(|set| (&set.uuid != media_set_uuid) && (&set.pool == &pool));
+
+        let mut next_ctime = None;
+
+        for set in set_list {
+            if set.ctime > ctime {
+                match next_ctime {
+                    None => {
+                        next_ctime = Some(set.ctime);
+                    }
+                    Some(last_next_ctime) => {
+                        if set.ctime < last_next_ctime {
+                            next_ctime = Some(set.ctime);
+                        }
+                    }
+                }
+            }
+        }
+
+        next_ctime
+    }
+
+    pub fn media_expire_time(
+        &self,
+        media: &MediaId,
+        media_set_policy: &MediaSetPolicy,
+        retention_policy: &RetentionPolicy,
+    ) -> i64 {
+
+        if let RetentionPolicy::KeepForever = retention_policy {
+            return i64::MAX;
+        }
+
+        let set = match media.media_set_label {
+            None => return i64::MAX,
+            Some(ref set) => set,
+        };
+
+        let set_start_time = match self.media_set_start_time(&set.uuid) {
+            None => {
+                // missing information, use ctime from this
+                // set (always greater than ctime from seq_nr 0)
+                set.ctime
+            }
+            Some(time) => time,
+        };
+
+        let max_use_time = match media_set_policy {
+            MediaSetPolicy::ContinueCurrent => {
+                match self.media_set_next_start_time(&set.uuid) {
+                    Some(next_start_time) => next_start_time,
+                    None => return i64::MAX,
+                }
+            }
+            MediaSetPolicy::AlwaysCreate => {
+                set_start_time + 1
+            }
+            MediaSetPolicy::CreateAt(ref event) => {
+                match compute_next_event(event, set_start_time, false) {
+                    Ok(Some(next)) => next,
+                    Ok(None) | Err(_) => return i64::MAX,
+                }
+            }
+        };
+
+        match retention_policy {
+            RetentionPolicy::KeepForever => i64::MAX,
+            RetentionPolicy::OverwriteAlways => max_use_time,
+            RetentionPolicy::ProtectFor(time_span) => {
+                let seconds = f64::from(time_span.clone()) as i64;
+                max_use_time + seconds
+            }
+        }
+    }
+
+    /// Generate a human readable name for the media set
+    ///
+    /// The template can include strftime time format specifications.
+    pub fn generate_media_set_name(
+        &self,
+        media_set_uuid: &Uuid,
+        template: Option<String>,
+    ) -> Result<String, Error> {
+
+        if let Some(ctime) = self.media_set_start_time(media_set_uuid) {
+            let mut template = template.unwrap_or(String::from("%id%"));
+            template = template.replace("%id%", &media_set_uuid.to_string());
+            proxmox::tools::time::strftime_local(&template, ctime)
+        } else {
+            // We don't know the set start time, so we cannot use the template
+            Ok(media_set_uuid.to_string())
+        }
+    }
+
+    // Helpers to simplify testing
+
+    /// Genreate and insert a new free tape (test helper)
+    pub fn generate_free_tape(&mut self, changer_id: &str, ctime: i64) -> Uuid {
+
+        let label = DriveLabel {
+            changer_id: changer_id.to_string(),
+            uuid: Uuid::generate(),
+            ctime,
+        };
+        let uuid = label.uuid.clone();
+
+        self.store(MediaId { label, media_set_label: None }).unwrap();
+
+        uuid
+    }
+
+    /// Genreate and insert a new tape assigned to a specific pool
+    /// (test helper)
+    pub fn generate_assigned_tape(
+        &mut self,
+        changer_id: &str,
+        pool: &str,
+        ctime: i64,
+    ) -> Uuid {
+
+        let label = DriveLabel {
+            changer_id: changer_id.to_string(),
+            uuid: Uuid::generate(),
+            ctime,
+        };
+
+        let uuid = label.uuid.clone();
+
+        let set = MediaSetLabel::with_data(pool, [0u8; 16].into(), 0, ctime);
+
+        self.store(MediaId { label, media_set_label: Some(set) }).unwrap();
+
+        uuid
+    }
+
+    /// Genreate and insert a used tape (test helper)
+    pub fn generate_used_tape(
+        &mut self,
+        changer_id: &str,
+        set: MediaSetLabel,
+        ctime: i64,
+    ) -> Uuid {
+        let label = DriveLabel {
+            changer_id: changer_id.to_string(),
+            uuid: Uuid::generate(),
+            ctime,
+        };
+        let uuid = label.uuid.clone();
+
+        self.store(MediaId { label, media_set_label: Some(set) }).unwrap();
+
+        uuid
+    }
+}
+
+// shell completion helper
+
+/// List of known media uuids
+pub fn complete_media_uuid(
+    _arg: &str,
+    _param: &HashMap<String, String>,
+) -> Vec<String> {
+
+    let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) {
+        Ok(inventory) => inventory,
+        Err(_) => return Vec::new(),
+    };
+
+    inventory.map.keys().map(|uuid| uuid.to_string()).collect()
+}
+
+/// List of known media sets
+pub fn complete_media_set_uuid(
+    _arg: &str,
+    _param: &HashMap<String, String>,
+) -> Vec<String> {
+
+    let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) {
+        Ok(inventory) => inventory,
+        Err(_) => return Vec::new(),
+    };
+
+    inventory.map.values()
+        .filter_map(|media| media.media_set_label.as_ref())
+        .map(|set| set.uuid.to_string()).collect()
+}
+
+/// List of known media labels (barcodes)
+pub fn complete_media_changer_id(
+    _arg: &str,
+    _param: &HashMap<String, String>,
+) -> Vec<String> {
+
+    let inventory = match Inventory::load(Path::new(MEDIA_POOL_STATUS_DIR)) {
+        Ok(inventory) => inventory,
+        Err(_) => return Vec::new(),
+    };
+
+    inventory.map.values().map(|media| media.label.changer_id.clone()).collect()
+}

src/tape/mod.rs

@@ -6,6 +6,9 @@ pub use tape_write::*;
 mod tape_read;
 pub use tape_read::*;
 
+mod inventory;
+pub use inventory::*;
+
 /// Directory path where we stora all status information
 pub const MEDIA_POOL_STATUS_DIR: &str = "/var/lib/proxmox-backup/mediapool";